Fix CE not processing alert status

Fixes #2968
This commit is contained in:
Owen
2026-05-02 13:38:05 -07:00
parent e1afbc226c
commit b8822b4d25
22 changed files with 729 additions and 807 deletions

View File

@@ -1,27 +1,153 @@
// stub
import logger from "@server/logger";
import { processAlerts } from "@server/lib/alerts";
import {
db,
statusHistory,
targetHealthCheck,
targets,
resources,
Transaction,
logsDb
} from "@server/db";
import { eq } from "drizzle-orm";
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
import {
fireResourceDegradedAlert,
fireResourceHealthyAlert,
fireResourceUnhealthyAlert,
fireResourceUnknownAlert
} from "./resourceEvents";
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
* Fire a `health_check_healthy` alert for the given health check.
*
* Call this after a previously-failing health check has recovered so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the health check.
* @param healthCheckId - Numeric primary key of the health check.
* @param healthCheckName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireHealthCheckHealthyAlert(
orgId: string,
healthCheckId: number,
healthCheckName?: string,
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx?: unknown
trx: Transaction | typeof db = db
): Promise<void> {
return;
try {
await logsDb.insert(statusHistory).values({
entityType: "health_check",
entityId: healthCheckId,
orgId: orgId,
status: "healthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("health_check", healthCheckId);
await handleResource(orgId, healthCheckTargetId, send, trx);
if (!send) {
return;
}
await processAlerts({
eventType: "health_check_healthy",
orgId,
healthCheckId,
data: {
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
await processAlerts({
eventType: "health_check_toggle",
orgId,
healthCheckId,
data: {
healthCheckId,
status: "healthy",
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireHealthCheckHealthyAlert: unexpected error for healthCheckId ${healthCheckId}`,
err
);
}
}
/**
* Fire a `health_check_unhealthy` alert for the given health check.
*
* Call this after a health check has been detected as failing so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the health check.
* @param healthCheckId - Numeric primary key of the health check.
* @param healthCheckName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireHealthCheckUnhealthyAlert(
orgId: string,
healthCheckId: number,
healthCheckName?: string,
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx?: unknown
trx: Transaction | typeof db = db
): Promise<void> {
return;
try {
await logsDb.insert(statusHistory).values({
entityType: "health_check",
entityId: healthCheckId,
orgId: orgId,
status: "unhealthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("health_check", healthCheckId);
await handleResource(orgId, healthCheckTargetId, send, trx);
if (!send) {
return;
}
await processAlerts({
eventType: "health_check_unhealthy",
orgId,
healthCheckId,
data: {
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
await processAlerts({
eventType: "health_check_toggle",
orgId,
healthCheckId,
data: {
healthCheckId,
status: "unhealthy",
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireHealthCheckUnhealthyAlert: unexpected error for healthCheckId ${healthCheckId}`,
err
);
}
}
export async function fireHealthCheckUnknownAlert(
@@ -31,7 +157,137 @@ export async function fireHealthCheckUnknownAlert(
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx?: unknown
trx: Transaction | typeof db = db
): Promise<void> {
return;
try {
await logsDb.insert(statusHistory).values({
entityType: "health_check",
entityId: healthCheckId,
orgId: orgId,
status: "unknown",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("health_check", healthCheckId);
await handleResource(orgId, healthCheckTargetId, send, trx);
if (!send) {
return;
}
} catch (err) {
logger.error(
`fireHealthCheckUnknownAlert: unexpected error for healthCheckId ${healthCheckId}`,
err
);
}
}
async function handleResource(
orgId: string,
healthCheckTargetId?: number | null,
send: boolean = true,
trx: Transaction | typeof db = db
) {
if (!healthCheckTargetId) {
return;
}
// we have targets lets get them
const [target] = await trx
.select()
.from(targets)
.where(eq(targets.targetId, healthCheckTargetId))
.limit(1);
if (!target) {
return;
}
const [resource] = await trx
.select()
.from(resources)
.where(eq(resources.resourceId, target.resourceId))
.limit(1);
if (!resource) {
return;
}
const otherTargets = await trx
.select({ hcHealth: targetHealthCheck.hcHealth })
.from(targets)
.innerJoin(
targetHealthCheck,
eq(targetHealthCheck.targetId, targets.targetId)
)
.where(eq(targets.resourceId, resource.resourceId));
let health = "healthy";
const allUnknown = otherTargets.every((t) => t.hcHealth === "unknown");
const allHealthy = otherTargets.every((t) => t.hcHealth === "healthy");
const allUnhealthy = otherTargets.every((t) => t.hcHealth === "unhealthy");
if (allUnknown) {
logger.debug(
`Marking resource ${resource.resourceId} as unknown because all health checks are disabled`
);
health = "unknown";
} else if (allHealthy) {
health = "healthy";
} else if (allUnhealthy) {
logger.debug(
`Marking resource ${resource.resourceId} as unhealthy because all targets are unhealthy`
);
health = "unhealthy";
} else {
logger.debug(
`Marking resource ${resource.resourceId} as degraded because some targets are unhealthy`
);
health = "degraded";
}
if (health != resource.health) {
// it changed
await trx
.update(resources)
.set({ health })
.where(eq(resources.resourceId, resource.resourceId));
if (health === "unknown") {
await fireResourceUnknownAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
} else if (health === "unhealthy") {
await fireResourceUnhealthyAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
} else if (health === "healthy") {
await fireResourceHealthyAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
} else if (health === "degraded") {
await fireResourceDegradedAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
}
}
}

View File

@@ -1,26 +1,243 @@
import logger from "@server/logger";
import { processAlerts } from "@server/lib/alerts";
import { db, logsDb, statusHistory, Transaction } from "@server/db";
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
* Fire a `resource_healthy` alert for the given resource.
*
* Call this after a previously-unhealthy resource has recovered so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceHealthyAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx?: unknown
): Promise<void> {}
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "healthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
if (!send) {
return;
}
await processAlerts({
eventType: "resource_healthy",
orgId,
resourceId,
data: {
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "healthy",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceHealthyAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}
/**
* Fire a `resource_unhealthy` alert for the given resource.
*
* Call this after a resource has been detected as unhealthy so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceUnhealthyAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx?: unknown
): Promise<void> {}
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "unhealthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
export async function fireResourceToggleAlert(
if (!send) {
return;
}
await processAlerts({
eventType: "resource_unhealthy",
orgId,
resourceId,
data: {
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "unhealthy",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceUnhealthyAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}
/**
* Fire a `resource_degraded` alert for the given resource.
*
* Call this after a resource has been detected as degraded so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceDegradedAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx?: unknown
): Promise<void> {}
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "degraded",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
if (!send) {
return;
}
await processAlerts({
eventType: "resource_degraded",
orgId,
resourceId,
data: {
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "degraded",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceDegradedAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}
/**
* Fire a `resource_unknown` alert for the given resource.
*
* Call this when all health checks on a resource are disabled so that the
* resource status transitions to unknown.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceUnknownAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "unknown",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
if (!send) {
return;
}
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "unknown",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceUnknownAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}

View File

@@ -1,21 +1,156 @@
// stub
import logger from "@server/logger";
import { processAlerts } from "@server/lib/alerts";
import {
db,
logsDb,
statusHistory,
targetHealthCheck,
Transaction
} from "@server/db";
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
import { and, eq, inArray } from "drizzle-orm";
import { fireHealthCheckUnhealthyAlert } from "./healthCheckEvents";
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
* Fire a `site_online` alert for the given site.
*
* Call this after the site has been confirmed reachable / connected so that
* any matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the site.
* @param siteId - Numeric primary key of the site.
* @param siteName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireSiteOnlineAlert(
orgId: string,
siteId: number,
siteName?: string,
extra?: Record<string, unknown>,
trx?: unknown
trx: Transaction | typeof db = db
): Promise<void> {
return;
try {
await logsDb.insert(statusHistory).values({
entityType: "site",
entityId: siteId,
orgId: orgId,
status: "online",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("site", siteId);
await processAlerts({
eventType: "site_online",
orgId,
siteId,
data: {
...(siteName != null ? { siteName } : {}),
...extra
}
});
await processAlerts({
eventType: "site_toggle",
orgId,
siteId,
data: {
siteId,
status: "online",
...(siteName != null ? { siteName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireSiteOnlineAlert: unexpected error for siteId ${siteId}`,
err
);
}
}
/**
* Fire a `site_offline` alert for the given site.
*
* Call this after the site has been detected as unreachable / disconnected so
* that any matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the site.
* @param siteId - Numeric primary key of the site.
* @param siteName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireSiteOfflineAlert(
orgId: string,
siteId: number,
siteName?: string,
extra?: Record<string, unknown>,
trx?: unknown
trx: Transaction | typeof db = db
): Promise<void> {
return;
}
try {
await logsDb.insert(statusHistory).values({
entityType: "site",
entityId: siteId,
orgId: orgId,
status: "offline",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("site", siteId);
const unhealthyHealthChecks = await trx
.update(targetHealthCheck)
.set({ hcHealth: "unhealthy" })
.where(
and(
eq(targetHealthCheck.orgId, orgId),
eq(targetHealthCheck.siteId, siteId),
eq(targetHealthCheck.hcEnabled, true) // only effect the ones that are enabled
)
)
.returning();
for (const healthCheck of unhealthyHealthChecks) {
logger.info(
`Marking health check ${healthCheck.targetHealthCheckId} unhealthy due to site ${siteId} being marked offline`
);
await fireHealthCheckUnhealthyAlert(
healthCheck.orgId,
healthCheck.targetHealthCheckId,
healthCheck.name,
healthCheck.targetId, // for the resource if we have one
undefined,
true,
trx
);
}
await processAlerts({
eventType: "site_offline",
orgId,
siteId,
data: {
...(siteName != null ? { siteName } : {}),
...extra
}
});
await processAlerts({
eventType: "site_toggle",
orgId,
siteId,
data: {
siteId,
status: "offline",
...(siteName != null ? { siteName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireSiteOfflineAlert: unexpected error for siteId ${siteId}`,
err
);
}
}

View File

@@ -1,3 +1,4 @@
export * from "./events/siteEvents";
export * from "./events/healthCheckEvents";
export * from "./events/resourceEvents";
export * from "./processAlerts";

View File

@@ -0,0 +1,5 @@
import { AlertContext } from "@server/routers/alertRule/types";
export async function processAlerts(context: AlertContext): Promise<void> {
return;
}

View File

@@ -1,8 +1,9 @@
export async function getOrgTierData(
orgId: string
): Promise<{ tier: string | null; active: boolean }> {
): Promise<{ tier: string | null; active: boolean; isTrial: boolean }> {
const tier = null;
const active = false;
const isTrial = false;
return { tier, active };
return { tier, active, isTrial };
}

View File

@@ -34,7 +34,7 @@ import { hashPassword } from "@server/auth/password";
import { isValidCIDR, isValidIP, isValidUrlGlobPattern } from "../validators";
import { isValidRegionId } from "@server/db/regions";
import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed";
import { fireHealthCheckUnknownAlert } from "#dynamic/lib/alerts";
import { fireHealthCheckUnknownAlert } from "@server/lib/alerts";
import { tierMatrix } from "../billing/tierMatrix";
export type ProxyResourcesResults = {
@@ -165,7 +165,8 @@ export async function updateProxyResources(
hcStatus: healthcheckData?.status,
hcHealth: "unknown",
hcHealthyThreshold: healthcheckData?.["healthy-threshold"],
hcUnhealthyThreshold: healthcheckData?.["unhealthy-threshold"]
hcUnhealthyThreshold:
healthcheckData?.["unhealthy-threshold"]
})
.returning();
@@ -544,8 +545,10 @@ export async function updateProxyResources(
healthcheckData?.["follow-redirects"],
hcMethod: healthcheckData?.method,
hcStatus: healthcheckData?.status,
hcHealthyThreshold: healthcheckData?.["healthy-threshold"],
hcUnhealthyThreshold: healthcheckData?.["unhealthy-threshold"]
hcHealthyThreshold:
healthcheckData?.["healthy-threshold"],
hcUnhealthyThreshold:
healthcheckData?.["unhealthy-threshold"]
})
.where(
eq(
@@ -1120,8 +1123,10 @@ function checkIfHealthcheckChanged(
JSON.stringify(incoming.hcHeaders)
)
return true;
if (existing.hcHealthyThreshold !== incoming.hcHealthyThreshold) return true;
if (existing.hcUnhealthyThreshold !== incoming.hcUnhealthyThreshold) return true;
if (existing.hcHealthyThreshold !== incoming.hcHealthyThreshold)
return true;
if (existing.hcUnhealthyThreshold !== incoming.hcUnhealthyThreshold)
return true;
return false;
}
@@ -1184,7 +1189,11 @@ async function getDomainId(
orgId: string,
fullDomain: string,
trx: Transaction
): Promise<{ subdomain: string | null; domainId: string; wildcard: boolean } | null> {
): Promise<{
subdomain: string | null;
domainId: string;
wildcard: boolean;
} | null> {
const isWildcardFullDomain = fullDomain.startsWith("*.");
const possibleDomains = await trx

View File

@@ -1,306 +0,0 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
import logger from "@server/logger";
import { processAlerts } from "../processAlerts";
import {
db,
statusHistory,
targetHealthCheck,
targets,
resources,
Transaction,
logsDb
} from "@server/db";
import { eq } from "drizzle-orm";
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
import {
fireResourceDegradedAlert,
fireResourceHealthyAlert,
fireResourceUnhealthyAlert,
fireResourceUnknownAlert
} from "./resourceEvents";
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
* Fire a `health_check_healthy` alert for the given health check.
*
* Call this after a previously-failing health check has recovered so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the health check.
* @param healthCheckId - Numeric primary key of the health check.
* @param healthCheckName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireHealthCheckHealthyAlert(
orgId: string,
healthCheckId: number,
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "health_check",
entityId: healthCheckId,
orgId: orgId,
status: "healthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("health_check", healthCheckId);
await handleResource(orgId, healthCheckTargetId, send, trx);
if (!send) {
return;
}
await processAlerts({
eventType: "health_check_healthy",
orgId,
healthCheckId,
data: {
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
await processAlerts({
eventType: "health_check_toggle",
orgId,
healthCheckId,
data: {
healthCheckId,
status: "healthy",
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireHealthCheckHealthyAlert: unexpected error for healthCheckId ${healthCheckId}`,
err
);
}
}
/**
* Fire a `health_check_unhealthy` alert for the given health check.
*
* Call this after a health check has been detected as failing so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the health check.
* @param healthCheckId - Numeric primary key of the health check.
* @param healthCheckName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireHealthCheckUnhealthyAlert(
orgId: string,
healthCheckId: number,
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "health_check",
entityId: healthCheckId,
orgId: orgId,
status: "unhealthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("health_check", healthCheckId);
await handleResource(orgId, healthCheckTargetId, send, trx);
if (!send) {
return;
}
await processAlerts({
eventType: "health_check_unhealthy",
orgId,
healthCheckId,
data: {
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
await processAlerts({
eventType: "health_check_toggle",
orgId,
healthCheckId,
data: {
healthCheckId,
status: "unhealthy",
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireHealthCheckUnhealthyAlert: unexpected error for healthCheckId ${healthCheckId}`,
err
);
}
}
export async function fireHealthCheckUnknownAlert(
orgId: string,
healthCheckId: number,
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "health_check",
entityId: healthCheckId,
orgId: orgId,
status: "unknown",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("health_check", healthCheckId);
await handleResource(orgId, healthCheckTargetId, send, trx);
if (!send) {
return;
}
} catch (err) {
logger.error(
`fireHealthCheckUnknownAlert: unexpected error for healthCheckId ${healthCheckId}`,
err
);
}
}
async function handleResource(
orgId: string,
healthCheckTargetId?: number | null,
send: boolean = true,
trx: Transaction | typeof db = db
) {
if (!healthCheckTargetId) {
return;
}
// we have targets lets get them
const [target] = await trx
.select()
.from(targets)
.where(eq(targets.targetId, healthCheckTargetId))
.limit(1);
if (!target) {
return;
}
const [resource] = await trx
.select()
.from(resources)
.where(eq(resources.resourceId, target.resourceId))
.limit(1);
if (!resource) {
return;
}
const otherTargets = await trx
.select({ hcHealth: targetHealthCheck.hcHealth })
.from(targets)
.innerJoin(
targetHealthCheck,
eq(targetHealthCheck.targetId, targets.targetId)
)
.where(eq(targets.resourceId, resource.resourceId));
let health = "healthy";
const allUnknown = otherTargets.every((t) => t.hcHealth === "unknown");
const allHealthy = otherTargets.every((t) => t.hcHealth === "healthy");
const allUnhealthy = otherTargets.every((t) => t.hcHealth === "unhealthy");
if (allUnknown) {
logger.debug(
`Marking resource ${resource.resourceId} as unknown because all health checks are disabled`
);
health = "unknown";
} else if (allHealthy) {
health = "healthy";
} else if (allUnhealthy) {
logger.debug(
`Marking resource ${resource.resourceId} as unhealthy because all targets are unhealthy`
);
health = "unhealthy";
} else {
logger.debug(
`Marking resource ${resource.resourceId} as degraded because some targets are unhealthy`
);
health = "degraded";
}
if (health != resource.health) {
// it changed
await trx
.update(resources)
.set({ health })
.where(eq(resources.resourceId, resource.resourceId));
if (health === "unknown") {
await fireResourceUnknownAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
} else if (health === "unhealthy") {
await fireResourceUnhealthyAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
} else if (health === "healthy") {
await fireResourceHealthyAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
} else if (health === "degraded") {
await fireResourceDegradedAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
}
}
}

View File

@@ -1,256 +0,0 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
import logger from "@server/logger";
import { processAlerts } from "../processAlerts";
import { db, logsDb, statusHistory, Transaction } from "@server/db";
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
* Fire a `resource_healthy` alert for the given resource.
*
* Call this after a previously-unhealthy resource has recovered so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceHealthyAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "healthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
if (!send) {
return;
}
await processAlerts({
eventType: "resource_healthy",
orgId,
resourceId,
data: {
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "healthy",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceHealthyAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}
/**
* Fire a `resource_unhealthy` alert for the given resource.
*
* Call this after a resource has been detected as unhealthy so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceUnhealthyAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "unhealthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
if (!send) {
return;
}
await processAlerts({
eventType: "resource_unhealthy",
orgId,
resourceId,
data: {
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "unhealthy",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceUnhealthyAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}
/**
* Fire a `resource_degraded` alert for the given resource.
*
* Call this after a resource has been detected as degraded so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceDegradedAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "degraded",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
if (!send) {
return;
}
await processAlerts({
eventType: "resource_degraded",
orgId,
resourceId,
data: {
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "degraded",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceDegradedAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}
/**
* Fire a `resource_unknown` alert for the given resource.
*
* Call this when all health checks on a resource are disabled so that the
* resource status transitions to unknown.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceUnknownAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "unknown",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
if (!send) {
return;
}
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "unknown",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceUnknownAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}

View File

@@ -1,169 +0,0 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
import logger from "@server/logger";
import { processAlerts } from "../processAlerts";
import {
db,
logsDb,
statusHistory,
targetHealthCheck,
Transaction
} from "@server/db";
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
import { and, eq, inArray } from "drizzle-orm";
import { fireHealthCheckUnhealthyAlert } from "./healthCheckEvents";
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
* Fire a `site_online` alert for the given site.
*
* Call this after the site has been confirmed reachable / connected so that
* any matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the site.
* @param siteId - Numeric primary key of the site.
* @param siteName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireSiteOnlineAlert(
orgId: string,
siteId: number,
siteName?: string,
extra?: Record<string, unknown>,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "site",
entityId: siteId,
orgId: orgId,
status: "online",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("site", siteId);
await processAlerts({
eventType: "site_online",
orgId,
siteId,
data: {
...(siteName != null ? { siteName } : {}),
...extra
}
});
await processAlerts({
eventType: "site_toggle",
orgId,
siteId,
data: {
siteId,
status: "online",
...(siteName != null ? { siteName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireSiteOnlineAlert: unexpected error for siteId ${siteId}`,
err
);
}
}
/**
* Fire a `site_offline` alert for the given site.
*
* Call this after the site has been detected as unreachable / disconnected so
* that any matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the site.
* @param siteId - Numeric primary key of the site.
* @param siteName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireSiteOfflineAlert(
orgId: string,
siteId: number,
siteName?: string,
extra?: Record<string, unknown>,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "site",
entityId: siteId,
orgId: orgId,
status: "offline",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("site", siteId);
const unhealthyHealthChecks = await trx
.update(targetHealthCheck)
.set({ hcHealth: "unhealthy" })
.where(
and(
eq(targetHealthCheck.orgId, orgId),
eq(targetHealthCheck.siteId, siteId),
eq(targetHealthCheck.hcEnabled, true) // only effect the ones that are enabled
)
)
.returning();
for (const healthCheck of unhealthyHealthChecks) {
logger.info(
`Marking health check ${healthCheck.targetHealthCheckId} unhealthy due to site ${siteId} being marked offline`
);
await fireHealthCheckUnhealthyAlert(
healthCheck.orgId,
healthCheck.targetHealthCheckId,
healthCheck.name,
healthCheck.targetId, // for the resource if we have one
undefined,
true,
trx
);
}
await processAlerts({
eventType: "site_offline",
orgId,
siteId,
data: {
...(siteName != null ? { siteName } : {}),
...extra
}
});
await processAlerts({
eventType: "site_toggle",
orgId,
siteId,
data: {
siteId,
status: "offline",
...(siteName != null ? { siteName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireSiteOfflineAlert: unexpected error for siteId ${siteId}`,
err
);
}
}

View File

@@ -14,6 +14,3 @@
export * from "./processAlerts";
export * from "./sendAlertWebhook";
export * from "./sendAlertEmail";
export * from "./events/siteEvents";
export * from "./events/healthCheckEvents";
export * from "./events/resourceEvents";

View File

@@ -24,7 +24,7 @@ import { eq, and } from "drizzle-orm";
import {
fireHealthCheckHealthyAlert,
fireHealthCheckUnhealthyAlert
} from "#private/lib/alerts/events/healthCheckEvents";
} from "@server/lib/alerts";
const paramsSchema = z.strictObject({
orgId: z.string().nonempty(),
@@ -73,10 +73,7 @@ export async function triggerHealthCheckAlert(
.from(targetHealthCheck)
.where(
and(
eq(
targetHealthCheck.targetHealthCheckId,
healthCheckId
),
eq(targetHealthCheck.targetHealthCheckId, healthCheckId),
eq(targetHealthCheck.orgId, orgId)
)
)

View File

@@ -25,7 +25,7 @@ import {
fireResourceHealthyAlert,
fireResourceUnhealthyAlert,
fireResourceDegradedAlert
} from "#private/lib/alerts/events/resourceEvents";
} from "@server/lib/alerts";
const paramsSchema = z.strictObject({
orgId: z.string().nonempty(),

View File

@@ -21,10 +21,7 @@ import createHttpError from "http-errors";
import logger from "@server/logger";
import { fromError } from "zod-validation-error";
import { eq, and } from "drizzle-orm";
import {
fireSiteOnlineAlert,
fireSiteOfflineAlert
} from "#private/lib/alerts/events/siteEvents";
import { fireSiteOnlineAlert, fireSiteOfflineAlert } from "@server/lib/alerts";
const paramsSchema = z.strictObject({
orgId: z.string().nonempty(),

View File

@@ -22,7 +22,7 @@ import logger from "@server/logger";
import { fromError } from "zod-validation-error";
import { OpenAPITags, registry } from "@server/openApi";
import { addStandaloneHealthCheck } from "@server/routers/newt/targets";
import { fireHealthCheckUnhealthyAlert } from "#private/lib/alerts";
import { fireHealthCheckUnhealthyAlert } from "@server/lib/alerts";
const paramsSchema = z.strictObject({
orgId: z.string().nonempty()

View File

@@ -22,7 +22,11 @@ import { fromError } from "zod-validation-error";
import { OpenAPITags, registry } from "@server/openApi";
import { and, eq, isNull } from "drizzle-orm";
import { addStandaloneHealthCheck } from "@server/routers/newt/targets";
import { fireHealthCheckUnhealthyAlert, fireHealthCheckUnknownAlert, fireHealthCheckHealthyAlert } from "#private/lib/alerts";
import {
fireHealthCheckUnhealthyAlert,
fireHealthCheckUnknownAlert,
fireHealthCheckHealthyAlert
} from "@server/lib/alerts";
const paramsSchema = z
.object({
@@ -234,7 +238,10 @@ export async function updateHealthCheck(
)
.returning();
if (updated.hcHealth === "unhealthy" && existingHealthCheck.hcHealth !== "unhealthy") {
if (
updated.hcHealth === "unhealthy" &&
existingHealthCheck.hcHealth !== "unhealthy"
) {
await fireHealthCheckUnhealthyAlert(
updated.orgId,
updated.targetHealthCheckId,
@@ -243,7 +250,10 @@ export async function updateHealthCheck(
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
} else if (updated.hcHealth === "unknown" && existingHealthCheck.hcHealth !== "unknown") {
} else if (
updated.hcHealth === "unknown" &&
existingHealthCheck.hcHealth !== "unknown"
) {
// if the health is unknown, we want to fire an alert to notify users to enable health checks
await fireHealthCheckUnknownAlert(
updated.orgId,
@@ -253,7 +263,10 @@ export async function updateHealthCheck(
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
} else if (updated.hcHealth === "healthy" && existingHealthCheck.hcHealth !== "healthy") {
} else if (
updated.hcHealth === "healthy" &&
existingHealthCheck.hcHealth !== "healthy"
) {
await fireHealthCheckHealthyAlert(
updated.orgId,
updated.targetHealthCheckId,
@@ -264,7 +277,6 @@ export async function updateHealthCheck(
);
}
// Push updated health check to newt if the site is a newt site
const [newt] = await db
.select()

View File

@@ -1,12 +1,8 @@
import { MessageHandler } from "@server/routers/ws";
import {
db,
Newt,
sites
} from "@server/db";
import { db, Newt, sites } from "@server/db";
import { eq } from "drizzle-orm";
import logger from "@server/logger";
import { fireSiteOfflineAlert } from "#dynamic/lib/alerts";
import { fireSiteOfflineAlert } from "@server/lib/alerts";
/**
* Handles disconnecting messages from sites to show disconnected in the ui
@@ -38,7 +34,13 @@ export const handleNewtDisconnectingMessage: MessageHandler = async (
.where(eq(sites.siteId, newt.siteId!))
.returning();
await fireSiteOfflineAlert(site.orgId, site.siteId, site.name, undefined, trx);
await fireSiteOfflineAlert(
site.orgId,
site.siteId,
site.name,
undefined,
trx
);
});
} catch (error) {
logger.error("Error handling disconnecting message", { error });

View File

@@ -1,12 +1,8 @@
import {
db,
newts,
sites
} from "@server/db";
import { db, newts, sites } from "@server/db";
import { hasActiveConnections } from "#dynamic/routers/ws";
import { eq, lt, isNull, and, or, ne, not, inArray } from "drizzle-orm";
import logger from "@server/logger";
import { fireSiteOfflineAlert, fireSiteOnlineAlert } from "#dynamic/lib/alerts";
import { fireSiteOfflineAlert, fireSiteOnlineAlert } from "@server/lib/alerts";
// Track if the offline checker interval is running
let offlineCheckerInterval: NodeJS.Timeout | null = null;

View File

@@ -2,7 +2,7 @@ import { db } from "@server/db";
import { sites, clients, olms } from "@server/db";
import { and, eq, inArray } from "drizzle-orm";
import logger from "@server/logger";
import { fireSiteOnlineAlert } from "#dynamic/lib/alerts";
import { fireSiteOnlineAlert } from "@server/lib/alerts";
/**
* Ping Accumulator
@@ -127,7 +127,11 @@ async function flushSitePingsToDb(): Promise<void> {
eq(sites.online, false)
)
)
.returning({ siteId: sites.siteId, orgId: sites.orgId, name: sites.name });
.returning({
siteId: sites.siteId,
orgId: sites.orgId,
name: sites.name
});
// Update lastPing for sites that were already online.
// After the update above, the newly-online sites now have
@@ -148,7 +152,13 @@ async function flushSitePingsToDb(): Promise<void> {
for (const site of newlyOnlineSites) {
await db.transaction(async (trx) => {
await fireSiteOnlineAlert(site.orgId, site.siteId, site.name, undefined, trx);
await fireSiteOnlineAlert(
site.orgId,
site.siteId,
site.name,
undefined,
trx
);
});
}
} catch (error) {

View File

@@ -23,7 +23,7 @@ import {
fireHealthCheckHealthyAlert,
fireHealthCheckUnhealthyAlert,
fireHealthCheckUnknownAlert
} from "#dynamic/lib/alerts";
} from "@server/lib/alerts";
const createTargetParamsSchema = z.strictObject({
resourceId: z.string().transform(Number).pipe(z.int().positive())

View File

@@ -6,7 +6,7 @@ import logger from "@server/logger";
import {
fireHealthCheckHealthyAlert,
fireHealthCheckUnhealthyAlert
} from "#dynamic/lib/alerts";
} from "@server/lib/alerts";
interface TargetHealthStatus {
status: string;

View File

@@ -10,7 +10,11 @@ import logger from "@server/logger";
import { fromError } from "zod-validation-error";
import { addPeer } from "../gerbil/peers";
import { addTargets } from "../newt/targets";
import { fireHealthCheckHealthyAlert, fireHealthCheckUnknownAlert, fireHealthCheckUnhealthyAlert } from "#dynamic/lib/alerts";
import {
fireHealthCheckHealthyAlert,
fireHealthCheckUnknownAlert,
fireHealthCheckUnhealthyAlert
} from "@server/lib/alerts";
import { pickPort } from "./helpers";
import { isTargetValid } from "@server/lib/validators";
import { OpenAPITags, registry } from "@server/openApi";
@@ -169,7 +173,7 @@ export async function updateTarget(
let updatedTarget: any;
let updatedHc: any;
await db.transaction(async (trx) => {
[updatedTarget] = await trx
[updatedTarget] = await trx
.update(targets)
.set({
siteId: parsedBody.data.siteId,
@@ -181,8 +185,12 @@ export async function updateTarget(
path: parsedBody.data.path,
pathMatchType: parsedBody.data.pathMatchType,
priority: parsedBody.data.priority,
rewritePath: pathMatchTypeRemoved ? null : parsedBody.data.rewritePath,
rewritePathType: pathMatchTypeRemoved ? null : parsedBody.data.rewritePathType
rewritePath: pathMatchTypeRemoved
? null
: parsedBody.data.rewritePath,
rewritePathType: pathMatchTypeRemoved
? null
: parsedBody.data.rewritePathType
})
.where(eq(targets.targetId, targetId))
.returning();
@@ -213,7 +221,8 @@ export async function updateTarget(
// If hcEnabled is being turned on (was false, now true), set to "unhealthy"
// so the target must pass a health check before being considered healthy.
const hcEnabledTurnedOn =
parsedBody.data.hcEnabled === true && existingHc.hcEnabled === false;
parsedBody.data.hcEnabled === true &&
existingHc.hcEnabled === false;
let hcHealthValue: "unknown" | "healthy" | "unhealthy" | undefined;
if (
@@ -253,7 +262,10 @@ export async function updateTarget(
.where(eq(targetHealthCheck.targetId, targetId))
.returning();
if (updatedHc.hcHealth === "unhealthy" && existingHc.hcHealth !== "unhealthy") {
if (
updatedHc.hcHealth === "unhealthy" &&
existingHc.hcHealth !== "unhealthy"
) {
logger.debug(
`Health check ${updatedHc.targetHealthCheckId} for target ${targetId} is now unhealthy, firing alert`
);
@@ -266,7 +278,10 @@ export async function updateTarget(
false, // dont send the alert because we just want to create the alert, not notify users yet
trx
);
} else if (updatedHc.hcHealth === "unknown" && existingHc.hcHealth !== "unknown") {
} else if (
updatedHc.hcHealth === "unknown" &&
existingHc.hcHealth !== "unknown"
) {
logger.debug(
`Health check ${updatedHc.targetHealthCheckId} for target ${targetId} is now unknown, firing alert`
);
@@ -280,7 +295,10 @@ export async function updateTarget(
false, // dont send the alert because we just want to create the alert, not notify users yet
trx
);
} else if (updatedHc.hcHealth === "healthy" && existingHc.hcHealth !== "healthy") {
} else if (
updatedHc.hcHealth === "healthy" &&
existingHc.hcHealth !== "healthy"
) {
logger.debug(
`Health check ${updatedHc.targetHealthCheckId} for target ${targetId} is now healthy, firing alert`
);