mirror of
https://github.com/fosrl/pangolin.git
synced 2026-05-17 06:24:32 +00:00
@@ -1,27 +1,153 @@
|
|||||||
// stub
|
import logger from "@server/logger";
|
||||||
|
import { processAlerts } from "@server/lib/alerts";
|
||||||
|
import {
|
||||||
|
db,
|
||||||
|
statusHistory,
|
||||||
|
targetHealthCheck,
|
||||||
|
targets,
|
||||||
|
resources,
|
||||||
|
Transaction,
|
||||||
|
logsDb
|
||||||
|
} from "@server/db";
|
||||||
|
import { eq } from "drizzle-orm";
|
||||||
|
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
|
||||||
|
import {
|
||||||
|
fireResourceDegradedAlert,
|
||||||
|
fireResourceHealthyAlert,
|
||||||
|
fireResourceUnhealthyAlert,
|
||||||
|
fireResourceUnknownAlert
|
||||||
|
} from "./resourceEvents";
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Public API
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fire a `health_check_healthy` alert for the given health check.
|
||||||
|
*
|
||||||
|
* Call this after a previously-failing health check has recovered so that any
|
||||||
|
* matching `alertRules` can dispatch their email and webhook actions.
|
||||||
|
*
|
||||||
|
* @param orgId - Organisation that owns the health check.
|
||||||
|
* @param healthCheckId - Numeric primary key of the health check.
|
||||||
|
* @param healthCheckName - Human-readable name shown in notifications (optional).
|
||||||
|
* @param extra - Any additional key/value pairs to include in the payload.
|
||||||
|
*/
|
||||||
export async function fireHealthCheckHealthyAlert(
|
export async function fireHealthCheckHealthyAlert(
|
||||||
orgId: string,
|
orgId: string,
|
||||||
healthCheckId: number,
|
healthCheckId: number,
|
||||||
healthCheckName?: string,
|
healthCheckName?: string | null,
|
||||||
healthCheckTargetId?: number | null,
|
healthCheckTargetId?: number | null,
|
||||||
extra?: Record<string, unknown>,
|
extra?: Record<string, unknown>,
|
||||||
send: boolean = true,
|
send: boolean = true,
|
||||||
trx?: unknown
|
trx: Transaction | typeof db = db
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
return;
|
try {
|
||||||
|
await logsDb.insert(statusHistory).values({
|
||||||
|
entityType: "health_check",
|
||||||
|
entityId: healthCheckId,
|
||||||
|
orgId: orgId,
|
||||||
|
status: "healthy",
|
||||||
|
timestamp: Math.floor(Date.now() / 1000)
|
||||||
|
});
|
||||||
|
await invalidateStatusHistoryCache("health_check", healthCheckId);
|
||||||
|
|
||||||
|
await handleResource(orgId, healthCheckTargetId, send, trx);
|
||||||
|
|
||||||
|
if (!send) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "health_check_healthy",
|
||||||
|
orgId,
|
||||||
|
healthCheckId,
|
||||||
|
data: {
|
||||||
|
...(healthCheckName != null ? { healthCheckName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "health_check_toggle",
|
||||||
|
orgId,
|
||||||
|
healthCheckId,
|
||||||
|
data: {
|
||||||
|
healthCheckId,
|
||||||
|
status: "healthy",
|
||||||
|
...(healthCheckName != null ? { healthCheckName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(
|
||||||
|
`fireHealthCheckHealthyAlert: unexpected error for healthCheckId ${healthCheckId}`,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fire a `health_check_unhealthy` alert for the given health check.
|
||||||
|
*
|
||||||
|
* Call this after a health check has been detected as failing so that any
|
||||||
|
* matching `alertRules` can dispatch their email and webhook actions.
|
||||||
|
*
|
||||||
|
* @param orgId - Organisation that owns the health check.
|
||||||
|
* @param healthCheckId - Numeric primary key of the health check.
|
||||||
|
* @param healthCheckName - Human-readable name shown in notifications (optional).
|
||||||
|
* @param extra - Any additional key/value pairs to include in the payload.
|
||||||
|
*/
|
||||||
export async function fireHealthCheckUnhealthyAlert(
|
export async function fireHealthCheckUnhealthyAlert(
|
||||||
orgId: string,
|
orgId: string,
|
||||||
healthCheckId: number,
|
healthCheckId: number,
|
||||||
healthCheckName?: string,
|
healthCheckName?: string | null,
|
||||||
healthCheckTargetId?: number | null,
|
healthCheckTargetId?: number | null,
|
||||||
extra?: Record<string, unknown>,
|
extra?: Record<string, unknown>,
|
||||||
send: boolean = true,
|
send: boolean = true,
|
||||||
trx?: unknown
|
trx: Transaction | typeof db = db
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
return;
|
try {
|
||||||
|
await logsDb.insert(statusHistory).values({
|
||||||
|
entityType: "health_check",
|
||||||
|
entityId: healthCheckId,
|
||||||
|
orgId: orgId,
|
||||||
|
status: "unhealthy",
|
||||||
|
timestamp: Math.floor(Date.now() / 1000)
|
||||||
|
});
|
||||||
|
await invalidateStatusHistoryCache("health_check", healthCheckId);
|
||||||
|
|
||||||
|
await handleResource(orgId, healthCheckTargetId, send, trx);
|
||||||
|
|
||||||
|
if (!send) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "health_check_unhealthy",
|
||||||
|
orgId,
|
||||||
|
healthCheckId,
|
||||||
|
data: {
|
||||||
|
...(healthCheckName != null ? { healthCheckName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "health_check_toggle",
|
||||||
|
orgId,
|
||||||
|
healthCheckId,
|
||||||
|
data: {
|
||||||
|
healthCheckId,
|
||||||
|
status: "unhealthy",
|
||||||
|
...(healthCheckName != null ? { healthCheckName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(
|
||||||
|
`fireHealthCheckUnhealthyAlert: unexpected error for healthCheckId ${healthCheckId}`,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function fireHealthCheckUnknownAlert(
|
export async function fireHealthCheckUnknownAlert(
|
||||||
@@ -31,7 +157,137 @@ export async function fireHealthCheckUnknownAlert(
|
|||||||
healthCheckTargetId?: number | null,
|
healthCheckTargetId?: number | null,
|
||||||
extra?: Record<string, unknown>,
|
extra?: Record<string, unknown>,
|
||||||
send: boolean = true,
|
send: boolean = true,
|
||||||
trx?: unknown
|
trx: Transaction | typeof db = db
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
return;
|
try {
|
||||||
|
await logsDb.insert(statusHistory).values({
|
||||||
|
entityType: "health_check",
|
||||||
|
entityId: healthCheckId,
|
||||||
|
orgId: orgId,
|
||||||
|
status: "unknown",
|
||||||
|
timestamp: Math.floor(Date.now() / 1000)
|
||||||
|
});
|
||||||
|
await invalidateStatusHistoryCache("health_check", healthCheckId);
|
||||||
|
|
||||||
|
await handleResource(orgId, healthCheckTargetId, send, trx);
|
||||||
|
|
||||||
|
if (!send) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(
|
||||||
|
`fireHealthCheckUnknownAlert: unexpected error for healthCheckId ${healthCheckId}`,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleResource(
|
||||||
|
orgId: string,
|
||||||
|
healthCheckTargetId?: number | null,
|
||||||
|
send: boolean = true,
|
||||||
|
trx: Transaction | typeof db = db
|
||||||
|
) {
|
||||||
|
if (!healthCheckTargetId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// we have targets lets get them
|
||||||
|
const [target] = await trx
|
||||||
|
.select()
|
||||||
|
.from(targets)
|
||||||
|
.where(eq(targets.targetId, healthCheckTargetId))
|
||||||
|
.limit(1);
|
||||||
|
|
||||||
|
if (!target) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const [resource] = await trx
|
||||||
|
.select()
|
||||||
|
.from(resources)
|
||||||
|
.where(eq(resources.resourceId, target.resourceId))
|
||||||
|
.limit(1);
|
||||||
|
|
||||||
|
if (!resource) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const otherTargets = await trx
|
||||||
|
.select({ hcHealth: targetHealthCheck.hcHealth })
|
||||||
|
.from(targets)
|
||||||
|
.innerJoin(
|
||||||
|
targetHealthCheck,
|
||||||
|
eq(targetHealthCheck.targetId, targets.targetId)
|
||||||
|
)
|
||||||
|
.where(eq(targets.resourceId, resource.resourceId));
|
||||||
|
|
||||||
|
let health = "healthy";
|
||||||
|
const allUnknown = otherTargets.every((t) => t.hcHealth === "unknown");
|
||||||
|
const allHealthy = otherTargets.every((t) => t.hcHealth === "healthy");
|
||||||
|
const allUnhealthy = otherTargets.every((t) => t.hcHealth === "unhealthy");
|
||||||
|
|
||||||
|
if (allUnknown) {
|
||||||
|
logger.debug(
|
||||||
|
`Marking resource ${resource.resourceId} as unknown because all health checks are disabled`
|
||||||
|
);
|
||||||
|
health = "unknown";
|
||||||
|
} else if (allHealthy) {
|
||||||
|
health = "healthy";
|
||||||
|
} else if (allUnhealthy) {
|
||||||
|
logger.debug(
|
||||||
|
`Marking resource ${resource.resourceId} as unhealthy because all targets are unhealthy`
|
||||||
|
);
|
||||||
|
health = "unhealthy";
|
||||||
|
} else {
|
||||||
|
logger.debug(
|
||||||
|
`Marking resource ${resource.resourceId} as degraded because some targets are unhealthy`
|
||||||
|
);
|
||||||
|
health = "degraded";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (health != resource.health) {
|
||||||
|
// it changed
|
||||||
|
await trx
|
||||||
|
.update(resources)
|
||||||
|
.set({ health })
|
||||||
|
.where(eq(resources.resourceId, resource.resourceId));
|
||||||
|
|
||||||
|
if (health === "unknown") {
|
||||||
|
await fireResourceUnknownAlert(
|
||||||
|
orgId,
|
||||||
|
resource.resourceId,
|
||||||
|
resource.name,
|
||||||
|
undefined,
|
||||||
|
send,
|
||||||
|
trx
|
||||||
|
);
|
||||||
|
} else if (health === "unhealthy") {
|
||||||
|
await fireResourceUnhealthyAlert(
|
||||||
|
orgId,
|
||||||
|
resource.resourceId,
|
||||||
|
resource.name,
|
||||||
|
undefined,
|
||||||
|
send,
|
||||||
|
trx
|
||||||
|
);
|
||||||
|
} else if (health === "healthy") {
|
||||||
|
await fireResourceHealthyAlert(
|
||||||
|
orgId,
|
||||||
|
resource.resourceId,
|
||||||
|
resource.name,
|
||||||
|
undefined,
|
||||||
|
send,
|
||||||
|
trx
|
||||||
|
);
|
||||||
|
} else if (health === "degraded") {
|
||||||
|
await fireResourceDegradedAlert(
|
||||||
|
orgId,
|
||||||
|
resource.resourceId,
|
||||||
|
resource.name,
|
||||||
|
undefined,
|
||||||
|
send,
|
||||||
|
trx
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,26 +1,243 @@
|
|||||||
|
import logger from "@server/logger";
|
||||||
|
import { processAlerts } from "@server/lib/alerts";
|
||||||
|
import { db, logsDb, statusHistory, Transaction } from "@server/db";
|
||||||
|
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Public API
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fire a `resource_healthy` alert for the given resource.
|
||||||
|
*
|
||||||
|
* Call this after a previously-unhealthy resource has recovered so that any
|
||||||
|
* matching `alertRules` can dispatch their email and webhook actions.
|
||||||
|
*
|
||||||
|
* @param orgId - Organisation that owns the resource.
|
||||||
|
* @param resourceId - Numeric primary key of the resource.
|
||||||
|
* @param resourceName - Human-readable name shown in notifications (optional).
|
||||||
|
* @param extra - Any additional key/value pairs to include in the payload.
|
||||||
|
*/
|
||||||
export async function fireResourceHealthyAlert(
|
export async function fireResourceHealthyAlert(
|
||||||
orgId: string,
|
orgId: string,
|
||||||
resourceId: number,
|
resourceId: number,
|
||||||
resourceName?: string | null,
|
resourceName?: string | null,
|
||||||
extra?: Record<string, unknown>,
|
extra?: Record<string, unknown>,
|
||||||
send: boolean = true,
|
send: boolean = true,
|
||||||
trx?: unknown
|
trx: Transaction | typeof db = db
|
||||||
): Promise<void> {}
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await logsDb.insert(statusHistory).values({
|
||||||
|
entityType: "resource",
|
||||||
|
entityId: resourceId,
|
||||||
|
orgId: orgId,
|
||||||
|
status: "healthy",
|
||||||
|
timestamp: Math.floor(Date.now() / 1000)
|
||||||
|
});
|
||||||
|
await invalidateStatusHistoryCache("resource", resourceId);
|
||||||
|
|
||||||
|
if (!send) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "resource_healthy",
|
||||||
|
orgId,
|
||||||
|
resourceId,
|
||||||
|
data: {
|
||||||
|
...(resourceName != null ? { resourceName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "resource_toggle",
|
||||||
|
orgId,
|
||||||
|
resourceId,
|
||||||
|
data: {
|
||||||
|
resourceId,
|
||||||
|
status: "healthy",
|
||||||
|
...(resourceName != null ? { resourceName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(
|
||||||
|
`fireResourceHealthyAlert: unexpected error for resourceId ${resourceId}`,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fire a `resource_unhealthy` alert for the given resource.
|
||||||
|
*
|
||||||
|
* Call this after a resource has been detected as unhealthy so that any
|
||||||
|
* matching `alertRules` can dispatch their email and webhook actions.
|
||||||
|
*
|
||||||
|
* @param orgId - Organisation that owns the resource.
|
||||||
|
* @param resourceId - Numeric primary key of the resource.
|
||||||
|
* @param resourceName - Human-readable name shown in notifications (optional).
|
||||||
|
* @param extra - Any additional key/value pairs to include in the payload.
|
||||||
|
*/
|
||||||
export async function fireResourceUnhealthyAlert(
|
export async function fireResourceUnhealthyAlert(
|
||||||
orgId: string,
|
orgId: string,
|
||||||
resourceId: number,
|
resourceId: number,
|
||||||
resourceName?: string | null,
|
resourceName?: string | null,
|
||||||
extra?: Record<string, unknown>,
|
extra?: Record<string, unknown>,
|
||||||
send: boolean = true,
|
send: boolean = true,
|
||||||
trx?: unknown
|
trx: Transaction | typeof db = db
|
||||||
): Promise<void> {}
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await logsDb.insert(statusHistory).values({
|
||||||
|
entityType: "resource",
|
||||||
|
entityId: resourceId,
|
||||||
|
orgId: orgId,
|
||||||
|
status: "unhealthy",
|
||||||
|
timestamp: Math.floor(Date.now() / 1000)
|
||||||
|
});
|
||||||
|
await invalidateStatusHistoryCache("resource", resourceId);
|
||||||
|
|
||||||
export async function fireResourceToggleAlert(
|
if (!send) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "resource_unhealthy",
|
||||||
|
orgId,
|
||||||
|
resourceId,
|
||||||
|
data: {
|
||||||
|
...(resourceName != null ? { resourceName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "resource_toggle",
|
||||||
|
orgId,
|
||||||
|
resourceId,
|
||||||
|
data: {
|
||||||
|
resourceId,
|
||||||
|
status: "unhealthy",
|
||||||
|
...(resourceName != null ? { resourceName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(
|
||||||
|
`fireResourceUnhealthyAlert: unexpected error for resourceId ${resourceId}`,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fire a `resource_degraded` alert for the given resource.
|
||||||
|
*
|
||||||
|
* Call this after a resource has been detected as degraded so that any
|
||||||
|
* matching `alertRules` can dispatch their email and webhook actions.
|
||||||
|
*
|
||||||
|
* @param orgId - Organisation that owns the resource.
|
||||||
|
* @param resourceId - Numeric primary key of the resource.
|
||||||
|
* @param resourceName - Human-readable name shown in notifications (optional).
|
||||||
|
* @param extra - Any additional key/value pairs to include in the payload.
|
||||||
|
*/
|
||||||
|
export async function fireResourceDegradedAlert(
|
||||||
orgId: string,
|
orgId: string,
|
||||||
resourceId: number,
|
resourceId: number,
|
||||||
resourceName?: string | null,
|
resourceName?: string | null,
|
||||||
extra?: Record<string, unknown>,
|
extra?: Record<string, unknown>,
|
||||||
send: boolean = true,
|
send: boolean = true,
|
||||||
trx?: unknown
|
trx: Transaction | typeof db = db
|
||||||
): Promise<void> {}
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await logsDb.insert(statusHistory).values({
|
||||||
|
entityType: "resource",
|
||||||
|
entityId: resourceId,
|
||||||
|
orgId: orgId,
|
||||||
|
status: "degraded",
|
||||||
|
timestamp: Math.floor(Date.now() / 1000)
|
||||||
|
});
|
||||||
|
await invalidateStatusHistoryCache("resource", resourceId);
|
||||||
|
|
||||||
|
if (!send) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "resource_degraded",
|
||||||
|
orgId,
|
||||||
|
resourceId,
|
||||||
|
data: {
|
||||||
|
...(resourceName != null ? { resourceName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "resource_toggle",
|
||||||
|
orgId,
|
||||||
|
resourceId,
|
||||||
|
data: {
|
||||||
|
resourceId,
|
||||||
|
status: "degraded",
|
||||||
|
...(resourceName != null ? { resourceName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(
|
||||||
|
`fireResourceDegradedAlert: unexpected error for resourceId ${resourceId}`,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fire a `resource_unknown` alert for the given resource.
|
||||||
|
*
|
||||||
|
* Call this when all health checks on a resource are disabled so that the
|
||||||
|
* resource status transitions to unknown.
|
||||||
|
*
|
||||||
|
* @param orgId - Organisation that owns the resource.
|
||||||
|
* @param resourceId - Numeric primary key of the resource.
|
||||||
|
* @param resourceName - Human-readable name shown in notifications (optional).
|
||||||
|
* @param extra - Any additional key/value pairs to include in the payload.
|
||||||
|
*/
|
||||||
|
export async function fireResourceUnknownAlert(
|
||||||
|
orgId: string,
|
||||||
|
resourceId: number,
|
||||||
|
resourceName?: string | null,
|
||||||
|
extra?: Record<string, unknown>,
|
||||||
|
send: boolean = true,
|
||||||
|
trx: Transaction | typeof db = db
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await logsDb.insert(statusHistory).values({
|
||||||
|
entityType: "resource",
|
||||||
|
entityId: resourceId,
|
||||||
|
orgId: orgId,
|
||||||
|
status: "unknown",
|
||||||
|
timestamp: Math.floor(Date.now() / 1000)
|
||||||
|
});
|
||||||
|
await invalidateStatusHistoryCache("resource", resourceId);
|
||||||
|
|
||||||
|
if (!send) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "resource_toggle",
|
||||||
|
orgId,
|
||||||
|
resourceId,
|
||||||
|
data: {
|
||||||
|
resourceId,
|
||||||
|
status: "unknown",
|
||||||
|
...(resourceName != null ? { resourceName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(
|
||||||
|
`fireResourceUnknownAlert: unexpected error for resourceId ${resourceId}`,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,21 +1,156 @@
|
|||||||
// stub
|
import logger from "@server/logger";
|
||||||
|
import { processAlerts } from "@server/lib/alerts";
|
||||||
|
import {
|
||||||
|
db,
|
||||||
|
logsDb,
|
||||||
|
statusHistory,
|
||||||
|
targetHealthCheck,
|
||||||
|
Transaction
|
||||||
|
} from "@server/db";
|
||||||
|
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
|
||||||
|
import { and, eq, inArray } from "drizzle-orm";
|
||||||
|
import { fireHealthCheckUnhealthyAlert } from "./healthCheckEvents";
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Public API
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fire a `site_online` alert for the given site.
|
||||||
|
*
|
||||||
|
* Call this after the site has been confirmed reachable / connected so that
|
||||||
|
* any matching `alertRules` can dispatch their email and webhook actions.
|
||||||
|
*
|
||||||
|
* @param orgId - Organisation that owns the site.
|
||||||
|
* @param siteId - Numeric primary key of the site.
|
||||||
|
* @param siteName - Human-readable name shown in notifications (optional).
|
||||||
|
* @param extra - Any additional key/value pairs to include in the payload.
|
||||||
|
*/
|
||||||
export async function fireSiteOnlineAlert(
|
export async function fireSiteOnlineAlert(
|
||||||
orgId: string,
|
orgId: string,
|
||||||
siteId: number,
|
siteId: number,
|
||||||
siteName?: string,
|
siteName?: string,
|
||||||
extra?: Record<string, unknown>,
|
extra?: Record<string, unknown>,
|
||||||
trx?: unknown
|
trx: Transaction | typeof db = db
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
return;
|
try {
|
||||||
|
await logsDb.insert(statusHistory).values({
|
||||||
|
entityType: "site",
|
||||||
|
entityId: siteId,
|
||||||
|
orgId: orgId,
|
||||||
|
status: "online",
|
||||||
|
timestamp: Math.floor(Date.now() / 1000)
|
||||||
|
});
|
||||||
|
await invalidateStatusHistoryCache("site", siteId);
|
||||||
|
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "site_online",
|
||||||
|
orgId,
|
||||||
|
siteId,
|
||||||
|
data: {
|
||||||
|
...(siteName != null ? { siteName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "site_toggle",
|
||||||
|
orgId,
|
||||||
|
siteId,
|
||||||
|
data: {
|
||||||
|
siteId,
|
||||||
|
status: "online",
|
||||||
|
...(siteName != null ? { siteName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(
|
||||||
|
`fireSiteOnlineAlert: unexpected error for siteId ${siteId}`,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fire a `site_offline` alert for the given site.
|
||||||
|
*
|
||||||
|
* Call this after the site has been detected as unreachable / disconnected so
|
||||||
|
* that any matching `alertRules` can dispatch their email and webhook actions.
|
||||||
|
*
|
||||||
|
* @param orgId - Organisation that owns the site.
|
||||||
|
* @param siteId - Numeric primary key of the site.
|
||||||
|
* @param siteName - Human-readable name shown in notifications (optional).
|
||||||
|
* @param extra - Any additional key/value pairs to include in the payload.
|
||||||
|
*/
|
||||||
export async function fireSiteOfflineAlert(
|
export async function fireSiteOfflineAlert(
|
||||||
orgId: string,
|
orgId: string,
|
||||||
siteId: number,
|
siteId: number,
|
||||||
siteName?: string,
|
siteName?: string,
|
||||||
extra?: Record<string, unknown>,
|
extra?: Record<string, unknown>,
|
||||||
trx?: unknown
|
trx: Transaction | typeof db = db
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
return;
|
try {
|
||||||
}
|
await logsDb.insert(statusHistory).values({
|
||||||
|
entityType: "site",
|
||||||
|
entityId: siteId,
|
||||||
|
orgId: orgId,
|
||||||
|
status: "offline",
|
||||||
|
timestamp: Math.floor(Date.now() / 1000)
|
||||||
|
});
|
||||||
|
await invalidateStatusHistoryCache("site", siteId);
|
||||||
|
|
||||||
|
const unhealthyHealthChecks = await trx
|
||||||
|
.update(targetHealthCheck)
|
||||||
|
.set({ hcHealth: "unhealthy" })
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(targetHealthCheck.orgId, orgId),
|
||||||
|
eq(targetHealthCheck.siteId, siteId),
|
||||||
|
eq(targetHealthCheck.hcEnabled, true) // only effect the ones that are enabled
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.returning();
|
||||||
|
|
||||||
|
for (const healthCheck of unhealthyHealthChecks) {
|
||||||
|
logger.info(
|
||||||
|
`Marking health check ${healthCheck.targetHealthCheckId} unhealthy due to site ${siteId} being marked offline`
|
||||||
|
);
|
||||||
|
|
||||||
|
await fireHealthCheckUnhealthyAlert(
|
||||||
|
healthCheck.orgId,
|
||||||
|
healthCheck.targetHealthCheckId,
|
||||||
|
healthCheck.name,
|
||||||
|
healthCheck.targetId, // for the resource if we have one
|
||||||
|
undefined,
|
||||||
|
true,
|
||||||
|
trx
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "site_offline",
|
||||||
|
orgId,
|
||||||
|
siteId,
|
||||||
|
data: {
|
||||||
|
...(siteName != null ? { siteName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await processAlerts({
|
||||||
|
eventType: "site_toggle",
|
||||||
|
orgId,
|
||||||
|
siteId,
|
||||||
|
data: {
|
||||||
|
siteId,
|
||||||
|
status: "offline",
|
||||||
|
...(siteName != null ? { siteName } : {}),
|
||||||
|
...extra
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(
|
||||||
|
`fireSiteOfflineAlert: unexpected error for siteId ${siteId}`,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
export * from "./events/siteEvents";
|
export * from "./events/siteEvents";
|
||||||
export * from "./events/healthCheckEvents";
|
export * from "./events/healthCheckEvents";
|
||||||
export * from "./events/resourceEvents";
|
export * from "./events/resourceEvents";
|
||||||
|
export * from "./processAlerts";
|
||||||
|
|||||||
5
server/lib/alerts/processAlerts.ts
Normal file
5
server/lib/alerts/processAlerts.ts
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
import { AlertContext } from "@server/routers/alertRule/types";
|
||||||
|
|
||||||
|
export async function processAlerts(context: AlertContext): Promise<void> {
|
||||||
|
return;
|
||||||
|
}
|
||||||
@@ -1,8 +1,9 @@
|
|||||||
export async function getOrgTierData(
|
export async function getOrgTierData(
|
||||||
orgId: string
|
orgId: string
|
||||||
): Promise<{ tier: string | null; active: boolean }> {
|
): Promise<{ tier: string | null; active: boolean; isTrial: boolean }> {
|
||||||
const tier = null;
|
const tier = null;
|
||||||
const active = false;
|
const active = false;
|
||||||
|
const isTrial = false;
|
||||||
|
|
||||||
return { tier, active };
|
return { tier, active, isTrial };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ import { hashPassword } from "@server/auth/password";
|
|||||||
import { isValidCIDR, isValidIP, isValidUrlGlobPattern } from "../validators";
|
import { isValidCIDR, isValidIP, isValidUrlGlobPattern } from "../validators";
|
||||||
import { isValidRegionId } from "@server/db/regions";
|
import { isValidRegionId } from "@server/db/regions";
|
||||||
import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed";
|
import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed";
|
||||||
import { fireHealthCheckUnknownAlert } from "#dynamic/lib/alerts";
|
import { fireHealthCheckUnknownAlert } from "@server/lib/alerts";
|
||||||
import { tierMatrix } from "../billing/tierMatrix";
|
import { tierMatrix } from "../billing/tierMatrix";
|
||||||
|
|
||||||
export type ProxyResourcesResults = {
|
export type ProxyResourcesResults = {
|
||||||
@@ -165,7 +165,8 @@ export async function updateProxyResources(
|
|||||||
hcStatus: healthcheckData?.status,
|
hcStatus: healthcheckData?.status,
|
||||||
hcHealth: "unknown",
|
hcHealth: "unknown",
|
||||||
hcHealthyThreshold: healthcheckData?.["healthy-threshold"],
|
hcHealthyThreshold: healthcheckData?.["healthy-threshold"],
|
||||||
hcUnhealthyThreshold: healthcheckData?.["unhealthy-threshold"]
|
hcUnhealthyThreshold:
|
||||||
|
healthcheckData?.["unhealthy-threshold"]
|
||||||
})
|
})
|
||||||
.returning();
|
.returning();
|
||||||
|
|
||||||
@@ -544,8 +545,10 @@ export async function updateProxyResources(
|
|||||||
healthcheckData?.["follow-redirects"],
|
healthcheckData?.["follow-redirects"],
|
||||||
hcMethod: healthcheckData?.method,
|
hcMethod: healthcheckData?.method,
|
||||||
hcStatus: healthcheckData?.status,
|
hcStatus: healthcheckData?.status,
|
||||||
hcHealthyThreshold: healthcheckData?.["healthy-threshold"],
|
hcHealthyThreshold:
|
||||||
hcUnhealthyThreshold: healthcheckData?.["unhealthy-threshold"]
|
healthcheckData?.["healthy-threshold"],
|
||||||
|
hcUnhealthyThreshold:
|
||||||
|
healthcheckData?.["unhealthy-threshold"]
|
||||||
})
|
})
|
||||||
.where(
|
.where(
|
||||||
eq(
|
eq(
|
||||||
@@ -1120,8 +1123,10 @@ function checkIfHealthcheckChanged(
|
|||||||
JSON.stringify(incoming.hcHeaders)
|
JSON.stringify(incoming.hcHeaders)
|
||||||
)
|
)
|
||||||
return true;
|
return true;
|
||||||
if (existing.hcHealthyThreshold !== incoming.hcHealthyThreshold) return true;
|
if (existing.hcHealthyThreshold !== incoming.hcHealthyThreshold)
|
||||||
if (existing.hcUnhealthyThreshold !== incoming.hcUnhealthyThreshold) return true;
|
return true;
|
||||||
|
if (existing.hcUnhealthyThreshold !== incoming.hcUnhealthyThreshold)
|
||||||
|
return true;
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -1184,7 +1189,11 @@ async function getDomainId(
|
|||||||
orgId: string,
|
orgId: string,
|
||||||
fullDomain: string,
|
fullDomain: string,
|
||||||
trx: Transaction
|
trx: Transaction
|
||||||
): Promise<{ subdomain: string | null; domainId: string; wildcard: boolean } | null> {
|
): Promise<{
|
||||||
|
subdomain: string | null;
|
||||||
|
domainId: string;
|
||||||
|
wildcard: boolean;
|
||||||
|
} | null> {
|
||||||
const isWildcardFullDomain = fullDomain.startsWith("*.");
|
const isWildcardFullDomain = fullDomain.startsWith("*.");
|
||||||
|
|
||||||
const possibleDomains = await trx
|
const possibleDomains = await trx
|
||||||
|
|||||||
@@ -1,306 +0,0 @@
|
|||||||
/*
|
|
||||||
* This file is part of a proprietary work.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2025-2026 Fossorial, Inc.
|
|
||||||
* All rights reserved.
|
|
||||||
*
|
|
||||||
* This file is licensed under the Fossorial Commercial License.
|
|
||||||
* You may not use this file except in compliance with the License.
|
|
||||||
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
|
|
||||||
*
|
|
||||||
* This file is not licensed under the AGPLv3.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import logger from "@server/logger";
|
|
||||||
import { processAlerts } from "../processAlerts";
|
|
||||||
import {
|
|
||||||
db,
|
|
||||||
statusHistory,
|
|
||||||
targetHealthCheck,
|
|
||||||
targets,
|
|
||||||
resources,
|
|
||||||
Transaction,
|
|
||||||
logsDb
|
|
||||||
} from "@server/db";
|
|
||||||
import { eq } from "drizzle-orm";
|
|
||||||
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
|
|
||||||
import {
|
|
||||||
fireResourceDegradedAlert,
|
|
||||||
fireResourceHealthyAlert,
|
|
||||||
fireResourceUnhealthyAlert,
|
|
||||||
fireResourceUnknownAlert
|
|
||||||
} from "./resourceEvents";
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Public API
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fire a `health_check_healthy` alert for the given health check.
|
|
||||||
*
|
|
||||||
* Call this after a previously-failing health check has recovered so that any
|
|
||||||
* matching `alertRules` can dispatch their email and webhook actions.
|
|
||||||
*
|
|
||||||
* @param orgId - Organisation that owns the health check.
|
|
||||||
* @param healthCheckId - Numeric primary key of the health check.
|
|
||||||
* @param healthCheckName - Human-readable name shown in notifications (optional).
|
|
||||||
* @param extra - Any additional key/value pairs to include in the payload.
|
|
||||||
*/
|
|
||||||
export async function fireHealthCheckHealthyAlert(
|
|
||||||
orgId: string,
|
|
||||||
healthCheckId: number,
|
|
||||||
healthCheckName?: string | null,
|
|
||||||
healthCheckTargetId?: number | null,
|
|
||||||
extra?: Record<string, unknown>,
|
|
||||||
send: boolean = true,
|
|
||||||
trx: Transaction | typeof db = db
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
await logsDb.insert(statusHistory).values({
|
|
||||||
entityType: "health_check",
|
|
||||||
entityId: healthCheckId,
|
|
||||||
orgId: orgId,
|
|
||||||
status: "healthy",
|
|
||||||
timestamp: Math.floor(Date.now() / 1000)
|
|
||||||
});
|
|
||||||
await invalidateStatusHistoryCache("health_check", healthCheckId);
|
|
||||||
|
|
||||||
await handleResource(orgId, healthCheckTargetId, send, trx);
|
|
||||||
|
|
||||||
if (!send) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "health_check_healthy",
|
|
||||||
orgId,
|
|
||||||
healthCheckId,
|
|
||||||
data: {
|
|
||||||
...(healthCheckName != null ? { healthCheckName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "health_check_toggle",
|
|
||||||
orgId,
|
|
||||||
healthCheckId,
|
|
||||||
data: {
|
|
||||||
healthCheckId,
|
|
||||||
status: "healthy",
|
|
||||||
...(healthCheckName != null ? { healthCheckName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
`fireHealthCheckHealthyAlert: unexpected error for healthCheckId ${healthCheckId}`,
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fire a `health_check_unhealthy` alert for the given health check.
|
|
||||||
*
|
|
||||||
* Call this after a health check has been detected as failing so that any
|
|
||||||
* matching `alertRules` can dispatch their email and webhook actions.
|
|
||||||
*
|
|
||||||
* @param orgId - Organisation that owns the health check.
|
|
||||||
* @param healthCheckId - Numeric primary key of the health check.
|
|
||||||
* @param healthCheckName - Human-readable name shown in notifications (optional).
|
|
||||||
* @param extra - Any additional key/value pairs to include in the payload.
|
|
||||||
*/
|
|
||||||
export async function fireHealthCheckUnhealthyAlert(
|
|
||||||
orgId: string,
|
|
||||||
healthCheckId: number,
|
|
||||||
healthCheckName?: string | null,
|
|
||||||
healthCheckTargetId?: number | null,
|
|
||||||
extra?: Record<string, unknown>,
|
|
||||||
send: boolean = true,
|
|
||||||
trx: Transaction | typeof db = db
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
await logsDb.insert(statusHistory).values({
|
|
||||||
entityType: "health_check",
|
|
||||||
entityId: healthCheckId,
|
|
||||||
orgId: orgId,
|
|
||||||
status: "unhealthy",
|
|
||||||
timestamp: Math.floor(Date.now() / 1000)
|
|
||||||
});
|
|
||||||
await invalidateStatusHistoryCache("health_check", healthCheckId);
|
|
||||||
|
|
||||||
await handleResource(orgId, healthCheckTargetId, send, trx);
|
|
||||||
|
|
||||||
if (!send) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "health_check_unhealthy",
|
|
||||||
orgId,
|
|
||||||
healthCheckId,
|
|
||||||
data: {
|
|
||||||
...(healthCheckName != null ? { healthCheckName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "health_check_toggle",
|
|
||||||
orgId,
|
|
||||||
healthCheckId,
|
|
||||||
data: {
|
|
||||||
healthCheckId,
|
|
||||||
status: "unhealthy",
|
|
||||||
...(healthCheckName != null ? { healthCheckName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
`fireHealthCheckUnhealthyAlert: unexpected error for healthCheckId ${healthCheckId}`,
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function fireHealthCheckUnknownAlert(
|
|
||||||
orgId: string,
|
|
||||||
healthCheckId: number,
|
|
||||||
healthCheckName?: string | null,
|
|
||||||
healthCheckTargetId?: number | null,
|
|
||||||
extra?: Record<string, unknown>,
|
|
||||||
send: boolean = true,
|
|
||||||
trx: Transaction | typeof db = db
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
await logsDb.insert(statusHistory).values({
|
|
||||||
entityType: "health_check",
|
|
||||||
entityId: healthCheckId,
|
|
||||||
orgId: orgId,
|
|
||||||
status: "unknown",
|
|
||||||
timestamp: Math.floor(Date.now() / 1000)
|
|
||||||
});
|
|
||||||
await invalidateStatusHistoryCache("health_check", healthCheckId);
|
|
||||||
|
|
||||||
await handleResource(orgId, healthCheckTargetId, send, trx);
|
|
||||||
|
|
||||||
if (!send) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
`fireHealthCheckUnknownAlert: unexpected error for healthCheckId ${healthCheckId}`,
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function handleResource(
|
|
||||||
orgId: string,
|
|
||||||
healthCheckTargetId?: number | null,
|
|
||||||
send: boolean = true,
|
|
||||||
trx: Transaction | typeof db = db
|
|
||||||
) {
|
|
||||||
if (!healthCheckTargetId) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// we have targets lets get them
|
|
||||||
const [target] = await trx
|
|
||||||
.select()
|
|
||||||
.from(targets)
|
|
||||||
.where(eq(targets.targetId, healthCheckTargetId))
|
|
||||||
.limit(1);
|
|
||||||
|
|
||||||
if (!target) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const [resource] = await trx
|
|
||||||
.select()
|
|
||||||
.from(resources)
|
|
||||||
.where(eq(resources.resourceId, target.resourceId))
|
|
||||||
.limit(1);
|
|
||||||
|
|
||||||
if (!resource) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const otherTargets = await trx
|
|
||||||
.select({ hcHealth: targetHealthCheck.hcHealth })
|
|
||||||
.from(targets)
|
|
||||||
.innerJoin(
|
|
||||||
targetHealthCheck,
|
|
||||||
eq(targetHealthCheck.targetId, targets.targetId)
|
|
||||||
)
|
|
||||||
.where(eq(targets.resourceId, resource.resourceId));
|
|
||||||
|
|
||||||
let health = "healthy";
|
|
||||||
const allUnknown = otherTargets.every((t) => t.hcHealth === "unknown");
|
|
||||||
const allHealthy = otherTargets.every((t) => t.hcHealth === "healthy");
|
|
||||||
const allUnhealthy = otherTargets.every((t) => t.hcHealth === "unhealthy");
|
|
||||||
|
|
||||||
if (allUnknown) {
|
|
||||||
logger.debug(
|
|
||||||
`Marking resource ${resource.resourceId} as unknown because all health checks are disabled`
|
|
||||||
);
|
|
||||||
health = "unknown";
|
|
||||||
} else if (allHealthy) {
|
|
||||||
health = "healthy";
|
|
||||||
} else if (allUnhealthy) {
|
|
||||||
logger.debug(
|
|
||||||
`Marking resource ${resource.resourceId} as unhealthy because all targets are unhealthy`
|
|
||||||
);
|
|
||||||
health = "unhealthy";
|
|
||||||
} else {
|
|
||||||
logger.debug(
|
|
||||||
`Marking resource ${resource.resourceId} as degraded because some targets are unhealthy`
|
|
||||||
);
|
|
||||||
health = "degraded";
|
|
||||||
}
|
|
||||||
|
|
||||||
if (health != resource.health) {
|
|
||||||
// it changed
|
|
||||||
await trx
|
|
||||||
.update(resources)
|
|
||||||
.set({ health })
|
|
||||||
.where(eq(resources.resourceId, resource.resourceId));
|
|
||||||
|
|
||||||
if (health === "unknown") {
|
|
||||||
await fireResourceUnknownAlert(
|
|
||||||
orgId,
|
|
||||||
resource.resourceId,
|
|
||||||
resource.name,
|
|
||||||
undefined,
|
|
||||||
send,
|
|
||||||
trx
|
|
||||||
);
|
|
||||||
} else if (health === "unhealthy") {
|
|
||||||
await fireResourceUnhealthyAlert(
|
|
||||||
orgId,
|
|
||||||
resource.resourceId,
|
|
||||||
resource.name,
|
|
||||||
undefined,
|
|
||||||
send,
|
|
||||||
trx
|
|
||||||
);
|
|
||||||
} else if (health === "healthy") {
|
|
||||||
await fireResourceHealthyAlert(
|
|
||||||
orgId,
|
|
||||||
resource.resourceId,
|
|
||||||
resource.name,
|
|
||||||
undefined,
|
|
||||||
send,
|
|
||||||
trx
|
|
||||||
);
|
|
||||||
} else if (health === "degraded") {
|
|
||||||
await fireResourceDegradedAlert(
|
|
||||||
orgId,
|
|
||||||
resource.resourceId,
|
|
||||||
resource.name,
|
|
||||||
undefined,
|
|
||||||
send,
|
|
||||||
trx
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,256 +0,0 @@
|
|||||||
/*
|
|
||||||
* This file is part of a proprietary work.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2025-2026 Fossorial, Inc.
|
|
||||||
* All rights reserved.
|
|
||||||
*
|
|
||||||
* This file is licensed under the Fossorial Commercial License.
|
|
||||||
* You may not use this file except in compliance with the License.
|
|
||||||
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
|
|
||||||
*
|
|
||||||
* This file is not licensed under the AGPLv3.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import logger from "@server/logger";
|
|
||||||
import { processAlerts } from "../processAlerts";
|
|
||||||
import { db, logsDb, statusHistory, Transaction } from "@server/db";
|
|
||||||
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Public API
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fire a `resource_healthy` alert for the given resource.
|
|
||||||
*
|
|
||||||
* Call this after a previously-unhealthy resource has recovered so that any
|
|
||||||
* matching `alertRules` can dispatch their email and webhook actions.
|
|
||||||
*
|
|
||||||
* @param orgId - Organisation that owns the resource.
|
|
||||||
* @param resourceId - Numeric primary key of the resource.
|
|
||||||
* @param resourceName - Human-readable name shown in notifications (optional).
|
|
||||||
* @param extra - Any additional key/value pairs to include in the payload.
|
|
||||||
*/
|
|
||||||
export async function fireResourceHealthyAlert(
|
|
||||||
orgId: string,
|
|
||||||
resourceId: number,
|
|
||||||
resourceName?: string | null,
|
|
||||||
extra?: Record<string, unknown>,
|
|
||||||
send: boolean = true,
|
|
||||||
trx: Transaction | typeof db = db
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
await logsDb.insert(statusHistory).values({
|
|
||||||
entityType: "resource",
|
|
||||||
entityId: resourceId,
|
|
||||||
orgId: orgId,
|
|
||||||
status: "healthy",
|
|
||||||
timestamp: Math.floor(Date.now() / 1000)
|
|
||||||
});
|
|
||||||
await invalidateStatusHistoryCache("resource", resourceId);
|
|
||||||
|
|
||||||
if (!send) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "resource_healthy",
|
|
||||||
orgId,
|
|
||||||
resourceId,
|
|
||||||
data: {
|
|
||||||
...(resourceName != null ? { resourceName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "resource_toggle",
|
|
||||||
orgId,
|
|
||||||
resourceId,
|
|
||||||
data: {
|
|
||||||
resourceId,
|
|
||||||
status: "healthy",
|
|
||||||
...(resourceName != null ? { resourceName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
`fireResourceHealthyAlert: unexpected error for resourceId ${resourceId}`,
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fire a `resource_unhealthy` alert for the given resource.
|
|
||||||
*
|
|
||||||
* Call this after a resource has been detected as unhealthy so that any
|
|
||||||
* matching `alertRules` can dispatch their email and webhook actions.
|
|
||||||
*
|
|
||||||
* @param orgId - Organisation that owns the resource.
|
|
||||||
* @param resourceId - Numeric primary key of the resource.
|
|
||||||
* @param resourceName - Human-readable name shown in notifications (optional).
|
|
||||||
* @param extra - Any additional key/value pairs to include in the payload.
|
|
||||||
*/
|
|
||||||
export async function fireResourceUnhealthyAlert(
|
|
||||||
orgId: string,
|
|
||||||
resourceId: number,
|
|
||||||
resourceName?: string | null,
|
|
||||||
extra?: Record<string, unknown>,
|
|
||||||
send: boolean = true,
|
|
||||||
trx: Transaction | typeof db = db
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
await logsDb.insert(statusHistory).values({
|
|
||||||
entityType: "resource",
|
|
||||||
entityId: resourceId,
|
|
||||||
orgId: orgId,
|
|
||||||
status: "unhealthy",
|
|
||||||
timestamp: Math.floor(Date.now() / 1000)
|
|
||||||
});
|
|
||||||
await invalidateStatusHistoryCache("resource", resourceId);
|
|
||||||
|
|
||||||
if (!send) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "resource_unhealthy",
|
|
||||||
orgId,
|
|
||||||
resourceId,
|
|
||||||
data: {
|
|
||||||
...(resourceName != null ? { resourceName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "resource_toggle",
|
|
||||||
orgId,
|
|
||||||
resourceId,
|
|
||||||
data: {
|
|
||||||
resourceId,
|
|
||||||
status: "unhealthy",
|
|
||||||
...(resourceName != null ? { resourceName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
`fireResourceUnhealthyAlert: unexpected error for resourceId ${resourceId}`,
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fire a `resource_degraded` alert for the given resource.
|
|
||||||
*
|
|
||||||
* Call this after a resource has been detected as degraded so that any
|
|
||||||
* matching `alertRules` can dispatch their email and webhook actions.
|
|
||||||
*
|
|
||||||
* @param orgId - Organisation that owns the resource.
|
|
||||||
* @param resourceId - Numeric primary key of the resource.
|
|
||||||
* @param resourceName - Human-readable name shown in notifications (optional).
|
|
||||||
* @param extra - Any additional key/value pairs to include in the payload.
|
|
||||||
*/
|
|
||||||
export async function fireResourceDegradedAlert(
|
|
||||||
orgId: string,
|
|
||||||
resourceId: number,
|
|
||||||
resourceName?: string | null,
|
|
||||||
extra?: Record<string, unknown>,
|
|
||||||
send: boolean = true,
|
|
||||||
trx: Transaction | typeof db = db
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
await logsDb.insert(statusHistory).values({
|
|
||||||
entityType: "resource",
|
|
||||||
entityId: resourceId,
|
|
||||||
orgId: orgId,
|
|
||||||
status: "degraded",
|
|
||||||
timestamp: Math.floor(Date.now() / 1000)
|
|
||||||
});
|
|
||||||
await invalidateStatusHistoryCache("resource", resourceId);
|
|
||||||
|
|
||||||
if (!send) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "resource_degraded",
|
|
||||||
orgId,
|
|
||||||
resourceId,
|
|
||||||
data: {
|
|
||||||
...(resourceName != null ? { resourceName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "resource_toggle",
|
|
||||||
orgId,
|
|
||||||
resourceId,
|
|
||||||
data: {
|
|
||||||
resourceId,
|
|
||||||
status: "degraded",
|
|
||||||
...(resourceName != null ? { resourceName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
`fireResourceDegradedAlert: unexpected error for resourceId ${resourceId}`,
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fire a `resource_unknown` alert for the given resource.
|
|
||||||
*
|
|
||||||
* Call this when all health checks on a resource are disabled so that the
|
|
||||||
* resource status transitions to unknown.
|
|
||||||
*
|
|
||||||
* @param orgId - Organisation that owns the resource.
|
|
||||||
* @param resourceId - Numeric primary key of the resource.
|
|
||||||
* @param resourceName - Human-readable name shown in notifications (optional).
|
|
||||||
* @param extra - Any additional key/value pairs to include in the payload.
|
|
||||||
*/
|
|
||||||
export async function fireResourceUnknownAlert(
|
|
||||||
orgId: string,
|
|
||||||
resourceId: number,
|
|
||||||
resourceName?: string | null,
|
|
||||||
extra?: Record<string, unknown>,
|
|
||||||
send: boolean = true,
|
|
||||||
trx: Transaction | typeof db = db
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
await logsDb.insert(statusHistory).values({
|
|
||||||
entityType: "resource",
|
|
||||||
entityId: resourceId,
|
|
||||||
orgId: orgId,
|
|
||||||
status: "unknown",
|
|
||||||
timestamp: Math.floor(Date.now() / 1000)
|
|
||||||
});
|
|
||||||
await invalidateStatusHistoryCache("resource", resourceId);
|
|
||||||
|
|
||||||
if (!send) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "resource_toggle",
|
|
||||||
orgId,
|
|
||||||
resourceId,
|
|
||||||
data: {
|
|
||||||
resourceId,
|
|
||||||
status: "unknown",
|
|
||||||
...(resourceName != null ? { resourceName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
`fireResourceUnknownAlert: unexpected error for resourceId ${resourceId}`,
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,169 +0,0 @@
|
|||||||
/*
|
|
||||||
* This file is part of a proprietary work.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2025-2026 Fossorial, Inc.
|
|
||||||
* All rights reserved.
|
|
||||||
*
|
|
||||||
* This file is licensed under the Fossorial Commercial License.
|
|
||||||
* You may not use this file except in compliance with the License.
|
|
||||||
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
|
|
||||||
*
|
|
||||||
* This file is not licensed under the AGPLv3.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import logger from "@server/logger";
|
|
||||||
import { processAlerts } from "../processAlerts";
|
|
||||||
import {
|
|
||||||
db,
|
|
||||||
logsDb,
|
|
||||||
statusHistory,
|
|
||||||
targetHealthCheck,
|
|
||||||
Transaction
|
|
||||||
} from "@server/db";
|
|
||||||
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
|
|
||||||
import { and, eq, inArray } from "drizzle-orm";
|
|
||||||
import { fireHealthCheckUnhealthyAlert } from "./healthCheckEvents";
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Public API
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fire a `site_online` alert for the given site.
|
|
||||||
*
|
|
||||||
* Call this after the site has been confirmed reachable / connected so that
|
|
||||||
* any matching `alertRules` can dispatch their email and webhook actions.
|
|
||||||
*
|
|
||||||
* @param orgId - Organisation that owns the site.
|
|
||||||
* @param siteId - Numeric primary key of the site.
|
|
||||||
* @param siteName - Human-readable name shown in notifications (optional).
|
|
||||||
* @param extra - Any additional key/value pairs to include in the payload.
|
|
||||||
*/
|
|
||||||
export async function fireSiteOnlineAlert(
|
|
||||||
orgId: string,
|
|
||||||
siteId: number,
|
|
||||||
siteName?: string,
|
|
||||||
extra?: Record<string, unknown>,
|
|
||||||
trx: Transaction | typeof db = db
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
await logsDb.insert(statusHistory).values({
|
|
||||||
entityType: "site",
|
|
||||||
entityId: siteId,
|
|
||||||
orgId: orgId,
|
|
||||||
status: "online",
|
|
||||||
timestamp: Math.floor(Date.now() / 1000)
|
|
||||||
});
|
|
||||||
await invalidateStatusHistoryCache("site", siteId);
|
|
||||||
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "site_online",
|
|
||||||
orgId,
|
|
||||||
siteId,
|
|
||||||
data: {
|
|
||||||
...(siteName != null ? { siteName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "site_toggle",
|
|
||||||
orgId,
|
|
||||||
siteId,
|
|
||||||
data: {
|
|
||||||
siteId,
|
|
||||||
status: "online",
|
|
||||||
...(siteName != null ? { siteName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
`fireSiteOnlineAlert: unexpected error for siteId ${siteId}`,
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fire a `site_offline` alert for the given site.
|
|
||||||
*
|
|
||||||
* Call this after the site has been detected as unreachable / disconnected so
|
|
||||||
* that any matching `alertRules` can dispatch their email and webhook actions.
|
|
||||||
*
|
|
||||||
* @param orgId - Organisation that owns the site.
|
|
||||||
* @param siteId - Numeric primary key of the site.
|
|
||||||
* @param siteName - Human-readable name shown in notifications (optional).
|
|
||||||
* @param extra - Any additional key/value pairs to include in the payload.
|
|
||||||
*/
|
|
||||||
export async function fireSiteOfflineAlert(
|
|
||||||
orgId: string,
|
|
||||||
siteId: number,
|
|
||||||
siteName?: string,
|
|
||||||
extra?: Record<string, unknown>,
|
|
||||||
trx: Transaction | typeof db = db
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
await logsDb.insert(statusHistory).values({
|
|
||||||
entityType: "site",
|
|
||||||
entityId: siteId,
|
|
||||||
orgId: orgId,
|
|
||||||
status: "offline",
|
|
||||||
timestamp: Math.floor(Date.now() / 1000)
|
|
||||||
});
|
|
||||||
await invalidateStatusHistoryCache("site", siteId);
|
|
||||||
|
|
||||||
const unhealthyHealthChecks = await trx
|
|
||||||
.update(targetHealthCheck)
|
|
||||||
.set({ hcHealth: "unhealthy" })
|
|
||||||
.where(
|
|
||||||
and(
|
|
||||||
eq(targetHealthCheck.orgId, orgId),
|
|
||||||
eq(targetHealthCheck.siteId, siteId),
|
|
||||||
eq(targetHealthCheck.hcEnabled, true) // only effect the ones that are enabled
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.returning();
|
|
||||||
|
|
||||||
for (const healthCheck of unhealthyHealthChecks) {
|
|
||||||
logger.info(
|
|
||||||
`Marking health check ${healthCheck.targetHealthCheckId} unhealthy due to site ${siteId} being marked offline`
|
|
||||||
);
|
|
||||||
|
|
||||||
await fireHealthCheckUnhealthyAlert(
|
|
||||||
healthCheck.orgId,
|
|
||||||
healthCheck.targetHealthCheckId,
|
|
||||||
healthCheck.name,
|
|
||||||
healthCheck.targetId, // for the resource if we have one
|
|
||||||
undefined,
|
|
||||||
true,
|
|
||||||
trx
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "site_offline",
|
|
||||||
orgId,
|
|
||||||
siteId,
|
|
||||||
data: {
|
|
||||||
...(siteName != null ? { siteName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
await processAlerts({
|
|
||||||
eventType: "site_toggle",
|
|
||||||
orgId,
|
|
||||||
siteId,
|
|
||||||
data: {
|
|
||||||
siteId,
|
|
||||||
status: "offline",
|
|
||||||
...(siteName != null ? { siteName } : {}),
|
|
||||||
...extra
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
`fireSiteOfflineAlert: unexpected error for siteId ${siteId}`,
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,6 +14,3 @@
|
|||||||
export * from "./processAlerts";
|
export * from "./processAlerts";
|
||||||
export * from "./sendAlertWebhook";
|
export * from "./sendAlertWebhook";
|
||||||
export * from "./sendAlertEmail";
|
export * from "./sendAlertEmail";
|
||||||
export * from "./events/siteEvents";
|
|
||||||
export * from "./events/healthCheckEvents";
|
|
||||||
export * from "./events/resourceEvents";
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ import { eq, and } from "drizzle-orm";
|
|||||||
import {
|
import {
|
||||||
fireHealthCheckHealthyAlert,
|
fireHealthCheckHealthyAlert,
|
||||||
fireHealthCheckUnhealthyAlert
|
fireHealthCheckUnhealthyAlert
|
||||||
} from "#private/lib/alerts/events/healthCheckEvents";
|
} from "@server/lib/alerts";
|
||||||
|
|
||||||
const paramsSchema = z.strictObject({
|
const paramsSchema = z.strictObject({
|
||||||
orgId: z.string().nonempty(),
|
orgId: z.string().nonempty(),
|
||||||
@@ -73,10 +73,7 @@ export async function triggerHealthCheckAlert(
|
|||||||
.from(targetHealthCheck)
|
.from(targetHealthCheck)
|
||||||
.where(
|
.where(
|
||||||
and(
|
and(
|
||||||
eq(
|
eq(targetHealthCheck.targetHealthCheckId, healthCheckId),
|
||||||
targetHealthCheck.targetHealthCheckId,
|
|
||||||
healthCheckId
|
|
||||||
),
|
|
||||||
eq(targetHealthCheck.orgId, orgId)
|
eq(targetHealthCheck.orgId, orgId)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ import {
|
|||||||
fireResourceHealthyAlert,
|
fireResourceHealthyAlert,
|
||||||
fireResourceUnhealthyAlert,
|
fireResourceUnhealthyAlert,
|
||||||
fireResourceDegradedAlert
|
fireResourceDegradedAlert
|
||||||
} from "#private/lib/alerts/events/resourceEvents";
|
} from "@server/lib/alerts";
|
||||||
|
|
||||||
const paramsSchema = z.strictObject({
|
const paramsSchema = z.strictObject({
|
||||||
orgId: z.string().nonempty(),
|
orgId: z.string().nonempty(),
|
||||||
|
|||||||
@@ -21,10 +21,7 @@ import createHttpError from "http-errors";
|
|||||||
import logger from "@server/logger";
|
import logger from "@server/logger";
|
||||||
import { fromError } from "zod-validation-error";
|
import { fromError } from "zod-validation-error";
|
||||||
import { eq, and } from "drizzle-orm";
|
import { eq, and } from "drizzle-orm";
|
||||||
import {
|
import { fireSiteOnlineAlert, fireSiteOfflineAlert } from "@server/lib/alerts";
|
||||||
fireSiteOnlineAlert,
|
|
||||||
fireSiteOfflineAlert
|
|
||||||
} from "#private/lib/alerts/events/siteEvents";
|
|
||||||
|
|
||||||
const paramsSchema = z.strictObject({
|
const paramsSchema = z.strictObject({
|
||||||
orgId: z.string().nonempty(),
|
orgId: z.string().nonempty(),
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ import logger from "@server/logger";
|
|||||||
import { fromError } from "zod-validation-error";
|
import { fromError } from "zod-validation-error";
|
||||||
import { OpenAPITags, registry } from "@server/openApi";
|
import { OpenAPITags, registry } from "@server/openApi";
|
||||||
import { addStandaloneHealthCheck } from "@server/routers/newt/targets";
|
import { addStandaloneHealthCheck } from "@server/routers/newt/targets";
|
||||||
import { fireHealthCheckUnhealthyAlert } from "#private/lib/alerts";
|
import { fireHealthCheckUnhealthyAlert } from "@server/lib/alerts";
|
||||||
|
|
||||||
const paramsSchema = z.strictObject({
|
const paramsSchema = z.strictObject({
|
||||||
orgId: z.string().nonempty()
|
orgId: z.string().nonempty()
|
||||||
|
|||||||
@@ -22,7 +22,11 @@ import { fromError } from "zod-validation-error";
|
|||||||
import { OpenAPITags, registry } from "@server/openApi";
|
import { OpenAPITags, registry } from "@server/openApi";
|
||||||
import { and, eq, isNull } from "drizzle-orm";
|
import { and, eq, isNull } from "drizzle-orm";
|
||||||
import { addStandaloneHealthCheck } from "@server/routers/newt/targets";
|
import { addStandaloneHealthCheck } from "@server/routers/newt/targets";
|
||||||
import { fireHealthCheckUnhealthyAlert, fireHealthCheckUnknownAlert, fireHealthCheckHealthyAlert } from "#private/lib/alerts";
|
import {
|
||||||
|
fireHealthCheckUnhealthyAlert,
|
||||||
|
fireHealthCheckUnknownAlert,
|
||||||
|
fireHealthCheckHealthyAlert
|
||||||
|
} from "@server/lib/alerts";
|
||||||
|
|
||||||
const paramsSchema = z
|
const paramsSchema = z
|
||||||
.object({
|
.object({
|
||||||
@@ -234,7 +238,10 @@ export async function updateHealthCheck(
|
|||||||
)
|
)
|
||||||
.returning();
|
.returning();
|
||||||
|
|
||||||
if (updated.hcHealth === "unhealthy" && existingHealthCheck.hcHealth !== "unhealthy") {
|
if (
|
||||||
|
updated.hcHealth === "unhealthy" &&
|
||||||
|
existingHealthCheck.hcHealth !== "unhealthy"
|
||||||
|
) {
|
||||||
await fireHealthCheckUnhealthyAlert(
|
await fireHealthCheckUnhealthyAlert(
|
||||||
updated.orgId,
|
updated.orgId,
|
||||||
updated.targetHealthCheckId,
|
updated.targetHealthCheckId,
|
||||||
@@ -243,7 +250,10 @@ export async function updateHealthCheck(
|
|||||||
undefined,
|
undefined,
|
||||||
false // dont send the alert because we just want to create the alert, not notify users yet
|
false // dont send the alert because we just want to create the alert, not notify users yet
|
||||||
);
|
);
|
||||||
} else if (updated.hcHealth === "unknown" && existingHealthCheck.hcHealth !== "unknown") {
|
} else if (
|
||||||
|
updated.hcHealth === "unknown" &&
|
||||||
|
existingHealthCheck.hcHealth !== "unknown"
|
||||||
|
) {
|
||||||
// if the health is unknown, we want to fire an alert to notify users to enable health checks
|
// if the health is unknown, we want to fire an alert to notify users to enable health checks
|
||||||
await fireHealthCheckUnknownAlert(
|
await fireHealthCheckUnknownAlert(
|
||||||
updated.orgId,
|
updated.orgId,
|
||||||
@@ -253,7 +263,10 @@ export async function updateHealthCheck(
|
|||||||
undefined,
|
undefined,
|
||||||
false // dont send the alert because we just want to create the alert, not notify users yet
|
false // dont send the alert because we just want to create the alert, not notify users yet
|
||||||
);
|
);
|
||||||
} else if (updated.hcHealth === "healthy" && existingHealthCheck.hcHealth !== "healthy") {
|
} else if (
|
||||||
|
updated.hcHealth === "healthy" &&
|
||||||
|
existingHealthCheck.hcHealth !== "healthy"
|
||||||
|
) {
|
||||||
await fireHealthCheckHealthyAlert(
|
await fireHealthCheckHealthyAlert(
|
||||||
updated.orgId,
|
updated.orgId,
|
||||||
updated.targetHealthCheckId,
|
updated.targetHealthCheckId,
|
||||||
@@ -264,7 +277,6 @@ export async function updateHealthCheck(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// Push updated health check to newt if the site is a newt site
|
// Push updated health check to newt if the site is a newt site
|
||||||
const [newt] = await db
|
const [newt] = await db
|
||||||
.select()
|
.select()
|
||||||
|
|||||||
@@ -1,12 +1,8 @@
|
|||||||
import { MessageHandler } from "@server/routers/ws";
|
import { MessageHandler } from "@server/routers/ws";
|
||||||
import {
|
import { db, Newt, sites } from "@server/db";
|
||||||
db,
|
|
||||||
Newt,
|
|
||||||
sites
|
|
||||||
} from "@server/db";
|
|
||||||
import { eq } from "drizzle-orm";
|
import { eq } from "drizzle-orm";
|
||||||
import logger from "@server/logger";
|
import logger from "@server/logger";
|
||||||
import { fireSiteOfflineAlert } from "#dynamic/lib/alerts";
|
import { fireSiteOfflineAlert } from "@server/lib/alerts";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handles disconnecting messages from sites to show disconnected in the ui
|
* Handles disconnecting messages from sites to show disconnected in the ui
|
||||||
@@ -38,7 +34,13 @@ export const handleNewtDisconnectingMessage: MessageHandler = async (
|
|||||||
.where(eq(sites.siteId, newt.siteId!))
|
.where(eq(sites.siteId, newt.siteId!))
|
||||||
.returning();
|
.returning();
|
||||||
|
|
||||||
await fireSiteOfflineAlert(site.orgId, site.siteId, site.name, undefined, trx);
|
await fireSiteOfflineAlert(
|
||||||
|
site.orgId,
|
||||||
|
site.siteId,
|
||||||
|
site.name,
|
||||||
|
undefined,
|
||||||
|
trx
|
||||||
|
);
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error("Error handling disconnecting message", { error });
|
logger.error("Error handling disconnecting message", { error });
|
||||||
|
|||||||
@@ -1,12 +1,8 @@
|
|||||||
import {
|
import { db, newts, sites } from "@server/db";
|
||||||
db,
|
|
||||||
newts,
|
|
||||||
sites
|
|
||||||
} from "@server/db";
|
|
||||||
import { hasActiveConnections } from "#dynamic/routers/ws";
|
import { hasActiveConnections } from "#dynamic/routers/ws";
|
||||||
import { eq, lt, isNull, and, or, ne, not, inArray } from "drizzle-orm";
|
import { eq, lt, isNull, and, or, ne, not, inArray } from "drizzle-orm";
|
||||||
import logger from "@server/logger";
|
import logger from "@server/logger";
|
||||||
import { fireSiteOfflineAlert, fireSiteOnlineAlert } from "#dynamic/lib/alerts";
|
import { fireSiteOfflineAlert, fireSiteOnlineAlert } from "@server/lib/alerts";
|
||||||
|
|
||||||
// Track if the offline checker interval is running
|
// Track if the offline checker interval is running
|
||||||
let offlineCheckerInterval: NodeJS.Timeout | null = null;
|
let offlineCheckerInterval: NodeJS.Timeout | null = null;
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { db } from "@server/db";
|
|||||||
import { sites, clients, olms } from "@server/db";
|
import { sites, clients, olms } from "@server/db";
|
||||||
import { and, eq, inArray } from "drizzle-orm";
|
import { and, eq, inArray } from "drizzle-orm";
|
||||||
import logger from "@server/logger";
|
import logger from "@server/logger";
|
||||||
import { fireSiteOnlineAlert } from "#dynamic/lib/alerts";
|
import { fireSiteOnlineAlert } from "@server/lib/alerts";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Ping Accumulator
|
* Ping Accumulator
|
||||||
@@ -127,7 +127,11 @@ async function flushSitePingsToDb(): Promise<void> {
|
|||||||
eq(sites.online, false)
|
eq(sites.online, false)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
.returning({ siteId: sites.siteId, orgId: sites.orgId, name: sites.name });
|
.returning({
|
||||||
|
siteId: sites.siteId,
|
||||||
|
orgId: sites.orgId,
|
||||||
|
name: sites.name
|
||||||
|
});
|
||||||
|
|
||||||
// Update lastPing for sites that were already online.
|
// Update lastPing for sites that were already online.
|
||||||
// After the update above, the newly-online sites now have
|
// After the update above, the newly-online sites now have
|
||||||
@@ -148,7 +152,13 @@ async function flushSitePingsToDb(): Promise<void> {
|
|||||||
|
|
||||||
for (const site of newlyOnlineSites) {
|
for (const site of newlyOnlineSites) {
|
||||||
await db.transaction(async (trx) => {
|
await db.transaction(async (trx) => {
|
||||||
await fireSiteOnlineAlert(site.orgId, site.siteId, site.name, undefined, trx);
|
await fireSiteOnlineAlert(
|
||||||
|
site.orgId,
|
||||||
|
site.siteId,
|
||||||
|
site.name,
|
||||||
|
undefined,
|
||||||
|
trx
|
||||||
|
);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import {
|
|||||||
fireHealthCheckHealthyAlert,
|
fireHealthCheckHealthyAlert,
|
||||||
fireHealthCheckUnhealthyAlert,
|
fireHealthCheckUnhealthyAlert,
|
||||||
fireHealthCheckUnknownAlert
|
fireHealthCheckUnknownAlert
|
||||||
} from "#dynamic/lib/alerts";
|
} from "@server/lib/alerts";
|
||||||
|
|
||||||
const createTargetParamsSchema = z.strictObject({
|
const createTargetParamsSchema = z.strictObject({
|
||||||
resourceId: z.string().transform(Number).pipe(z.int().positive())
|
resourceId: z.string().transform(Number).pipe(z.int().positive())
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import logger from "@server/logger";
|
|||||||
import {
|
import {
|
||||||
fireHealthCheckHealthyAlert,
|
fireHealthCheckHealthyAlert,
|
||||||
fireHealthCheckUnhealthyAlert
|
fireHealthCheckUnhealthyAlert
|
||||||
} from "#dynamic/lib/alerts";
|
} from "@server/lib/alerts";
|
||||||
|
|
||||||
interface TargetHealthStatus {
|
interface TargetHealthStatus {
|
||||||
status: string;
|
status: string;
|
||||||
|
|||||||
@@ -10,7 +10,11 @@ import logger from "@server/logger";
|
|||||||
import { fromError } from "zod-validation-error";
|
import { fromError } from "zod-validation-error";
|
||||||
import { addPeer } from "../gerbil/peers";
|
import { addPeer } from "../gerbil/peers";
|
||||||
import { addTargets } from "../newt/targets";
|
import { addTargets } from "../newt/targets";
|
||||||
import { fireHealthCheckHealthyAlert, fireHealthCheckUnknownAlert, fireHealthCheckUnhealthyAlert } from "#dynamic/lib/alerts";
|
import {
|
||||||
|
fireHealthCheckHealthyAlert,
|
||||||
|
fireHealthCheckUnknownAlert,
|
||||||
|
fireHealthCheckUnhealthyAlert
|
||||||
|
} from "@server/lib/alerts";
|
||||||
import { pickPort } from "./helpers";
|
import { pickPort } from "./helpers";
|
||||||
import { isTargetValid } from "@server/lib/validators";
|
import { isTargetValid } from "@server/lib/validators";
|
||||||
import { OpenAPITags, registry } from "@server/openApi";
|
import { OpenAPITags, registry } from "@server/openApi";
|
||||||
@@ -169,7 +173,7 @@ export async function updateTarget(
|
|||||||
let updatedTarget: any;
|
let updatedTarget: any;
|
||||||
let updatedHc: any;
|
let updatedHc: any;
|
||||||
await db.transaction(async (trx) => {
|
await db.transaction(async (trx) => {
|
||||||
[updatedTarget] = await trx
|
[updatedTarget] = await trx
|
||||||
.update(targets)
|
.update(targets)
|
||||||
.set({
|
.set({
|
||||||
siteId: parsedBody.data.siteId,
|
siteId: parsedBody.data.siteId,
|
||||||
@@ -181,8 +185,12 @@ export async function updateTarget(
|
|||||||
path: parsedBody.data.path,
|
path: parsedBody.data.path,
|
||||||
pathMatchType: parsedBody.data.pathMatchType,
|
pathMatchType: parsedBody.data.pathMatchType,
|
||||||
priority: parsedBody.data.priority,
|
priority: parsedBody.data.priority,
|
||||||
rewritePath: pathMatchTypeRemoved ? null : parsedBody.data.rewritePath,
|
rewritePath: pathMatchTypeRemoved
|
||||||
rewritePathType: pathMatchTypeRemoved ? null : parsedBody.data.rewritePathType
|
? null
|
||||||
|
: parsedBody.data.rewritePath,
|
||||||
|
rewritePathType: pathMatchTypeRemoved
|
||||||
|
? null
|
||||||
|
: parsedBody.data.rewritePathType
|
||||||
})
|
})
|
||||||
.where(eq(targets.targetId, targetId))
|
.where(eq(targets.targetId, targetId))
|
||||||
.returning();
|
.returning();
|
||||||
@@ -213,7 +221,8 @@ export async function updateTarget(
|
|||||||
// If hcEnabled is being turned on (was false, now true), set to "unhealthy"
|
// If hcEnabled is being turned on (was false, now true), set to "unhealthy"
|
||||||
// so the target must pass a health check before being considered healthy.
|
// so the target must pass a health check before being considered healthy.
|
||||||
const hcEnabledTurnedOn =
|
const hcEnabledTurnedOn =
|
||||||
parsedBody.data.hcEnabled === true && existingHc.hcEnabled === false;
|
parsedBody.data.hcEnabled === true &&
|
||||||
|
existingHc.hcEnabled === false;
|
||||||
|
|
||||||
let hcHealthValue: "unknown" | "healthy" | "unhealthy" | undefined;
|
let hcHealthValue: "unknown" | "healthy" | "unhealthy" | undefined;
|
||||||
if (
|
if (
|
||||||
@@ -253,7 +262,10 @@ export async function updateTarget(
|
|||||||
.where(eq(targetHealthCheck.targetId, targetId))
|
.where(eq(targetHealthCheck.targetId, targetId))
|
||||||
.returning();
|
.returning();
|
||||||
|
|
||||||
if (updatedHc.hcHealth === "unhealthy" && existingHc.hcHealth !== "unhealthy") {
|
if (
|
||||||
|
updatedHc.hcHealth === "unhealthy" &&
|
||||||
|
existingHc.hcHealth !== "unhealthy"
|
||||||
|
) {
|
||||||
logger.debug(
|
logger.debug(
|
||||||
`Health check ${updatedHc.targetHealthCheckId} for target ${targetId} is now unhealthy, firing alert`
|
`Health check ${updatedHc.targetHealthCheckId} for target ${targetId} is now unhealthy, firing alert`
|
||||||
);
|
);
|
||||||
@@ -266,7 +278,10 @@ export async function updateTarget(
|
|||||||
false, // dont send the alert because we just want to create the alert, not notify users yet
|
false, // dont send the alert because we just want to create the alert, not notify users yet
|
||||||
trx
|
trx
|
||||||
);
|
);
|
||||||
} else if (updatedHc.hcHealth === "unknown" && existingHc.hcHealth !== "unknown") {
|
} else if (
|
||||||
|
updatedHc.hcHealth === "unknown" &&
|
||||||
|
existingHc.hcHealth !== "unknown"
|
||||||
|
) {
|
||||||
logger.debug(
|
logger.debug(
|
||||||
`Health check ${updatedHc.targetHealthCheckId} for target ${targetId} is now unknown, firing alert`
|
`Health check ${updatedHc.targetHealthCheckId} for target ${targetId} is now unknown, firing alert`
|
||||||
);
|
);
|
||||||
@@ -280,7 +295,10 @@ export async function updateTarget(
|
|||||||
false, // dont send the alert because we just want to create the alert, not notify users yet
|
false, // dont send the alert because we just want to create the alert, not notify users yet
|
||||||
trx
|
trx
|
||||||
);
|
);
|
||||||
} else if (updatedHc.hcHealth === "healthy" && existingHc.hcHealth !== "healthy") {
|
} else if (
|
||||||
|
updatedHc.hcHealth === "healthy" &&
|
||||||
|
existingHc.hcHealth !== "healthy"
|
||||||
|
) {
|
||||||
logger.debug(
|
logger.debug(
|
||||||
`Health check ${updatedHc.targetHealthCheckId} for target ${targetId} is now healthy, firing alert`
|
`Health check ${updatedHc.targetHealthCheckId} for target ${targetId} is now healthy, firing alert`
|
||||||
);
|
);
|
||||||
|
|||||||
Reference in New Issue
Block a user