mirror of
https://github.com/fosrl/pangolin.git
synced 2026-06-03 22:35:59 +00:00
Merge branch 'dev' of https://github.com/fosrl/pangolin into dev
This commit is contained in:
@@ -484,6 +484,7 @@ export const alertRules = pgTable("alertRules", {
|
||||
| "health_check_toggle"
|
||||
| "resource_healthy"
|
||||
| "resource_unhealthy"
|
||||
| "resource_degraded"
|
||||
| "resource_toggle"
|
||||
>()
|
||||
.notNull(),
|
||||
|
||||
@@ -425,10 +425,18 @@ export const eventStreamingDestinations = sqliteTable(
|
||||
orgId: text("orgId")
|
||||
.notNull()
|
||||
.references(() => orgs.orgId, { onDelete: "cascade" }),
|
||||
sendConnectionLogs: integer("sendConnectionLogs", { mode: "boolean" }).notNull().default(false),
|
||||
sendRequestLogs: integer("sendRequestLogs", { mode: "boolean" }).notNull().default(false),
|
||||
sendActionLogs: integer("sendActionLogs", { mode: "boolean" }).notNull().default(false),
|
||||
sendAccessLogs: integer("sendAccessLogs", { mode: "boolean" }).notNull().default(false),
|
||||
sendConnectionLogs: integer("sendConnectionLogs", { mode: "boolean" })
|
||||
.notNull()
|
||||
.default(false),
|
||||
sendRequestLogs: integer("sendRequestLogs", { mode: "boolean" })
|
||||
.notNull()
|
||||
.default(false),
|
||||
sendActionLogs: integer("sendActionLogs", { mode: "boolean" })
|
||||
.notNull()
|
||||
.default(false),
|
||||
sendAccessLogs: integer("sendAccessLogs", { mode: "boolean" })
|
||||
.notNull()
|
||||
.default(false),
|
||||
type: text("type").notNull(), // e.g. "http", "kafka", etc.
|
||||
config: text("config").notNull(), // JSON string with the configuration for the destination
|
||||
enabled: integer("enabled", { mode: "boolean" })
|
||||
@@ -476,14 +484,19 @@ export const alertRules = sqliteTable("alertRules", {
|
||||
| "health_check_toggle"
|
||||
| "resource_healthy"
|
||||
| "resource_unhealthy"
|
||||
| "resource_degraded"
|
||||
| "resource_toggle"
|
||||
>()
|
||||
.notNull(),
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
cooldownSeconds: integer("cooldownSeconds").notNull().default(300),
|
||||
allSites: integer("allSites", { mode: "boolean" }).notNull().default(false),
|
||||
allHealthChecks: integer("allHealthChecks", { mode: "boolean" }).notNull().default(false),
|
||||
allResources: integer("allResources", { mode: "boolean" }).notNull().default(false),
|
||||
allHealthChecks: integer("allHealthChecks", { mode: "boolean" })
|
||||
.notNull()
|
||||
.default(false),
|
||||
allResources: integer("allResources", { mode: "boolean" })
|
||||
.notNull()
|
||||
.default(false),
|
||||
lastTriggeredAt: integer("lastTriggeredAt"),
|
||||
createdAt: integer("createdAt").notNull(),
|
||||
updatedAt: integer("updatedAt").notNull()
|
||||
@@ -531,19 +544,27 @@ export const alertEmailRecipients = sqliteTable("alertEmailRecipients", {
|
||||
recipientId: integer("recipientId").primaryKey({ autoIncrement: true }),
|
||||
emailActionId: integer("emailActionId")
|
||||
.notNull()
|
||||
.references(() => alertEmailActions.emailActionId, { onDelete: "cascade" }),
|
||||
userId: text("userId").references(() => users.userId, { onDelete: "cascade" }),
|
||||
roleId: integer("roleId").references(() => roles.roleId, { onDelete: "cascade" }),
|
||||
.references(() => alertEmailActions.emailActionId, {
|
||||
onDelete: "cascade"
|
||||
}),
|
||||
userId: text("userId").references(() => users.userId, {
|
||||
onDelete: "cascade"
|
||||
}),
|
||||
roleId: integer("roleId").references(() => roles.roleId, {
|
||||
onDelete: "cascade"
|
||||
}),
|
||||
email: text("email")
|
||||
});
|
||||
|
||||
export const alertWebhookActions = sqliteTable("alertWebhookActions", {
|
||||
webhookActionId: integer("webhookActionId").primaryKey({ autoIncrement: true }),
|
||||
webhookActionId: integer("webhookActionId").primaryKey({
|
||||
autoIncrement: true
|
||||
}),
|
||||
alertRuleId: integer("alertRuleId")
|
||||
.notNull()
|
||||
.references(() => alertRules.alertRuleId, { onDelete: "cascade" }),
|
||||
webhookUrl: text("webhookUrl").notNull(),
|
||||
config: text("config"), // encrypted JSON with auth config (authType, credentials)
|
||||
config: text("config"), // encrypted JSON with auth config (authType, credentials)
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
lastSentAt: integer("lastSentAt")
|
||||
});
|
||||
|
||||
@@ -23,6 +23,7 @@ export type AlertEventType =
|
||||
| "health_check_toggle"
|
||||
| "resource_healthy"
|
||||
| "resource_unhealthy"
|
||||
| "resource_degraded"
|
||||
| "resource_toggle";
|
||||
|
||||
export type AlertNotificationProps = {
|
||||
@@ -114,6 +115,15 @@ function getEventMeta(eventType: AlertEventType): {
|
||||
statusLabel: "Unhealthy",
|
||||
statusColor: "#dc2626"
|
||||
};
|
||||
case "resource_degraded":
|
||||
return {
|
||||
heading: "Resource Unhealthy",
|
||||
previewText: "A resource in your organization is not healthy.",
|
||||
summary:
|
||||
"A resource in your organization is currently unhealthy.",
|
||||
statusLabel: "Unhealthy",
|
||||
statusColor: "#dc2626"
|
||||
};
|
||||
case "resource_toggle":
|
||||
return {
|
||||
heading: "Resource Status Changed",
|
||||
@@ -135,7 +145,10 @@ function getEventMeta(eventType: AlertEventType): {
|
||||
}
|
||||
}
|
||||
|
||||
function resolveToggleStatus(status: unknown): { label: string; color: string } {
|
||||
function resolveToggleStatus(status: unknown): {
|
||||
label: string;
|
||||
color: string;
|
||||
} {
|
||||
switch (String(status).toLowerCase()) {
|
||||
case "online":
|
||||
return { label: "Online", color: "#16a34a" };
|
||||
|
||||
@@ -20,4 +20,15 @@ export async function fireHealthCheckUnhealthyAlert(
|
||||
trx?: unknown
|
||||
): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
export async function fireHealthCheckUnknownAlert(
|
||||
orgId: string,
|
||||
healthCheckId: number,
|
||||
healthCheckName?: string | null,
|
||||
healthCheckTargetId?: number | null,
|
||||
extra?: Record<string, unknown>,
|
||||
trx?: unknown
|
||||
): Promise<void> {
|
||||
return;
|
||||
}
|
||||
@@ -34,6 +34,7 @@ import { hashPassword } from "@server/auth/password";
|
||||
import { isValidCIDR, isValidIP, isValidUrlGlobPattern } from "../validators";
|
||||
import { isValidRegionId } from "@server/db/regions";
|
||||
import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed";
|
||||
import { fireHealthCheckUnknownAlert } from "#dynamic/lib/alerts";
|
||||
import { tierMatrix } from "../billing/tierMatrix";
|
||||
|
||||
export type ProxyResourcesResults = {
|
||||
@@ -169,6 +170,18 @@ export async function updateProxyResources(
|
||||
.returning();
|
||||
|
||||
healthchecksToUpdate.push(newHealthcheck);
|
||||
|
||||
// Insert unknown status history when HC is created in disabled state
|
||||
if (!healthcheckData?.enabled) {
|
||||
await fireHealthCheckUnknownAlert(
|
||||
orgId,
|
||||
newHealthcheck.targetHealthCheckId,
|
||||
newHealthcheck.name,
|
||||
newHealthcheck.targetId,
|
||||
undefined,
|
||||
trx
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Find existing resource by niceId and orgId
|
||||
@@ -557,6 +570,20 @@ export async function updateProxyResources(
|
||||
targetsToUpdate.push(updatedTarget);
|
||||
}
|
||||
}
|
||||
|
||||
// Insert unknown status history when HC is disabled
|
||||
const isDisablingHc =
|
||||
!healthcheckData?.enabled && oldHealthcheck?.hcEnabled;
|
||||
if (isDisablingHc) {
|
||||
await fireHealthCheckUnknownAlert(
|
||||
orgId,
|
||||
newHealthcheck.targetHealthCheckId,
|
||||
newHealthcheck.name,
|
||||
newHealthcheck.targetId,
|
||||
undefined,
|
||||
trx
|
||||
);
|
||||
}
|
||||
} else {
|
||||
await createTarget(existingResource.resourceId, targetData);
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ export interface StatusHistoryDayBucket {
|
||||
uptimePercent: number; // 0-100
|
||||
totalDowntimeSeconds: number;
|
||||
downtimeWindows: { start: number; end: number | null; status: string }[];
|
||||
status: "good" | "degraded" | "bad" | "no_data";
|
||||
status: "good" | "degraded" | "bad" | "no_data" | "unknown";
|
||||
}
|
||||
|
||||
export interface StatusHistoryResponse {
|
||||
@@ -54,6 +54,7 @@ export function computeBuckets(
|
||||
|
||||
const windows: { start: number; end: number | null; status: string }[] = [];
|
||||
let dayDowntime = 0;
|
||||
let dayDegradedTime = 0;
|
||||
|
||||
let windowStart = dayStartSec;
|
||||
let windowStatus = currentStatus;
|
||||
@@ -63,8 +64,8 @@ export function computeBuckets(
|
||||
const windowEnd = evt.timestamp;
|
||||
const isDown =
|
||||
windowStatus === "offline" ||
|
||||
windowStatus === "unhealthy" ||
|
||||
windowStatus === "unknown";
|
||||
windowStatus === "unhealthy";
|
||||
const isDegraded = windowStatus === "degraded";
|
||||
if (isDown) {
|
||||
dayDowntime += windowEnd - windowStart;
|
||||
windows.push({
|
||||
@@ -72,6 +73,13 @@ export function computeBuckets(
|
||||
end: windowEnd,
|
||||
status: windowStatus,
|
||||
});
|
||||
} else if (isDegraded) {
|
||||
dayDegradedTime += windowEnd - windowStart;
|
||||
windows.push({
|
||||
start: windowStart,
|
||||
end: windowEnd,
|
||||
status: windowStatus,
|
||||
});
|
||||
}
|
||||
}
|
||||
windowStart = evt.timestamp;
|
||||
@@ -83,8 +91,8 @@ export function computeBuckets(
|
||||
const finalEnd = Math.min(dayEndSec, nowSec);
|
||||
const isDown =
|
||||
windowStatus === "offline" ||
|
||||
windowStatus === "unhealthy" ||
|
||||
windowStatus === "unknown";
|
||||
windowStatus === "unhealthy";
|
||||
const isDegraded = windowStatus === "degraded";
|
||||
if (isDown && finalEnd > windowStart) {
|
||||
dayDowntime += finalEnd - windowStart;
|
||||
windows.push({
|
||||
@@ -92,6 +100,13 @@ export function computeBuckets(
|
||||
end: finalEnd,
|
||||
status: windowStatus,
|
||||
});
|
||||
} else if (isDegraded && finalEnd > windowStart) {
|
||||
dayDegradedTime += finalEnd - windowStart;
|
||||
windows.push({
|
||||
start: windowStart,
|
||||
end: finalEnd,
|
||||
status: windowStatus,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,7 +120,7 @@ export function computeBuckets(
|
||||
effectiveDayLength > 0
|
||||
? Math.max(
|
||||
0,
|
||||
((effectiveDayLength - dayDowntime) /
|
||||
((effectiveDayLength - dayDowntime - dayDegradedTime) /
|
||||
effectiveDayLength) *
|
||||
100
|
||||
)
|
||||
@@ -113,11 +128,27 @@ export function computeBuckets(
|
||||
|
||||
const dateStr = new Date(dayStartSec * 1000).toISOString().slice(0, 10);
|
||||
|
||||
const hasAnyData = currentStatus !== null || dayEvents.length > 0;
|
||||
|
||||
// The whole observable window is "unknown" if every status we have seen is unknown
|
||||
const allStatuses = [
|
||||
...(currentStatus !== null ? [currentStatus] : []),
|
||||
...dayEvents.map((e) => e.status)
|
||||
];
|
||||
const onlyUnknownData =
|
||||
hasAnyData && allStatuses.every((s) => s === "unknown");
|
||||
|
||||
let status: StatusHistoryDayBucket["status"] = "no_data";
|
||||
if (currentStatus !== null || dayEvents.length > 0) {
|
||||
if (uptimePct >= 99) status = "good";
|
||||
else if (uptimePct >= 50) status = "degraded";
|
||||
else status = "bad";
|
||||
if (hasAnyData) {
|
||||
if (onlyUnknownData) {
|
||||
status = "unknown";
|
||||
} else if (dayDowntime > 0 && uptimePct < 50) {
|
||||
status = "bad";
|
||||
} else if (dayDowntime > 0 || dayDegradedTime > 0) {
|
||||
status = "degraded";
|
||||
} else {
|
||||
status = "good";
|
||||
}
|
||||
}
|
||||
|
||||
buckets.push({
|
||||
|
||||
@@ -23,8 +23,10 @@ import {
|
||||
} from "@server/db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import {
|
||||
fireResourceDegradedAlert,
|
||||
fireResourceHealthyAlert,
|
||||
fireResourceUnhealthyAlert
|
||||
fireResourceUnhealthyAlert,
|
||||
fireResourceUnknownAlert
|
||||
} from "./resourceEvents";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -147,6 +149,32 @@ export async function fireHealthCheckUnhealthyAlert(
|
||||
}
|
||||
}
|
||||
|
||||
export async function fireHealthCheckUnknownAlert(
|
||||
orgId: string,
|
||||
healthCheckId: number,
|
||||
healthCheckName?: string | null,
|
||||
healthCheckTargetId?: number | null,
|
||||
extra?: Record<string, unknown>,
|
||||
trx: Transaction | typeof db = db
|
||||
): Promise<void> {
|
||||
try {
|
||||
await trx.insert(statusHistory).values({
|
||||
entityType: "health_check",
|
||||
entityId: healthCheckId,
|
||||
orgId: orgId,
|
||||
status: "unknown",
|
||||
timestamp: Math.floor(Date.now() / 1000)
|
||||
});
|
||||
|
||||
await handleResource(orgId, healthCheckTargetId, trx);
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`fireHealthCheckUnknownAlert: unexpected error for healthCheckId ${healthCheckId}`,
|
||||
err
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleResource(orgId: string, healthCheckTargetId?: number | null, trx: Transaction | typeof db = db) {
|
||||
if (!healthCheckTargetId) {
|
||||
return;
|
||||
@@ -177,10 +205,16 @@ async function handleResource(orgId: string, healthCheckTargetId?: number | null
|
||||
.where(eq(targets.resourceId, resource.resourceId));
|
||||
|
||||
let health = "healthy";
|
||||
const allUnknown = otherTargets.every((t) => t.hcHealth === "unknown");
|
||||
const allHealthy = otherTargets.every((t) => t.hcHealth === "healthy");
|
||||
const allUnhealthy = otherTargets.every((t) => t.hcHealth === "unhealthy");
|
||||
|
||||
if (allHealthy) {
|
||||
if (allUnknown) {
|
||||
logger.debug(
|
||||
`Marking resource ${resource.resourceId} as unknown because all health checks are disabled`
|
||||
);
|
||||
health = "unknown";
|
||||
} else if (allHealthy) {
|
||||
health = "healthy";
|
||||
} else if (allUnhealthy) {
|
||||
logger.debug(
|
||||
@@ -201,7 +235,15 @@ async function handleResource(orgId: string, healthCheckTargetId?: number | null
|
||||
.set({ health })
|
||||
.where(eq(resources.resourceId, resource.resourceId));
|
||||
|
||||
if (health === "unhealthy") {
|
||||
if (health === "unknown") {
|
||||
await fireResourceUnknownAlert(
|
||||
orgId,
|
||||
resource.resourceId,
|
||||
resource.name,
|
||||
undefined,
|
||||
trx
|
||||
);
|
||||
} else if (health === "unhealthy") {
|
||||
await fireResourceUnhealthyAlert(
|
||||
orgId,
|
||||
resource.resourceId,
|
||||
@@ -217,6 +259,14 @@ async function handleResource(orgId: string, healthCheckTargetId?: number | null
|
||||
undefined,
|
||||
trx
|
||||
);
|
||||
} else if (health === "degraded") {
|
||||
await fireResourceDegradedAlert(
|
||||
orgId,
|
||||
resource.resourceId,
|
||||
resource.name,
|
||||
undefined,
|
||||
trx
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,9 +130,9 @@ export async function fireResourceUnhealthyAlert(
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire a `resource_toggle` alert for the given resource.
|
||||
* Fire a `resource_degraded` alert for the given resource.
|
||||
*
|
||||
* Call this when a resource's enabled/disabled status is toggled so that any
|
||||
* Call this after a resource has been detected as degraded so that any
|
||||
* matching `alertRules` can dispatch their email and webhook actions.
|
||||
*
|
||||
* @param orgId - Organisation that owns the resource.
|
||||
@@ -140,7 +140,7 @@ export async function fireResourceUnhealthyAlert(
|
||||
* @param resourceName - Human-readable name shown in notifications (optional).
|
||||
* @param extra - Any additional key/value pairs to include in the payload.
|
||||
*/
|
||||
export async function fireResourceToggleAlert(
|
||||
export async function fireResourceDegradedAlert(
|
||||
orgId: string,
|
||||
resourceId: number,
|
||||
resourceName?: string | null,
|
||||
@@ -148,8 +148,16 @@ export async function fireResourceToggleAlert(
|
||||
trx: Transaction | typeof db = db
|
||||
): Promise<void> {
|
||||
try {
|
||||
await trx.insert(statusHistory).values({
|
||||
entityType: "resource",
|
||||
entityId: resourceId,
|
||||
orgId: orgId,
|
||||
status: "degraded",
|
||||
timestamp: Math.floor(Date.now() / 1000)
|
||||
});
|
||||
|
||||
await processAlerts({
|
||||
eventType: "resource_toggle",
|
||||
eventType: "resource_degraded",
|
||||
orgId,
|
||||
resourceId,
|
||||
data: {
|
||||
@@ -157,9 +165,66 @@ export async function fireResourceToggleAlert(
|
||||
...extra
|
||||
}
|
||||
});
|
||||
await processAlerts({
|
||||
eventType: "resource_toggle",
|
||||
orgId,
|
||||
resourceId,
|
||||
data: {
|
||||
resourceId,
|
||||
status: "degraded",
|
||||
...(resourceName != null ? { resourceName } : {}),
|
||||
...extra
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`fireResourceToggleAlert: unexpected error for resourceId ${resourceId}`,
|
||||
`fireResourceDegradedAlert: unexpected error for resourceId ${resourceId}`,
|
||||
err
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire a `resource_unknown` alert for the given resource.
|
||||
*
|
||||
* Call this when all health checks on a resource are disabled so that the
|
||||
* resource status transitions to unknown.
|
||||
*
|
||||
* @param orgId - Organisation that owns the resource.
|
||||
* @param resourceId - Numeric primary key of the resource.
|
||||
* @param resourceName - Human-readable name shown in notifications (optional).
|
||||
* @param extra - Any additional key/value pairs to include in the payload.
|
||||
*/
|
||||
export async function fireResourceUnknownAlert(
|
||||
orgId: string,
|
||||
resourceId: number,
|
||||
resourceName?: string | null,
|
||||
extra?: Record<string, unknown>,
|
||||
trx: Transaction | typeof db = db
|
||||
): Promise<void> {
|
||||
try {
|
||||
await trx.insert(statusHistory).values({
|
||||
entityType: "resource",
|
||||
entityId: resourceId,
|
||||
orgId: orgId,
|
||||
status: "unknown",
|
||||
timestamp: Math.floor(Date.now() / 1000)
|
||||
});
|
||||
|
||||
await processAlerts({
|
||||
eventType: "resource_toggle",
|
||||
orgId,
|
||||
resourceId,
|
||||
data: {
|
||||
resourceId,
|
||||
status: "unknown",
|
||||
...(resourceName != null ? { resourceName } : {}),
|
||||
...extra
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`fireResourceUnknownAlert: unexpected error for resourceId ${resourceId}`,
|
||||
err
|
||||
);
|
||||
}
|
||||
|
||||
@@ -88,6 +88,8 @@ function buildSubject(context: AlertContext): string {
|
||||
return "[Alert] Resource Healthy";
|
||||
case "resource_unhealthy":
|
||||
return "[Alert] Resource Unhealthy";
|
||||
case "resource_degraded":
|
||||
return "[Alert] Resource Degraded";
|
||||
case "resource_toggle":
|
||||
return "[Alert] Resource Status Changed";
|
||||
default: {
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
*/
|
||||
|
||||
import logger from "@server/logger";
|
||||
import { AlertContext, WebhookAlertConfig } from "@server/routers/alertRule/types";
|
||||
import {
|
||||
AlertContext,
|
||||
WebhookAlertConfig
|
||||
} from "@server/routers/alertRule/types";
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 15_000;
|
||||
const MAX_RETRIES = 3;
|
||||
@@ -56,7 +59,10 @@ export async function sendAlertWebhook(
|
||||
|
||||
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||
const controller = new AbortController();
|
||||
const timeoutHandle = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
const timeoutHandle = setTimeout(
|
||||
() => controller.abort(),
|
||||
REQUEST_TIMEOUT_MS
|
||||
);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
@@ -75,7 +81,9 @@ export async function sendAlertWebhook(
|
||||
);
|
||||
} else {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
lastError = new Error(`Alert webhook: request to "${url}" failed – ${msg}`);
|
||||
lastError = new Error(
|
||||
`Alert webhook: request to "${url}" failed – ${msg}`
|
||||
);
|
||||
}
|
||||
if (attempt < MAX_RETRIES) {
|
||||
const delay = RETRY_BASE_DELAY_MS * 2 ** (attempt - 1);
|
||||
@@ -111,11 +119,18 @@ export async function sendAlertWebhook(
|
||||
continue;
|
||||
}
|
||||
|
||||
logger.debug(`Alert webhook sent successfully to "${url}" for event "${context.eventType}" (attempt ${attempt}/${MAX_RETRIES})`);
|
||||
logger.debug(
|
||||
`Alert webhook sent successfully to "${url}" for event "${context.eventType}" (attempt ${attempt}/${MAX_RETRIES})`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
throw lastError ?? new Error(`Alert webhook: all ${MAX_RETRIES} attempts failed for "${url}"`);
|
||||
throw (
|
||||
lastError ??
|
||||
new Error(
|
||||
`Alert webhook: all ${MAX_RETRIES} attempts failed for "${url}"`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -139,6 +154,8 @@ function deriveStatus(
|
||||
case "health_check_unhealthy":
|
||||
case "resource_unhealthy":
|
||||
return "unhealthy";
|
||||
case "resource_degraded":
|
||||
return "degraded";
|
||||
case "health_check_toggle":
|
||||
case "resource_toggle":
|
||||
return String(data.status ?? "unknown");
|
||||
@@ -154,7 +171,9 @@ function deriveStatus(
|
||||
// Header construction (mirrors HttpLogDestination.buildHeaders)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function buildHeaders(webhookConfig: WebhookAlertConfig): Record<string, string> {
|
||||
function buildHeaders(
|
||||
webhookConfig: WebhookAlertConfig
|
||||
): Record<string, string> {
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json"
|
||||
};
|
||||
|
||||
63
server/private/lib/alerts/types.ts
Normal file
63
server/private/lib/alerts/types.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* This file is part of a proprietary work.
|
||||
*
|
||||
* Copyright (c) 2025-2026 Fossorial, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This file is licensed under the Fossorial Commercial License.
|
||||
* You may not use this file except in compliance with the License.
|
||||
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
|
||||
*
|
||||
* This file is not licensed under the AGPLv3.
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Alert event types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type AlertEventType =
|
||||
| "site_online"
|
||||
| "site_offline"
|
||||
| "health_check_healthy"
|
||||
| "health_check_not_healthy";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Webhook authentication config (stored as encrypted JSON in the DB)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type WebhookAuthType = "none" | "bearer" | "basic" | "custom";
|
||||
|
||||
/**
|
||||
* Stored as an encrypted JSON blob in `alertWebhookActions.config`.
|
||||
*/
|
||||
export interface WebhookAlertConfig {
|
||||
/** Authentication strategy for the webhook endpoint */
|
||||
authType: WebhookAuthType;
|
||||
/** Bearer token – used when authType === "bearer" */
|
||||
bearerToken?: string;
|
||||
/** Basic credentials – "username:password" – used when authType === "basic" */
|
||||
basicCredentials?: string;
|
||||
/** Custom header name – used when authType === "custom" */
|
||||
customHeaderName?: string;
|
||||
/** Custom header value – used when authType === "custom" */
|
||||
customHeaderValue?: string;
|
||||
/** Extra headers to send with every webhook request */
|
||||
headers?: Array<{ key: string; value: string }>;
|
||||
/** HTTP method (default POST) */
|
||||
method?: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internal alert event passed through the processing pipeline
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface AlertContext {
|
||||
eventType: AlertEventType;
|
||||
orgId: string;
|
||||
/** Set for site_online / site_offline events */
|
||||
siteId?: number;
|
||||
/** Set for health_check_* events */
|
||||
healthCheckId?: number;
|
||||
/** Human-readable context data included in emails and webhook payloads */
|
||||
data: Record<string, unknown>;
|
||||
}
|
||||
@@ -24,7 +24,8 @@ import { eq, and } from "drizzle-orm";
|
||||
import {
|
||||
fireResourceHealthyAlert,
|
||||
fireResourceUnhealthyAlert,
|
||||
fireResourceToggleAlert
|
||||
fireResourceToggleAlert,
|
||||
fireResourceDegradedAlert
|
||||
} from "#private/lib/alerts/events/resourceEvents";
|
||||
|
||||
const paramsSchema = z.strictObject({
|
||||
@@ -33,7 +34,12 @@ const paramsSchema = z.strictObject({
|
||||
});
|
||||
|
||||
const bodySchema = z.strictObject({
|
||||
eventType: z.enum(["resource_healthy", "resource_unhealthy", "resource_toggle"])
|
||||
eventType: z.enum([
|
||||
"resource_healthy",
|
||||
"resource_unhealthy",
|
||||
"resource_degraded",
|
||||
"resource_toggle"
|
||||
])
|
||||
});
|
||||
|
||||
export type TriggerResourceAlertResponse = {
|
||||
@@ -101,8 +107,8 @@ export async function triggerResourceAlert(
|
||||
resourceId,
|
||||
resource.name ?? undefined
|
||||
);
|
||||
} else {
|
||||
await fireResourceToggleAlert(
|
||||
} else if (eventType === "resource_degraded") {
|
||||
await fireResourceDegradedAlert(
|
||||
orgId,
|
||||
resourceId,
|
||||
resource.name ?? undefined
|
||||
|
||||
@@ -33,7 +33,11 @@ import { encrypt } from "@server/lib/crypto";
|
||||
import config from "@server/lib/config";
|
||||
import { CreateAlertRuleResponse } from "@server/routers/alertRule/types";
|
||||
|
||||
export const SITE_EVENT_TYPES = ["site_online", "site_offline", "site_toggle"] as const;
|
||||
export const SITE_EVENT_TYPES = [
|
||||
"site_online",
|
||||
"site_offline",
|
||||
"site_toggle"
|
||||
] as const;
|
||||
export const HC_EVENT_TYPES = [
|
||||
"health_check_healthy",
|
||||
"health_check_unhealthy",
|
||||
@@ -42,6 +46,7 @@ export const HC_EVENT_TYPES = [
|
||||
export const RESOURCE_EVENT_TYPES = [
|
||||
"resource_healthy",
|
||||
"resource_unhealthy",
|
||||
"resource_degraded",
|
||||
"resource_toggle"
|
||||
] as const;
|
||||
|
||||
@@ -92,19 +97,24 @@ const bodySchema = z
|
||||
const isHcEvent = (HC_EVENT_TYPES as readonly string[]).includes(
|
||||
val.eventType
|
||||
);
|
||||
const isResourceEvent = (RESOURCE_EVENT_TYPES as readonly string[]).includes(
|
||||
val.eventType
|
||||
);
|
||||
const isResourceEvent = (
|
||||
RESOURCE_EVENT_TYPES as readonly string[]
|
||||
).includes(val.eventType);
|
||||
|
||||
if (isSiteEvent && !val.allSites && val.siteIds.length === 0) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "At least one siteId is required for site event types when allSites is false",
|
||||
message:
|
||||
"At least one siteId is required for site event types when allSites is false",
|
||||
path: ["siteIds"]
|
||||
});
|
||||
}
|
||||
|
||||
if (isHcEvent && !val.allHealthChecks && val.healthCheckIds.length === 0) {
|
||||
if (
|
||||
isHcEvent &&
|
||||
!val.allHealthChecks &&
|
||||
val.healthCheckIds.length === 0
|
||||
) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message:
|
||||
@@ -129,10 +139,15 @@ const bodySchema = z
|
||||
});
|
||||
}
|
||||
|
||||
if (isResourceEvent && !val.allResources && val.resourceIds.length === 0) {
|
||||
if (
|
||||
isResourceEvent &&
|
||||
!val.allResources &&
|
||||
val.resourceIds.length === 0
|
||||
) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "At least one resourceId is required for resource event types when allResources is false",
|
||||
message:
|
||||
"At least one resourceId is required for resource event types when allResources is false",
|
||||
path: ["resourceIds"]
|
||||
});
|
||||
}
|
||||
@@ -148,7 +163,8 @@ const bodySchema = z
|
||||
if (isResourceEvent && val.healthCheckIds.length > 0) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "healthCheckIds must not be set for resource event types",
|
||||
message:
|
||||
"healthCheckIds must not be set for resource event types",
|
||||
path: ["healthCheckIds"]
|
||||
});
|
||||
}
|
||||
@@ -164,7 +180,8 @@ const bodySchema = z
|
||||
if (isHcEvent && val.resourceIds.length > 0) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "resourceIds must not be set for health check event types",
|
||||
message:
|
||||
"resourceIds must not be set for health check event types",
|
||||
path: ["resourceIds"]
|
||||
});
|
||||
}
|
||||
@@ -284,9 +301,7 @@ export async function createAlertRule(
|
||||
// Create the email action pivot row and recipients if any recipients
|
||||
// were supplied (userIds, roleIds, or raw emails).
|
||||
const hasRecipients =
|
||||
userIds.length > 0 ||
|
||||
roleIds.length > 0 ||
|
||||
emails.length > 0;
|
||||
userIds.length > 0 || roleIds.length > 0 || emails.length > 0;
|
||||
|
||||
if (hasRecipients) {
|
||||
const [emailActionRow] = await db
|
||||
|
||||
@@ -76,6 +76,7 @@ const SITE_ALERT_EVENT_TYPES = [
|
||||
const RESOURCE_ALERT_EVENT_TYPES = [
|
||||
"resource_healthy",
|
||||
"resource_unhealthy",
|
||||
"resource_degraded",
|
||||
"resource_toggle"
|
||||
] as const;
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ export type GetAlertRuleResponse = {
|
||||
| "health_check_toggle"
|
||||
| "resource_healthy"
|
||||
| "resource_unhealthy"
|
||||
| "resource_degraded"
|
||||
| "resource_toggle";
|
||||
enabled: boolean;
|
||||
cooldownSeconds: number;
|
||||
@@ -94,6 +95,7 @@ export type AlertEventType =
|
||||
| "health_check_toggle"
|
||||
| "resource_healthy"
|
||||
| "resource_unhealthy"
|
||||
| "resource_degraded"
|
||||
| "resource_toggle";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -138,6 +138,7 @@ export type ResourceWithTargets = {
|
||||
domainId: string | null;
|
||||
niceId: string;
|
||||
headerAuthId: number | null;
|
||||
wildcard: boolean;
|
||||
targets: Array<{
|
||||
targetId: number;
|
||||
ip: string;
|
||||
@@ -171,6 +172,7 @@ function queryResourcesBase() {
|
||||
enabled: resources.enabled,
|
||||
domainId: resources.domainId,
|
||||
niceId: resources.niceId,
|
||||
wildcard: resources.wildcard,
|
||||
headerAuthId: resourceHeaderAuth.headerAuthId,
|
||||
headerAuthExtendedCompatibilityId:
|
||||
resourceHeaderAuthExtendedCompatibility.headerAuthExtendedCompatibilityId,
|
||||
@@ -453,6 +455,7 @@ export async function listResources(
|
||||
http: row.http,
|
||||
protocol: row.protocol,
|
||||
proxyPort: row.proxyPort,
|
||||
wildcard: row.wildcard,
|
||||
enabled: row.enabled,
|
||||
domainId: row.domainId,
|
||||
headerAuthId: row.headerAuthId,
|
||||
|
||||
@@ -10,6 +10,7 @@ import logger from "@server/logger";
|
||||
import { fromError } from "zod-validation-error";
|
||||
import { addPeer } from "../gerbil/peers";
|
||||
import { addTargets } from "../newt/targets";
|
||||
import { fireHealthCheckUnknownAlert } from "#dynamic/lib/alerts";
|
||||
import { pickPort } from "./helpers";
|
||||
import { isTargetValid } from "@server/lib/validators";
|
||||
import { OpenAPITags, registry } from "@server/openApi";
|
||||
@@ -225,6 +226,11 @@ export async function updateTarget(
|
||||
hcHealthValue = undefined;
|
||||
}
|
||||
|
||||
const isDisablingHc =
|
||||
(parsedBody.data.hcEnabled === false ||
|
||||
parsedBody.data.hcEnabled === null) &&
|
||||
existingHc.hcEnabled === true;
|
||||
|
||||
const [updatedHc] = await db
|
||||
.update(targetHealthCheck)
|
||||
.set({
|
||||
@@ -250,6 +256,15 @@ export async function updateTarget(
|
||||
.where(eq(targetHealthCheck.targetId, targetId))
|
||||
.returning();
|
||||
|
||||
if (isDisablingHc) {
|
||||
await fireHealthCheckUnknownAlert(
|
||||
resource.orgId,
|
||||
existingHc.targetHealthCheckId,
|
||||
existingHc.name,
|
||||
updatedHc.targetId
|
||||
);
|
||||
}
|
||||
|
||||
if (site.pubKey) {
|
||||
if (site.type == "wireguard") {
|
||||
await addPeer(site.exitNodeId!, {
|
||||
|
||||
Reference in New Issue
Block a user