Merge pull request #3541 from fosrl/feat/test-alert-button

Feat: test alert rules
This commit is contained in:
Milo Schwartz
2026-08-19 10:33:16 -04:00
committed by GitHub
14 changed files with 574 additions and 74 deletions
@@ -0,0 +1,105 @@
import { db, userOrgRoles, users } from "@server/db";
import logger from "@server/logger";
import type {
EmailAlertAction,
TestAlertContext,
WebhookAlertConfig
} from "@server/routers/alertRule/types";
import { eq, inArray } from "drizzle-orm";
import { sendAlertEmail } from "./sendAlertEmail";
import { sendAlertWebhook } from "./sendAlertWebhook";
export async function processTestAlerts(context: TestAlertContext) {
// Process email actions
const emailActions = context.actions.filter(
(action) => action.type === "email"
);
for (const action of emailActions) {
try {
const recipients = await resolveEmailRecipients(action);
if (recipients.length > 0) {
await sendAlertEmail(recipients, {
...context,
isTest: true
});
}
} catch (err) {
logger.error(`processTestAlerts: failed to send alert email`, err);
}
}
// Process webhook actions
const webhookActions = context.actions.filter(
(action) => action.type === "webhook"
);
for (const action of webhookActions) {
try {
let webhookConfig: WebhookAlertConfig = { authType: "none" };
if (action.config) {
try {
webhookConfig = JSON.parse(
action.config
) as WebhookAlertConfig;
} catch (err) {
logger.error(
`processTestAlerts: failed to decrypt webhook`,
err
);
continue;
}
}
await sendAlertWebhook(action.webhookUrl, webhookConfig, {
...context,
isTest: true
});
} catch (err) {
logger.error(
`processTestAlerts: failed to send alert webhook `,
err
);
}
}
}
/**
* Resolves all email addresses for a given `emailActionId`.
*
* Recipients may be:
* - Direct users (by `userId`)
* - All users in a role (by `roleId`, resolved via `userOrgRoles`)
* - Direct external email addresses
*/
async function resolveEmailRecipients(
action: EmailAlertAction
): Promise<string[]> {
const emailList: string[] = [];
emailList.push(...(action.emails ?? []));
if (action.userIds && action.userIds?.length > 0) {
const userList = await db
.select({ email: users.email })
.from(users)
.where(inArray(users.userId, action.userIds));
emailList.push(
...userList.filter((u) => u.email !== null).map((u) => u.email!)
);
}
if (action.roleIds && action.roleIds?.length > 0) {
const userList = await db
.select({ email: users.email })
.from(userOrgRoles)
.innerJoin(users, eq(userOrgRoles.userId, users.userId))
.where(inArray(userOrgRoles.roleId, action.roleIds.map(Number)));
emailList.push(
...userList.filter((u) => u.email !== null).map((u) => u.email!)
);
}
return [...new Set(emailList)];
}
+31 -15
View File
@@ -15,7 +15,21 @@ import { sendEmail } from "@server/emails";
import AlertNotification from "@server/emails/templates/AlertNotification";
import config from "@server/lib/config";
import logger from "@server/logger";
import { AlertContext } from "@server/routers/alertRule/types";
import { type AlertEventType } from "@server/routers/alertRule/types";
type EmailAlertContext = {
eventType: AlertEventType;
orgId: string;
/** Set for site_online / site_offline events */
siteId?: number;
/** Set for health_check_* events */
healthCheckId?: number;
/** Set for resource_* events */
resourceId?: number;
/** Human-readable context data included in emails and webhook payloads */
data: Record<string, unknown>;
isTest?: boolean;
};
/**
* Sends an alert notification email to every address in `recipients`.
@@ -27,7 +41,7 @@ import { AlertContext } from "@server/routers/alertRule/types";
*/
export async function sendAlertEmail(
recipients: string[],
context: AlertContext
context: EmailAlertContext
): Promise<void> {
if (recipients.length === 0) {
return;
@@ -46,7 +60,8 @@ export async function sendAlertEmail(
eventType: context.eventType,
orgId: context.orgId,
data: context.data,
dashboardLink
dashboardLink,
isTestAlert: context.isTest
}),
{
from,
@@ -70,34 +85,35 @@ export async function sendAlertEmail(
// Helpers
// ---------------------------------------------------------------------------
function buildSubject(context: AlertContext): string {
function buildSubject(context: EmailAlertContext): string {
const prefix = context.isTest ? "[Test Alert]" : "[Alert]";
switch (context.eventType) {
case "site_online":
return "[Alert] Site Back Online";
return `${prefix} Site Back Online`;
case "site_offline":
return "[Alert] Site Offline";
return `${prefix} Site Offline`;
case "site_toggle":
return "[Alert] Site Status Changed";
return `${prefix} Site Status Changed`;
case "health_check_healthy":
return "[Alert] Health Check Recovered";
return `${prefix} Health Check Recovered`;
case "health_check_unhealthy":
return "[Alert] Health Check Failing";
return `${prefix} Health Check Failing`;
case "health_check_toggle":
return "[Alert] Health Check Status Changed";
return `${prefix} Health Check Status Changed`;
case "resource_healthy":
return "[Alert] Resource Healthy";
return `${prefix} Resource Healthy`;
case "resource_unhealthy":
return "[Alert] Resource Unhealthy";
return `${prefix} Resource Unhealthy`;
case "resource_degraded":
return "[Alert] Resource Degraded";
return `${prefix} Resource Degraded`;
case "resource_toggle":
return "[Alert] Resource Status Changed";
return `${prefix} Resource Status Changed`;
default: {
// Exhaustiveness fallback should never be reached with a
// well-typed caller, but keeps runtime behaviour predictable.
const _exhaustive: never = context.eventType;
void _exhaustive;
return "[Alert] Event Notification";
return `${prefix} Event Notification`;
}
}
}
+31 -10
View File
@@ -14,13 +14,28 @@
import logger from "@server/logger";
import {
AlertContext,
WebhookAlertConfig
WebhookAlertConfig,
type AlertEventType
} from "@server/routers/alertRule/types";
const REQUEST_TIMEOUT_MS = 15_000;
const MAX_RETRIES = 3;
const RETRY_BASE_DELAY_MS = 500;
type WebhookAlertContext = {
eventType: AlertEventType;
orgId: string;
/** Set for site_online / site_offline events */
siteId?: number;
/** Set for health_check_* events */
healthCheckId?: number;
/** Set for resource_* events */
resourceId?: number;
/** Human-readable context data included in emails and webhook payloads */
data: Record<string, unknown>;
isTest?: boolean;
};
/**
* Sends a single webhook POST for an alert event.
*
@@ -40,14 +55,14 @@ const RETRY_BASE_DELAY_MS = 500;
export async function sendAlertWebhook(
url: string,
webhookConfig: WebhookAlertConfig,
context: AlertContext
context: WebhookAlertContext
): Promise<void> {
const eventType = context.eventType;
const timestamp = new Date().toISOString();
const status = deriveStatus(eventType, context.data);
const data = { orgId: context.orgId, ...context.data };
let body: string;
let body: Record<string, any>;
if (webhookConfig.useBodyTemplate && webhookConfig.bodyTemplate?.trim()) {
body = renderTemplate(webhookConfig.bodyTemplate, {
event: eventType,
@@ -56,7 +71,11 @@ export async function sendAlertWebhook(
data
});
} else {
body = JSON.stringify({ event: eventType, timestamp, status, data });
body = { event: eventType, timestamp, status, data };
}
if (body.data && context.isTest) {
body.data.test = true;
}
const headers = buildHeaders(webhookConfig);
@@ -75,7 +94,7 @@ export async function sendAlertWebhook(
response = await fetch(url, {
method: webhookConfig.method ?? "POST",
headers,
body,
body: JSON.stringify(body),
signal: controller.signal
});
} catch (err: unknown) {
@@ -247,7 +266,10 @@ interface TemplateContext {
* left untouched.
* 3. The fixed top-level keys: event, timestamp, status.
*/
function renderTemplate(template: string, ctx: TemplateContext): string {
function renderTemplate(
template: string,
ctx: TemplateContext
): Record<string, any> {
// Step 1 expand {{data}} first so its contents are already serialised
// and won't be touched by later passes.
let rendered = template.replace(/\{\{data\}\}/g, JSON.stringify(ctx.data));
@@ -280,20 +302,19 @@ function renderTemplate(template: string, ctx: TemplateContext): string {
// Validate the rendered result is valid JSON; if not, log a warning and
// fall back to the default payload so the webhook still fires.
try {
JSON.parse(rendered);
return rendered;
return JSON.parse(rendered);
} catch {
logger.warn(
`sendAlertWebhook: body template produced invalid JSON for event ` +
`"${ctx.event}" destined for a webhook. Falling back to default ` +
`payload. Check that {{data}} is NOT wrapped in quotes in your template.`
);
return JSON.stringify({
return {
event: ctx.event,
timestamp: ctx.timestamp,
status: ctx.status,
data: ctx.data
});
};
}
}