calvin.erfmann/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-06-03 22:35:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add resource degraded

Browse Source
This commit is contained in:
Owen
2026-04-24 17:47:08 -07:00
parent 8e16ff07a9
commit 82212af643
16 changed files with 162 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
messages/en-US.json
View File

@@ -1432,6 +1432,7 @@
"alertingTriggerHcToggle": "Health check status changes",
"alertingTriggerResourceHealthy": "Resource healthy",
"alertingTriggerResourceUnhealthy": "Resource unhealthy",
"alertingTriggerResourceDegraded": "Resource degraded",
"alertingSearchHealthChecks": "Search health checks…",
"alertingHealthChecksEmpty": "No health checks available.",
"alertingTriggerResourceToggle": "Resource status changes",

1
server/db/pg/schema/privateSchema.ts
View File

@@ -484,6 +484,7 @@ export const alertRules = pgTable("alertRules", {
| "health_check_toggle"
| "resource_healthy"
| "resource_unhealthy"
| "resource_degraded"
| "resource_toggle"
>()
.notNull(),

43
server/db/sqlite/schema/privateSchema.ts
View File

@@ -425,10 +425,18 @@ export const eventStreamingDestinations = sqliteTable(
orgId: text("orgId")
.notNull()
.references(() => orgs.orgId, { onDelete: "cascade" }),
sendConnectionLogs: integer("sendConnectionLogs", { mode: "boolean" }).notNull().default(false),
sendRequestLogs: integer("sendRequestLogs", { mode: "boolean" }).notNull().default(false),
sendActionLogs: integer("sendActionLogs", { mode: "boolean" }).notNull().default(false),
sendAccessLogs: integer("sendAccessLogs", { mode: "boolean" }).notNull().default(false),
sendConnectionLogs: integer("sendConnectionLogs", { mode: "boolean" })
.notNull()
.default(false),
sendRequestLogs: integer("sendRequestLogs", { mode: "boolean" })
.notNull()
.default(false),
sendActionLogs: integer("sendActionLogs", { mode: "boolean" })
.notNull()
.default(false),
sendAccessLogs: integer("sendAccessLogs", { mode: "boolean" })
.notNull()
.default(false),
type: text("type").notNull(), // e.g. "http", "kafka", etc.
config: text("config").notNull(), // JSON string with the configuration for the destination
enabled: integer("enabled", { mode: "boolean" })
@@ -476,14 +484,19 @@ export const alertRules = sqliteTable("alertRules", {
| "health_check_toggle"
| "resource_healthy"
| "resource_unhealthy"
| "resource_degraded"
| "resource_toggle"
>()
.notNull(),
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
cooldownSeconds: integer("cooldownSeconds").notNull().default(300),
allSites: integer("allSites", { mode: "boolean" }).notNull().default(false),
allHealthChecks: integer("allHealthChecks", { mode: "boolean" }).notNull().default(false),
allResources: integer("allResources", { mode: "boolean" }).notNull().default(false),
allHealthChecks: integer("allHealthChecks", { mode: "boolean" })
.notNull()
.default(false),
allResources: integer("allResources", { mode: "boolean" })
.notNull()
.default(false),
lastTriggeredAt: integer("lastTriggeredAt"),
createdAt: integer("createdAt").notNull(),
updatedAt: integer("updatedAt").notNull()
@@ -531,19 +544,27 @@ export const alertEmailRecipients = sqliteTable("alertEmailRecipients", {
recipientId: integer("recipientId").primaryKey({ autoIncrement: true }),
emailActionId: integer("emailActionId")
.notNull()
.references(() => alertEmailActions.emailActionId, { onDelete: "cascade" }),
userId: text("userId").references(() => users.userId, { onDelete: "cascade" }),
roleId: integer("roleId").references(() => roles.roleId, { onDelete: "cascade" }),
.references(() => alertEmailActions.emailActionId, {
onDelete: "cascade"
}),
userId: text("userId").references(() => users.userId, {
onDelete: "cascade"
}),
roleId: integer("roleId").references(() => roles.roleId, {
onDelete: "cascade"
}),
email: text("email")
});
export const alertWebhookActions = sqliteTable("alertWebhookActions", {
webhookActionId: integer("webhookActionId").primaryKey({ autoIncrement: true }),
webhookActionId: integer("webhookActionId").primaryKey({
autoIncrement: true
}),
alertRuleId: integer("alertRuleId")
.notNull()
.references(() => alertRules.alertRuleId, { onDelete: "cascade" }),
webhookUrl: text("webhookUrl").notNull(),
config: text("config"), // encrypted JSON with auth config (authType, credentials)
config: text("config"), // encrypted JSON with auth config (authType, credentials)
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
lastSentAt: integer("lastSentAt")
});

15
server/emails/templates/AlertNotification.tsx
View File

@@ -23,6 +23,7 @@ export type AlertEventType =
| "health_check_toggle"
| "resource_healthy"
| "resource_unhealthy"
| "resource_degraded"
| "resource_toggle";
export type AlertNotificationProps = {
@@ -114,6 +115,15 @@ function getEventMeta(eventType: AlertEventType): {
statusLabel: "Unhealthy",
statusColor: "#dc2626"
};
case "resource_degraded":
return {
heading: "Resource Unhealthy",
previewText: "A resource in your organization is not healthy.",
summary:
"A resource in your organization is currently unhealthy.",
statusLabel: "Unhealthy",
statusColor: "#dc2626"
};
case "resource_toggle":
return {
heading: "Resource Status Changed",
@@ -135,7 +145,10 @@ function getEventMeta(eventType: AlertEventType): {
}
}
function resolveToggleStatus(status: unknown): { label: string; color: string } {
function resolveToggleStatus(status: unknown): {
label: string;
color: string;
} {
switch (String(status).toLowerCase()) {
case "online":
return { label: "Online", color: "#16a34a" };

9
server/private/lib/alerts/events/healthCheckEvents.ts
View File

@@ -23,6 +23,7 @@ import {
} from "@server/db";
import { eq } from "drizzle-orm";
import {
fireResourceDegradedAlert,
fireResourceHealthyAlert,
fireResourceUnhealthyAlert
} from "./resourceEvents";
@@ -217,6 +218,14 @@ async function handleResource(orgId: string, healthCheckTargetId?: number | null
undefined,
trx
);
} else if (health === "degraded") {
await fireResourceDegradedAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
trx
);
}
}
}

29
server/private/lib/alerts/events/resourceEvents.ts
View File

@@ -130,9 +130,9 @@ export async function fireResourceUnhealthyAlert(
}
/**
* Fire a `resource_toggle` alert for the given resource.
* Fire a `resource_degraded` alert for the given resource.
*
* Call this when a resource's enabled/disabled status is toggled so that any
* Call this after a resource has been detected as degraded so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the resource.
@@ -140,7 +140,7 @@ export async function fireResourceUnhealthyAlert(
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceToggleAlert(
export async function fireResourceDegradedAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
@@ -148,8 +148,16 @@ export async function fireResourceToggleAlert(
trx: Transaction | typeof db = db
): Promise<void> {
try {
await trx.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "degraded",
timestamp: Math.floor(Date.now() / 1000)
});
await processAlerts({
eventType: "resource_toggle",
eventType: "resource_degraded",
orgId,
resourceId,
data: {
@@ -157,9 +165,20 @@ export async function fireResourceToggleAlert(
...extra
}
});
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "degraded",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceToggleAlert: unexpected error for resourceId ${resourceId}`,
`fireResourceDegradedAlert: unexpected error for resourceId ${resourceId}`,
err
);
}

2
server/private/lib/alerts/sendAlertEmail.ts
View File

@@ -88,6 +88,8 @@ function buildSubject(context: AlertContext): string {
return "[Alert] Resource Healthy";
case "resource_unhealthy":
return "[Alert] Resource Unhealthy";
case "resource_degraded":
return "[Alert] Resource Degraded";
case "resource_toggle":
return "[Alert] Resource Status Changed";
default: {

31
server/private/lib/alerts/sendAlertWebhook.ts
View File

@@ -12,7 +12,10 @@
*/
import logger from "@server/logger";
import { AlertContext, WebhookAlertConfig } from "@server/routers/alertRule/types";
import {
AlertContext,
WebhookAlertConfig
} from "@server/routers/alertRule/types";
const REQUEST_TIMEOUT_MS = 15_000;
const MAX_RETRIES = 3;
@@ -56,7 +59,10 @@ export async function sendAlertWebhook(
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
const controller = new AbortController();
const timeoutHandle = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
const timeoutHandle = setTimeout(
() => controller.abort(),
REQUEST_TIMEOUT_MS
);
let response: Response;
try {
@@ -75,7 +81,9 @@ export async function sendAlertWebhook(
);
} else {
const msg = err instanceof Error ? err.message : String(err);
lastError = new Error(`Alert webhook: request to "${url}" failed – ${msg}`);
lastError = new Error(
`Alert webhook: request to "${url}" failed – ${msg}`
);
}
if (attempt < MAX_RETRIES) {
const delay = RETRY_BASE_DELAY_MS * 2 ** (attempt - 1);
@@ -111,11 +119,18 @@ export async function sendAlertWebhook(
continue;
}
logger.debug(`Alert webhook sent successfully to "${url}" for event "${context.eventType}" (attempt ${attempt}/${MAX_RETRIES})`);
logger.debug(
`Alert webhook sent successfully to "${url}" for event "${context.eventType}" (attempt ${attempt}/${MAX_RETRIES})`
);
return;
}
throw lastError ?? new Error(`Alert webhook: all ${MAX_RETRIES} attempts failed for "${url}"`);
throw (
lastError ??
new Error(
`Alert webhook: all ${MAX_RETRIES} attempts failed for "${url}"`
)
);
}
// ---------------------------------------------------------------------------
@@ -139,6 +154,8 @@ function deriveStatus(
case "health_check_unhealthy":
case "resource_unhealthy":
return "unhealthy";
case "resource_degraded":
return "degraded";
case "health_check_toggle":
case "resource_toggle":
return String(data.status ?? "unknown");
@@ -154,7 +171,9 @@ function deriveStatus(
// Header construction (mirrors HttpLogDestination.buildHeaders)
// ---------------------------------------------------------------------------
function buildHeaders(webhookConfig: WebhookAlertConfig): Record<string, string> {
function buildHeaders(
webhookConfig: WebhookAlertConfig
): Record<string, string> {
const headers: Record<string, string> = {
"Content-Type": "application/json"
};

14
server/private/routers/alertEvents/triggerResourceAlert.ts
View File

@@ -24,7 +24,8 @@ import { eq, and } from "drizzle-orm";
import {
fireResourceHealthyAlert,
fireResourceUnhealthyAlert,
fireResourceToggleAlert
fireResourceToggleAlert,
fireResourceDegradedAlert
} from "#private/lib/alerts/events/resourceEvents";
const paramsSchema = z.strictObject({
@@ -33,7 +34,12 @@ const paramsSchema = z.strictObject({
});
const bodySchema = z.strictObject({
eventType: z.enum(["resource_healthy", "resource_unhealthy", "resource_toggle"])
eventType: z.enum([
"resource_healthy",
"resource_unhealthy",
"resource_degraded",
"resource_toggle"
])
});
export type TriggerResourceAlertResponse = {
@@ -101,8 +107,8 @@ export async function triggerResourceAlert(
resourceId,
resource.name ?? undefined
);
} else {
await fireResourceToggleAlert(
} else if (eventType === "resource_degraded") {
await fireResourceDegradedAlert(
orgId,
resourceId,
resource.name ?? undefined

41
server/private/routers/alertRule/createAlertRule.ts
View File

@@ -33,7 +33,11 @@ import { encrypt } from "@server/lib/crypto";
import config from "@server/lib/config";
import { CreateAlertRuleResponse } from "@server/routers/alertRule/types";
export const SITE_EVENT_TYPES = ["site_online", "site_offline", "site_toggle"] as const;
export const SITE_EVENT_TYPES = [
"site_online",
"site_offline",
"site_toggle"
] as const;
export const HC_EVENT_TYPES = [
"health_check_healthy",
"health_check_unhealthy",
@@ -42,6 +46,7 @@ export const HC_EVENT_TYPES = [
export const RESOURCE_EVENT_TYPES = [
"resource_healthy",
"resource_unhealthy",
"resource_degraded",
"resource_toggle"
] as const;
@@ -92,19 +97,24 @@ const bodySchema = z
const isHcEvent = (HC_EVENT_TYPES as readonly string[]).includes(
val.eventType
);
const isResourceEvent = (RESOURCE_EVENT_TYPES as readonly string[]).includes(
val.eventType
);
const isResourceEvent = (
RESOURCE_EVENT_TYPES as readonly string[]
).includes(val.eventType);
if (isSiteEvent && !val.allSites && val.siteIds.length === 0) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "At least one siteId is required for site event types when allSites is false",
message:
"At least one siteId is required for site event types when allSites is false",
path: ["siteIds"]
});
}
if (isHcEvent && !val.allHealthChecks && val.healthCheckIds.length === 0) {
if (
isHcEvent &&
!val.allHealthChecks &&
val.healthCheckIds.length === 0
) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message:
@@ -129,10 +139,15 @@ const bodySchema = z
});
}
if (isResourceEvent && !val.allResources && val.resourceIds.length === 0) {
if (
isResourceEvent &&
!val.allResources &&
val.resourceIds.length === 0
) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "At least one resourceId is required for resource event types when allResources is false",
message:
"At least one resourceId is required for resource event types when allResources is false",
path: ["resourceIds"]
});
}
@@ -148,7 +163,8 @@ const bodySchema = z
if (isResourceEvent && val.healthCheckIds.length > 0) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "healthCheckIds must not be set for resource event types",
message:
"healthCheckIds must not be set for resource event types",
path: ["healthCheckIds"]
});
}
@@ -164,7 +180,8 @@ const bodySchema = z
if (isHcEvent && val.resourceIds.length > 0) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "resourceIds must not be set for health check event types",
message:
"resourceIds must not be set for health check event types",
path: ["resourceIds"]
});
}
@@ -284,9 +301,7 @@ export async function createAlertRule(
// Create the email action pivot row and recipients if any recipients
// were supplied (userIds, roleIds, or raw emails).
const hasRecipients =
userIds.length > 0 ||
roleIds.length > 0 ||
emails.length > 0;
userIds.length > 0 || roleIds.length > 0 || emails.length > 0;
if (hasRecipients) {
const [emailActionRow] = await db

1
server/private/routers/alertRule/listAlertRules.ts
View File

@@ -76,6 +76,7 @@ const SITE_ALERT_EVENT_TYPES = [
const RESOURCE_ALERT_EVENT_TYPES = [
"resource_healthy",
"resource_unhealthy",
"resource_degraded",
"resource_toggle"
] as const;

2
server/routers/alertRule/types.ts
View File

@@ -37,6 +37,7 @@ export type GetAlertRuleResponse = {
| "health_check_toggle"
| "resource_healthy"
| "resource_unhealthy"
| "resource_degraded"
| "resource_toggle";
enabled: boolean;
cooldownSeconds: number;
@@ -94,6 +95,7 @@ export type AlertEventType =
| "health_check_toggle"
| "resource_healthy"
| "resource_unhealthy"
| "resource_degraded"
| "resource_toggle";
// ---------------------------------------------------------------------------

2
src/components/AlertingRulesTable.tsx
View File

@@ -118,6 +118,8 @@ function triggerLabel(rule: AlertRuleRow, t: (k: string) => string) {
return t("alertingTriggerResourceHealthy");
case "resource_unhealthy":
return t("alertingTriggerResourceUnhealthy");
case "resource_degraded":
return t("alertingTriggerResourceDegraded");
case "resource_toggle":
return t("alertingTriggerResourceToggle");
default:

1
src/components/ProxyResourcesTable.tsx
View File

@@ -32,7 +32,6 @@ import { toast } from "@app/hooks/useToast";
import { createApiClient, formatAxiosError } from "@app/lib/api";
import { UpdateResourceResponse } from "@server/routers/resource";
import type { PaginationState } from "@tanstack/react-table";
import { useQuery } from "@tanstack/react-query";
import { AxiosResponse } from "axios";
import {
ArrowDown01Icon,

4
src/components/alert-rule-editor/AlertRuleFields.tsx
View File

@@ -1147,6 +1147,7 @@ export function AlertRuleSourceFields({
if (
curTrigger !== "resource_healthy" &&
curTrigger !== "resource_unhealthy" &&
curTrigger !== "resource_degraded" &&
curTrigger !== "resource_toggle"
) {
setValue("trigger", "resource_toggle", {
@@ -1367,6 +1368,9 @@ export function AlertRuleTriggerFields({
<SelectItem value="resource_unhealthy">
{t("alertingTriggerResourceUnhealthy")}
</SelectItem>
<SelectItem value="resource_degraded">
{t("alertingTriggerResourceDegraded")}
</SelectItem>
</>
) : (
<>

8
src/lib/alertRuleForm.ts
View File

@@ -25,6 +25,7 @@ export type AlertTrigger =
| "health_check_toggle"
| "resource_healthy"
| "resource_unhealthy"
| "resource_degraded"
| "resource_toggle";
export type AlertRuleFormAction =
@@ -77,6 +78,7 @@ export type AlertRuleApiPayload = {
| "health_check_toggle"
| "resource_healthy"
| "resource_unhealthy"
| "resource_degraded"
| "resource_toggle";
enabled: boolean;
allSites: boolean;
@@ -160,6 +162,7 @@ export function buildFormSchema(t: (k: string) => string) {
"health_check_toggle",
"resource_healthy",
"resource_unhealthy",
"resource_degraded",
"resource_toggle"
]),
actions: z.array(
@@ -243,6 +246,7 @@ export function buildFormSchema(t: (k: string) => string) {
const resourceTriggers: AlertTrigger[] = [
"resource_healthy",
"resource_unhealthy",
"resource_degraded",
"resource_toggle"
];
if (
@@ -344,7 +348,9 @@ export function alertRuleAllResourcesSelected(
eventType: string,
resourceIds: number[] | undefined
): boolean {
return eventType.startsWith("resource_") && (resourceIds?.length ?? 0) === 0;
return (
eventType.startsWith("resource_") && (resourceIds?.length ?? 0) === 0
);
}
export function alertRuleAllHealthChecksSelected(