Merge pull request #3469 from fosrl/refactor/batch-status-requests

refactor: batch status histories
This commit is contained in:
Milo Schwartz
2026-07-29 17:27:11 -04:00
committed by GitHub
29 changed files with 1122 additions and 215 deletions
+1 -1
View File
@@ -18,7 +18,7 @@ export const domains = pgTable("domains", {
domainId: varchar("domainId").primaryKey(),
baseDomain: varchar("baseDomain").notNull(),
configManaged: boolean("configManaged").notNull().default(false),
type: varchar("type"), // "ns", "cname", "wildcard"
type: varchar("type").$type<"ns" | "cname" | "wildcard">(),
verified: boolean("verified").notNull().default(false),
failed: boolean("failed").notNull().default(false),
tries: integer("tries").notNull().default(0),
+1 -1
View File
@@ -15,7 +15,7 @@ export const domains = sqliteTable("domains", {
configManaged: integer("configManaged", { mode: "boolean" })
.notNull()
.default(false),
type: text("type"), // "ns", "cname", "wildcard"
type: text("type").$type<"ns" | "cname" | "wildcard">(),
verified: integer("verified", { mode: "boolean" }).notNull().default(false),
failed: integer("failed", { mode: "boolean" }).notNull().default(false),
tries: integer("tries").notNull().default(0),
+134 -5
View File
@@ -1,6 +1,6 @@
import { z } from "zod";
import { db, logsDb, statusHistory } from "@server/db";
import { and, eq, gte, lt, asc, desc } from "drizzle-orm";
import { and, eq, gte, lt, asc, desc, inArray, max, sql } from "drizzle-orm";
import { regionalCache as cache } from "#dynamic/lib/cache";
const STATUS_HISTORY_CACHE_TTL = 60; // seconds
@@ -41,6 +41,7 @@ export async function getCachedStatusHistory(
return cached;
}
console.time(`[getCachedStatusHistory/${entityType}=${entityId}]`);
// Anchor to local midnight (UTC when tzOffsetMinutes is 0) so the query
// window aligns with stable calendar days for the requesting client
const todayMidnightSec = localMidnightSec(tzOffsetMinutes);
@@ -76,12 +77,14 @@ export async function getCachedStatusHistory(
const priorStatus = lastKnownEvent?.status ?? null;
console.time(`[computeBuckets/${entityType}=${entityId}]`);
const { buckets, totalDowntime } = computeBuckets(
events,
days,
priorStatus,
tzOffsetMinutes
);
console.timeEnd(`[computeBuckets/${entityType}=${entityId}]`);
const totalWindow = days * 86400;
const overallUptime =
totalWindow > 0
@@ -96,6 +99,7 @@ export async function getCachedStatusHistory(
totalDowntimeSeconds: totalDowntime
};
console.timeEnd(`[getCachedStatusHistory/${entityType}=${entityId}]`);
await cache.set(cacheKey, result, STATUS_HISTORY_CACHE_TTL);
return result;
}
@@ -264,9 +268,7 @@ export function computeBuckets(
// Shift by the client's offset before formatting so the label reflects
// their local calendar date rather than the UTC date of dayStartSec
const dateStr = new Date(
(dayStartSec + tzOffsetMinutes * 60) * 1000
)
const dateStr = new Date((dayStartSec + tzOffsetMinutes * 60) * 1000)
.toISOString()
.slice(0, 10);
@@ -301,6 +303,133 @@ export function computeBuckets(
status
});
}
return { buckets, totalDowntime };
}
export type BatchedStatusHistoryResponse = Record<
string,
StatusHistoryResponse
>;
export async function getBatchedStatusHistory(
entityType: string,
entityIds: number[],
days: number,
tzOffsetMinutes: number = 0
): Promise<BatchedStatusHistoryResponse> {
console.time(
`[getBatchedStatusHistory/${entityType}=(${entityIds.join(" ,")})]`
);
// Anchor to local midnight (UTC when tzOffsetMinutes is 0) so the query
// window aligns with stable calendar days for the requesting client
const todayMidnightSec = localMidnightSec(tzOffsetMinutes);
const startSec = todayMidnightSec - days * 86400;
const events = await logsDb
.select()
.from(statusHistory)
.where(
and(
eq(statusHistory.entityType, entityType),
inArray(statusHistory.entityId, entityIds),
gte(statusHistory.timestamp, startSec)
)
)
.orderBy(asc(statusHistory.timestamp));
// Fetch the last known state before the window so that entities that
// haven't changed status recently still show the correct status rather
// than appearing as "no_data".
/**
* If we used only postgres, we would have used `SELECT DISTINCT ON` to get the
* latest event for each `entityId`,
* but it doesn't work on SQLite, so instead we use a subquery,
* the `ROW_NUMBER() OVER PARTITION` allows to assign a number
* to each row ordered by the timestamp, the number 1 is the first one appearing in
* the specified order, then the next and more, we only want the highest timestamp,
* so we get for `row_number=1`
*/
const lastKnowEventsSub = logsDb
.select({
entityId: statusHistory.entityId,
status: statusHistory.status,
timestamp: statusHistory.timestamp,
row_number:
sql<number>`ROW_NUMBER() OVER (PARTITION BY ${statusHistory.entityId} ORDER BY ${statusHistory.timestamp} DESC)`.as(
"row_number"
)
})
.from(statusHistory)
.where(
and(
eq(statusHistory.entityType, entityType),
inArray(statusHistory.entityId, entityIds),
lt(statusHistory.timestamp, startSec)
)
)
.as("sub");
const lastKnownEvents = await logsDb
.select({
entityId: lastKnowEventsSub.entityId,
status: lastKnowEventsSub.status,
timestamp: lastKnowEventsSub.timestamp
})
.from(lastKnowEventsSub)
.where(eq(lastKnowEventsSub.row_number, 1));
const eventStatusMap: Record<
number,
{
events: typeof events;
lastKnownEvent: (typeof lastKnownEvents)[number] | null;
}
> = {};
for (const entityId of entityIds) {
eventStatusMap[entityId] = {
events: events.filter((ev) => ev.entityId === entityId),
lastKnownEvent:
lastKnownEvents.find((ev) => ev.entityId === entityId) ?? null
};
}
const result: BatchedStatusHistoryResponse = {};
console.time(`[computeBuckets/${entityType}=(${entityIds.join(" ,")})]`);
for (const entityId in eventStatusMap) {
const event = eventStatusMap[Number(entityId)];
const priorStatus = event.lastKnownEvent?.status ?? null;
const { buckets, totalDowntime } = computeBuckets(
event.events,
days,
priorStatus,
tzOffsetMinutes
);
const totalWindow = days * 86400;
const overallUptime =
totalWindow > 0
? Math.max(
0,
((totalWindow - totalDowntime) / totalWindow) * 100
)
: 100;
result[entityId] = {
entityType,
entityId: Number(entityId),
days: buckets,
overallUptimePercent: Math.round(overallUptime * 100) / 100,
totalDowntimeSeconds: totalDowntime
};
}
console.timeEnd(`[computeBuckets/${entityType}=(${entityIds.join(" ,")})]`);
console.timeEnd(
`[getBatchedStatusHistory/${entityType}=(${entityIds.join(" ,")})]`
);
return result;
}
@@ -0,0 +1,190 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
import { certificates, db, domains, orgDomains } from "@server/db";
import response from "@server/lib/response";
import logger from "@server/logger";
import { type GetBatchedCertificateResponse } from "@server/routers/certificates/types";
import HttpCode from "@server/types/HttpCode";
import { and, eq, inArray, or } from "drizzle-orm";
import { NextFunction, Request, Response } from "express";
import createHttpError from "http-errors";
import { z } from "zod";
import { fromError } from "zod-validation-error";
const getCertificateParamSchema = z.strictObject({
orgId: z.string()
});
const getCertificateQuerySchema = z.object({
domains: z.preprocess(
(val) => {
if (val === undefined || val === null || val === "") {
return undefined;
}
if (Array.isArray(val)) {
return val;
}
// the array is returned as this
if (typeof val === "string") {
return val.split(",");
}
return undefined;
},
z.array(z.string().min(1).max(255))
)
});
async function query(orgId: string, domainList: string[]) {
// Try to get CNAME certificates first
let existingCertificates = await db
.select({
certId: certificates.certId,
domain: certificates.domain,
wildcard: certificates.wildcard,
status: certificates.status,
expiresAt: certificates.expiresAt,
lastRenewalAttempt: certificates.lastRenewalAttempt,
createdAt: certificates.createdAt,
updatedAt: certificates.updatedAt,
errorMessage: certificates.errorMessage,
renewalCount: certificates.renewalCount,
domainId: domains.domainId,
domainType: domains.type
})
.from(certificates)
.innerJoin(domains, eq(certificates.domainId, domains.domainId))
.innerJoin(
orgDomains,
and(
eq(domains.domainId, orgDomains.domainId),
eq(orgDomains.orgId, orgId)
)
)
.where(and(inArray(certificates.domain, domainList)));
// All non resolved domain certificates might be `ns` or `wildcard`,
// which means exact domain certificates do not
const nonAvailableCertificates = existingCertificates
.filter((cert) => !domainList.includes(cert.domain))
.map((cert) => cert.domain);
if (nonAvailableCertificates.length > 0) {
const domainLevelDownSet = new Set<string>();
const wildcardDomainSet = new Set<string>();
for (const domain of nonAvailableCertificates) {
const domainLevelDown = domain.split(".").slice(1).join(".");
const wildcardPrefixed = `*.${domainLevelDown}`;
domainLevelDownSet.add(domainLevelDown);
wildcardDomainSet.add(wildcardPrefixed);
}
// Need to map the certificates to each domain
const wildcardCertificates = await db
.select({
certId: certificates.certId,
domain: certificates.domain,
wildcard: certificates.wildcard,
status: certificates.status,
expiresAt: certificates.expiresAt,
lastRenewalAttempt: certificates.lastRenewalAttempt,
createdAt: certificates.createdAt,
updatedAt: certificates.updatedAt,
errorMessage: certificates.errorMessage,
renewalCount: certificates.renewalCount,
domainId: domains.domainId,
domainType: domains.type
})
.from(certificates)
.innerJoin(domains, eq(certificates.domainId, domains.domainId))
.innerJoin(
orgDomains,
and(
eq(domains.domainId, orgDomains.domainId),
eq(orgDomains.orgId, orgId)
)
)
.where(
and(
eq(certificates.wildcard, true),
or(
inArray(certificates.domain, [...domainLevelDownSet]),
inArray(certificates.domain, [...wildcardDomainSet])
)
)
);
existingCertificates.push(...wildcardCertificates);
}
const certificateMap: Record<string, any> = {};
for (const domain of domainList) {
const domainLevelDown = domain.split(".").slice(1).join(".");
const wildcardPrefixed = `*.${domainLevelDown}`;
certificateMap[domain] =
existingCertificates.find(
(cert) =>
cert.domain === domain ||
cert.domain === domainLevelDown ||
cert.domain === wildcardPrefixed
) ?? null;
}
return certificateMap;
}
export async function getBatchedCertificates(
req: Request,
res: Response,
next: NextFunction
): Promise<any> {
try {
const parsedParams = getCertificateParamSchema.safeParse(req.params);
if (!parsedParams.success) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
fromError(parsedParams.error).toString()
)
);
}
const { orgId } = parsedParams.data;
const parsedQuery = getCertificateQuerySchema.safeParse(req.query);
if (!parsedQuery.success) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
fromError(parsedQuery.error).toString()
)
);
}
const { domains } = parsedQuery.data;
const cert = await query(orgId, domains);
return response<GetBatchedCertificateResponse>(res, {
data: cert,
success: true,
error: false,
message: "Certificates retrieved successfully",
status: HttpCode.OK
});
} catch (error) {
logger.error(error);
return next(
createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
);
}
}
@@ -14,3 +14,4 @@
export * from "./getCertificate";
export * from "./restartCertificate";
export * from "./syncCertToNewts";
export * from "./getBatchedCertificates";
@@ -11,18 +11,16 @@
* This file is not licensed under the AGPLv3.
*/
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { certificates, db } from "@server/db";
import { sites } from "@server/db";
import { eq, and } from "drizzle-orm";
import response from "@server/lib/response";
import HttpCode from "@server/types/HttpCode";
import createHttpError from "http-errors";
import logger from "@server/logger";
import stoi from "@server/lib/stoi";
import { registry } from "@server/openApi";
import HttpCode from "@server/types/HttpCode";
import { eq } from "drizzle-orm";
import { NextFunction, Request, Response } from "express";
import createHttpError from "http-errors";
import { z } from "zod";
import { fromError } from "zod-validation-error";
import { OpenAPITags, registry } from "@server/openApi";
const restartCertificateParamsSchema = z.strictObject({
certId: z.coerce.number().int().positive(),
+15
View File
@@ -175,6 +175,13 @@ authenticated.get(
certificates.getCertificate
);
authenticated.get(
"/org/:orgId/batched-certificates",
verifyOrgAccess,
verifyUserHasAction(ActionsEnum.getCertificate),
certificates.getBatchedCertificates
);
authenticated.post(
"/org/:orgId/certificate/:certId/restart",
verifyValidLicense,
@@ -853,6 +860,14 @@ authenticated.get(
healthChecks.getHealthCheckStatusHistory
);
authenticated.get(
"/org/:orgId/health-check-status-histories",
verifyValidLicense,
verifyOrgAccess,
verifyUserHasAction(ActionsEnum.getTarget),
healthChecks.getBatchedHealthCheckStatusHistory
);
authenticated.get(
"/client/:clientId/verify-associations-cache",
verifyClientAccess,
@@ -0,0 +1,96 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
import response from "@server/lib/response";
import {
getBatchedStatusHistory,
type BatchedStatusHistoryResponse
} from "@server/lib/statusHistory";
import logger from "@server/logger";
import HttpCode from "@server/types/HttpCode";
import { NextFunction, Request, Response } from "express";
import createHttpError from "http-errors";
import { z } from "zod";
import { fromError } from "zod-validation-error";
const healthCheckIdParamsSchema = z.object({
days: z
.string()
.optional()
.transform((v) => (v ? parseInt(v, 10) : 90)),
// Minutes to add to UTC to get the requesting client's local time
// (e.g. Australia/Sydney standard time is 600). Optional and
// defaults to 0 (UTC) so older clients keep the prior behavior.
tzOffsetMinutes: z
.string()
.optional()
.transform((v) => (v ? parseInt(v, 10) : 0)),
healthCheckIds: z
.preprocess((val) => {
if (val === undefined || val === null || val === "") {
return undefined;
}
const raw = Array.isArray(val) ? val : [val];
const nums = raw
.map((v) =>
typeof v === "string" ? parseInt(v, 10) : Number(v)
)
.filter((n) => Number.isInteger(n) && n > 0);
const unique = [...new Set(nums)];
return unique.length ? unique : undefined;
}, z.array(z.number().int().positive()))
.openapi({
description: "Filter by healthCheckIds (repeat query param)"
})
});
export async function getBatchedHealthCheckStatusHistory(
req: Request,
res: Response,
next: NextFunction
): Promise<any> {
try {
const parsedQuery = healthCheckIdParamsSchema.safeParse(req.query);
if (!parsedQuery.success) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
fromError(parsedQuery.error).toString()
)
);
}
const entityType = "health_check";
const { days, healthCheckIds, tzOffsetMinutes } = parsedQuery.data;
const data = await getBatchedStatusHistory(
entityType,
healthCheckIds,
days,
tzOffsetMinutes
);
return response<BatchedStatusHistoryResponse>(res, {
data,
success: true,
error: false,
message: "Status history retrieved successfully",
status: HttpCode.OK
});
} catch (error) {
logger.error(error);
return next(
createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
);
}
}
@@ -16,3 +16,4 @@ export * from "./createHealthCheck";
export * from "./updateHealthCheck";
export * from "./deleteHealthCheck";
export * from "./getStatusHistory";
export * from "./getBatchedStatusHistory";
+9 -2
View File
@@ -1,9 +1,11 @@
import type { Domain } from "@server/db";
export type GetCertificateResponse = {
certId: number;
domain: string;
domainId: string;
wildcard: boolean;
domainType: string;
domainType: Domain["type"];
status: string; // pending, requested, valid, expired, failed
expiresAt: string | null;
lastRenewalAttempt: Date | null;
@@ -11,4 +13,9 @@ export type GetCertificateResponse = {
updatedAt: number;
errorMessage?: string | null;
renewalCount: number;
};
};
export type GetBatchedCertificateResponse = Record<
string,
GetCertificateResponse | null
>;
+14
View File
@@ -319,6 +319,13 @@ authenticated.get(
site.getSiteStatusHistory
);
authenticated.get(
"/org/:orgId/site-status-histories",
verifyOrgAccess,
verifyUserHasAction(ActionsEnum.listSites),
site.getBatchedSiteStatusHistory
);
// Site Resource endpoints
authenticated.put(
"/org/:orgId/site-resource",
@@ -469,6 +476,13 @@ authenticated.get(
resource.getResourceStatusHistory
);
authenticated.get(
"/org/:orgId/resource-status-histories",
verifyOrgAccess,
verifyUserHasAction(ActionsEnum.listResources),
resource.getBatchedResourceStatusHistory
);
authenticated.get(
"/org/:orgId/resources",
verifyOrgAccess,
@@ -0,0 +1,83 @@
import response from "@server/lib/response";
import {
getBatchedStatusHistory,
type BatchedStatusHistoryResponse
} from "@server/lib/statusHistory";
import logger from "@server/logger";
import HttpCode from "@server/types/HttpCode";
import { NextFunction, Request, Response } from "express";
import createHttpError from "http-errors";
import { z } from "zod";
import { fromError } from "zod-validation-error";
const resourceIdParamsSchema = z.object({
days: z
.string()
.optional()
.transform((v) => (v ? parseInt(v, 10) : 90)),
// Minutes to add to UTC to get the requesting client's local time
// (e.g. Australia/Sydney standard time is 600). Optional and
// defaults to 0 (UTC) so older clients keep the prior behavior.
tzOffsetMinutes: z
.string()
.optional()
.transform((v) => (v ? parseInt(v, 10) : 0)),
resourceIds: z
.preprocess((val) => {
if (val === undefined || val === null || val === "") {
return undefined;
}
const raw = Array.isArray(val) ? val : [val];
const nums = raw
.map((v) =>
typeof v === "string" ? parseInt(v, 10) : Number(v)
)
.filter((n) => Number.isInteger(n) && n > 0);
const unique = [...new Set(nums)];
return unique.length ? unique : undefined;
}, z.array(z.number().int().positive()))
.openapi({
description: "Filter by resourceIds (repeat query param)"
})
});
export async function getBatchedResourceStatusHistory(
req: Request,
res: Response,
next: NextFunction
): Promise<any> {
try {
const parsedQuery = resourceIdParamsSchema.safeParse(req.query);
if (!parsedQuery.success) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
fromError(parsedQuery.error).toString()
)
);
}
const entityType = "resource";
const { days, resourceIds, tzOffsetMinutes } = parsedQuery.data;
const data = await getBatchedStatusHistory(
entityType,
resourceIds,
days,
tzOffsetMinutes
);
return response<BatchedStatusHistoryResponse>(res, {
data,
success: true,
error: false,
message: "Status history retrieved successfully",
status: HttpCode.OK
});
} catch (error) {
logger.error(error);
return next(
createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
);
}
}
+1
View File
@@ -33,4 +33,5 @@ export * from "./removeUserFromResource";
export * from "./listAllResourceNames";
export * from "./removeEmailFromResourceWhitelist";
export * from "./getStatusHistory";
export * from "./getBatchedStatusHistory";
export * from "./getResourcePolicies";
@@ -0,0 +1,83 @@
import response from "@server/lib/response";
import {
getBatchedStatusHistory,
type BatchedStatusHistoryResponse
} from "@server/lib/statusHistory";
import logger from "@server/logger";
import HttpCode from "@server/types/HttpCode";
import { NextFunction, Request, Response } from "express";
import createHttpError from "http-errors";
import { z } from "zod";
import { fromError } from "zod-validation-error";
const siteIdParamsSchema = z.object({
days: z
.string()
.optional()
.transform((v) => (v ? parseInt(v, 10) : 90)),
// Minutes to add to UTC to get the requesting client's local time
// (e.g. Australia/Sydney standard time is 600). Optional and
// defaults to 0 (UTC) so older clients keep the prior behavior.
tzOffsetMinutes: z
.string()
.optional()
.transform((v) => (v ? parseInt(v, 10) : 0)),
siteIds: z
.preprocess((val) => {
if (val === undefined || val === null || val === "") {
return undefined;
}
const raw = Array.isArray(val) ? val : [val];
const nums = raw
.map((v) =>
typeof v === "string" ? parseInt(v, 10) : Number(v)
)
.filter((n) => Number.isInteger(n) && n > 0);
const unique = [...new Set(nums)];
return unique.length ? unique : undefined;
}, z.array(z.number().int().positive()))
.openapi({
description: "Filter by siteIds (repeat query param)"
})
});
export async function getBatchedSiteStatusHistory(
req: Request,
res: Response,
next: NextFunction
): Promise<any> {
try {
const parsedQuery = siteIdParamsSchema.safeParse(req.query);
if (!parsedQuery.success) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
fromError(parsedQuery.error).toString()
)
);
}
const entityType = "site";
const { days, siteIds, tzOffsetMinutes } = parsedQuery.data;
const data = await getBatchedStatusHistory(
entityType,
siteIds,
days,
tzOffsetMinutes
);
return response<BatchedStatusHistoryResponse>(res, {
data,
success: true,
error: false,
message: "Status history retrieved successfully",
status: HttpCode.OK
});
} catch (error) {
logger.error(error);
return next(
createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
);
}
}
+1
View File
@@ -1,5 +1,6 @@
export * from "./getSite";
export * from "./getStatusHistory";
export * from "./getBatchedStatusHistory";
export * from "./createSite";
export * from "./deleteSite";
export * from "./updateSite";