mirror of
https://github.com/fosrl/pangolin.git
synced 2026-05-26 18:52:41 +00:00
fix(audit-logs): route request audit log reads through logsDb
Route the read paths in queryRequestAuditLog.ts and
queryRequestAnalytics.ts through `logsDb` instead of
`primaryLogsDb`, matching the existing private audit log routes
(queryActionAuditLog, queryAccessAuditLog, queryConnectionAuditLog
all already use `logsDb`). In PostgreSQL deployments configured
with a read replica via `withReplicas` (see server/db/pg/logsDriver.ts),
this keeps high-volume audit log reads off the primary. No-op
in OSS-SQLite where `logsDb === primaryDb`.
Investigated rewriting `queryUniqueFilterAttributes` per the
in-line TODO ("SOMEONE PLEASE OPTIMIZE THIS!!!!!"). A candidate
rewrite using UNION ALL with six GROUP BY...LIMIT 500 arms
benchmarked 48-61% slower than the current SELECT DISTINCT
LIMIT 501 approach on SQLite (100k/300k/1M rows, 20 runs each):
each grouped arm materializes a temp B-tree before applying LIMIT,
while DISTINCT short-circuits via hash dedup with early exit.
A materialized facets table is likely the right long-term fix,
not a query-shape rewrite.
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
import { logsDb, requestAuditLog, driver, primaryLogsDb } from "@server/db";
|
||||
import { logsDb, requestAuditLog, driver } from "@server/db";
|
||||
import { registry } from "@server/openApi";
|
||||
import { NextFunction } from "express";
|
||||
import { Request, Response } from "express";
|
||||
@@ -74,12 +74,12 @@ async function query(query: Q) {
|
||||
);
|
||||
}
|
||||
|
||||
const [all] = await primaryLogsDb
|
||||
const [all] = await logsDb
|
||||
.select({ total: count() })
|
||||
.from(requestAuditLog)
|
||||
.where(baseConditions);
|
||||
|
||||
const [blocked] = await primaryLogsDb
|
||||
const [blocked] = await logsDb
|
||||
.select({ total: count() })
|
||||
.from(requestAuditLog)
|
||||
.where(and(baseConditions, eq(requestAuditLog.action, false)));
|
||||
@@ -90,7 +90,7 @@ async function query(query: Q) {
|
||||
|
||||
const DISTINCT_LIMIT = 500;
|
||||
|
||||
const requestsPerCountry = await primaryLogsDb
|
||||
const requestsPerCountry = await logsDb
|
||||
.selectDistinct({
|
||||
code: requestAuditLog.location,
|
||||
count: totalQ
|
||||
@@ -118,7 +118,7 @@ async function query(query: Q) {
|
||||
const booleanTrue = driver === "pg" ? sql`true` : sql`1`;
|
||||
const booleanFalse = driver === "pg" ? sql`false` : sql`0`;
|
||||
|
||||
const requestsPerDay = await primaryLogsDb
|
||||
const requestsPerDay = await logsDb
|
||||
.select({
|
||||
day: groupByDayFunction.as("day"),
|
||||
allowedCount:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { logsDb, primaryLogsDb, requestAuditLog, resources, siteResources, db, primaryDb } from "@server/db";
|
||||
import { logsDb, requestAuditLog, resources, siteResources, db, primaryDb } from "@server/db";
|
||||
import { registry } from "@server/openApi";
|
||||
import { NextFunction } from "express";
|
||||
import { Request, Response } from "express";
|
||||
@@ -110,7 +110,7 @@ function getWhere(data: Q) {
|
||||
}
|
||||
|
||||
export function queryRequest(data: Q) {
|
||||
return primaryLogsDb
|
||||
return logsDb
|
||||
.select({
|
||||
id: requestAuditLog.id,
|
||||
timestamp: requestAuditLog.timestamp,
|
||||
@@ -211,7 +211,7 @@ async function enrichWithResourceDetails(logs: Awaited<ReturnType<typeof queryRe
|
||||
}
|
||||
|
||||
export function countRequestQuery(data: Q) {
|
||||
const countQuery = primaryLogsDb
|
||||
const countQuery = logsDb
|
||||
.select({ count: count() })
|
||||
.from(requestAuditLog)
|
||||
.where(getWhere(data));
|
||||
@@ -254,34 +254,34 @@ async function queryUniqueFilterAttributes(
|
||||
uniqueResources,
|
||||
uniqueSiteResources
|
||||
] = await Promise.all([
|
||||
primaryLogsDb
|
||||
logsDb
|
||||
.selectDistinct({ actor: requestAuditLog.actor })
|
||||
.from(requestAuditLog)
|
||||
.where(baseConditions)
|
||||
.limit(DISTINCT_LIMIT + 1),
|
||||
primaryLogsDb
|
||||
logsDb
|
||||
.selectDistinct({ locations: requestAuditLog.location })
|
||||
.from(requestAuditLog)
|
||||
.where(baseConditions)
|
||||
.limit(DISTINCT_LIMIT + 1),
|
||||
primaryLogsDb
|
||||
logsDb
|
||||
.selectDistinct({ hosts: requestAuditLog.host })
|
||||
.from(requestAuditLog)
|
||||
.where(baseConditions)
|
||||
.limit(DISTINCT_LIMIT + 1),
|
||||
primaryLogsDb
|
||||
logsDb
|
||||
.selectDistinct({ paths: requestAuditLog.path })
|
||||
.from(requestAuditLog)
|
||||
.where(baseConditions)
|
||||
.limit(DISTINCT_LIMIT + 1),
|
||||
primaryLogsDb
|
||||
logsDb
|
||||
.selectDistinct({
|
||||
id: requestAuditLog.resourceId
|
||||
})
|
||||
.from(requestAuditLog)
|
||||
.where(baseConditions)
|
||||
.limit(DISTINCT_LIMIT + 1),
|
||||
primaryLogsDb
|
||||
logsDb
|
||||
.selectDistinct({
|
||||
id: requestAuditLog.siteResourceId
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user