From df4b9de334175c74ae74d5a04383a89cf7994934 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 10 Dec 2025 03:24:32 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A7=20wip:=20export=20limits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- messages/en-US.json | 2 ++ .../routers/auditLogs/exportAccessAuditLog.ts | 13 +++++++++- .../auditLogs/exportRequestAuditLog.ts | 22 +++++++++++++--- .../[orgId]/settings/logs/request/page.tsx | 26 ++++++++++++------- src/components/LogDataTable.tsx | 7 +++++ 5 files changed, 57 insertions(+), 13 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 3dd1c94e..7d5deded 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -2067,6 +2067,8 @@ "timestamp": "Timestamp", "accessLogs": "Access Logs", "exportCsv": "Export CSV", + "exportError": "Unknown error when exporting CSV", + "exportCsvTooltip": "Within Time Range", "actorId": "Actor ID", "allowedByRule": "Allowed by Rule", "allowedNoAuth": "Allowed No Auth", diff --git a/server/private/routers/auditLogs/exportAccessAuditLog.ts b/server/private/routers/auditLogs/exportAccessAuditLog.ts index fbeca932..80d6296b 100644 --- a/server/private/routers/auditLogs/exportAccessAuditLog.ts +++ b/server/private/routers/auditLogs/exportAccessAuditLog.ts @@ -22,9 +22,11 @@ import logger from "@server/logger"; import { queryAccessAuditLogsParams, queryAccessAuditLogsQuery, - queryAccess + queryAccess, + countAccessQuery } from "./queryAccessAuditLog"; import { generateCSV } from "@server/routers/auditLogs/generateCSV"; +import { MAX_EXPORT_LIMIT } from "@server/routers/auditLogs"; registry.registerPath({ method: "get", @@ -65,6 +67,15 @@ export async function exportAccessAuditLogs( } const data = { ...parsedQuery.data, ...parsedParams.data }; + const [{ count }] = await countAccessQuery(data); + if (count > MAX_EXPORT_LIMIT) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + `Export limit exceeded. Your selection contains ${count} rows, but the maximum is [${MAX_EXPORT_LIMIT}] rows. Please select a shorter time range to reduce the data.` + ) + ); + } const baseQuery = queryAccess(data); diff --git a/server/routers/auditLogs/exportRequestAuditLog.ts b/server/routers/auditLogs/exportRequestAuditLog.ts index 9e55cfc4..ee1e23ac 100644 --- a/server/routers/auditLogs/exportRequestAuditLog.ts +++ b/server/routers/auditLogs/exportRequestAuditLog.ts @@ -9,17 +9,23 @@ import logger from "@server/logger"; import { queryAccessAuditLogsQuery, queryRequestAuditLogsParams, - queryRequest + queryRequest, + countRequestQuery } from "./queryRequestAuditLog"; import { generateCSV } from "./generateCSV"; +export const MAX_EXPORT_LIMIT = 50_000; + registry.registerPath({ method: "get", path: "/org/{orgId}/logs/request", description: "Query the request audit log for an organization", tags: [OpenAPITags.Org], request: { - query: queryAccessAuditLogsQuery, + query: queryAccessAuditLogsQuery.omit({ + limit: true, + offset: true + }), params: queryRequestAuditLogsParams }, responses: {} @@ -53,9 +59,19 @@ export async function exportRequestAuditLogs( const data = { ...parsedQuery.data, ...parsedParams.data }; + const [{ count }] = await countRequestQuery(data); + if (count > MAX_EXPORT_LIMIT) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + `Export limit exceeded. Your selection contains ${count} rows, but the maximum is [${MAX_EXPORT_LIMIT}] rows. Please select a shorter time range to reduce the data.` + ) + ); + } + const baseQuery = queryRequest(data); - const log = await baseQuery.limit(data.limit).offset(data.offset); + const log = await baseQuery.limit(MAX_EXPORT_LIMIT); const csvData = generateCSV(log); diff --git a/src/app/[orgId]/settings/logs/request/page.tsx b/src/app/[orgId]/settings/logs/request/page.tsx index af52ace7..31b4bd72 100644 --- a/src/app/[orgId]/settings/logs/request/page.tsx +++ b/src/app/[orgId]/settings/logs/request/page.tsx @@ -11,14 +11,14 @@ import { Button } from "@app/components/ui/button"; import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import { createApiClient } from "@app/lib/api"; -import { useParams, useRouter, useSearchParams } from "next/navigation"; import { useTranslations } from "next-intl"; import { getSevenDaysAgo } from "@app/lib/getSevenDaysAgo"; import { ColumnDef } from "@tanstack/react-table"; +import axios from "axios"; import { ArrowUpRight, Key, Lock, Unlock, User } from "lucide-react"; import Link from "next/link"; - -import { useEffect, useState } from "react"; +import { useParams, useRouter, useSearchParams } from "next/navigation"; +import { useEffect, useState, useTransition } from "react"; export default function GeneralPage() { const router = useRouter(); @@ -29,7 +29,7 @@ export default function GeneralPage() { const [rows, setRows] = useState([]); const [isRefreshing, setIsRefreshing] = useState(false); - const [isExporting, setIsExporting] = useState(false); + const [isExporting, startTransition] = useTransition(); // Pagination state const [totalCount, setTotalCount] = useState(0); @@ -303,8 +303,6 @@ export default function GeneralPage() { const exportData = async () => { try { - setIsExporting(true); - // Prepare query params for export const params: any = { timeStart: dateRange.startDate?.date @@ -336,11 +334,21 @@ export default function GeneralPage() { document.body.appendChild(link); link.click(); link.parentNode?.removeChild(link); - setIsExporting(false); } catch (error) { + let apiErrorMessage: string | null = null; + if (axios.isAxiosError(error) && error.response) { + const data = error.response.data; + + if (data instanceof Blob && data.type === "application/json") { + // Parse the Blob as JSON + const text = await data.text(); + const errorData = JSON.parse(text); + apiErrorMessage = errorData.message; + } + } toast({ title: t("error"), - description: t("exportError"), + description: apiErrorMessage ?? t("exportError"), variant: "destructive" }); } @@ -774,7 +782,7 @@ export default function GeneralPage() { searchColumn="host" onRefresh={refreshData} isRefreshing={isRefreshing} - onExport={exportData} + onExport={() => startTransition(exportData)} isExporting={isExporting} onDateRangeChange={handleDateRangeChange} dateRange={{ diff --git a/src/components/LogDataTable.tsx b/src/components/LogDataTable.tsx index 492a8c15..7156e8c3 100644 --- a/src/components/LogDataTable.tsx +++ b/src/components/LogDataTable.tsx @@ -42,6 +42,12 @@ import { import { Tabs, TabsList, TabsTrigger } from "@app/components/ui/tabs"; import { useTranslations } from "next-intl"; import { DateRangePicker, DateTimeValue } from "@app/components/DateTimePicker"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger +} from "./ui/tooltip"; const STORAGE_KEYS = { PAGE_SIZE: "datatable-page-size", @@ -51,6 +57,7 @@ const STORAGE_KEYS = { export const getStoredPageSize = ( tableId?: string, + defaultSize = 20 ): number => { if (typeof window === "undefined") return defaultSize;