🚧 wip: export limits

messages/en-US.json

@@ -2067,6 +2067,8 @@
     "timestamp": "Timestamp",
     "accessLogs": "Access Logs",
     "exportCsv": "Export CSV",
+    "exportError": "Unknown error when exporting CSV",
+    "exportCsvTooltip": "Within Time Range",
     "actorId": "Actor ID",
     "allowedByRule": "Allowed by Rule",
     "allowedNoAuth": "Allowed No Auth",

server/private/routers/auditLogs/exportAccessAuditLog.ts

@@ -22,9 +22,11 @@ import logger from "@server/logger";
 import {
     queryAccessAuditLogsParams,
     queryAccessAuditLogsQuery,
-    queryAccess
+    queryAccess,
+    countAccessQuery
 } from "./queryAccessAuditLog";
 import { generateCSV } from "@server/routers/auditLogs/generateCSV";
+import { MAX_EXPORT_LIMIT } from "@server/routers/auditLogs";
 
 registry.registerPath({
     method: "get",
@@ -65,6 +67,15 @@ export async function exportAccessAuditLogs(
         }
 
         const data = { ...parsedQuery.data, ...parsedParams.data };
+        const [{ count }] = await countAccessQuery(data);
+        if (count > MAX_EXPORT_LIMIT) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    `Export limit exceeded. Your selection contains ${count} rows, but the maximum is [${MAX_EXPORT_LIMIT}] rows. Please select a shorter time range to reduce the data.`
+                )
+            );
+        }
 
         const baseQuery = queryAccess(data);
 

server/routers/auditLogs/exportRequestAuditLog.ts

@@ -9,17 +9,23 @@ import logger from "@server/logger";
 import {
     queryAccessAuditLogsQuery,
     queryRequestAuditLogsParams,
-    queryRequest
+    queryRequest,
+    countRequestQuery
 } from "./queryRequestAuditLog";
 import { generateCSV } from "./generateCSV";
 
+export const MAX_EXPORT_LIMIT = 50_000;
+
 registry.registerPath({
     method: "get",
     path: "/org/{orgId}/logs/request",
     description: "Query the request audit log for an organization",
     tags: [OpenAPITags.Org],
     request: {
-        query: queryAccessAuditLogsQuery,
+        query: queryAccessAuditLogsQuery.omit({
+            limit: true,
+            offset: true
+        }),
         params: queryRequestAuditLogsParams
     },
     responses: {}
@@ -53,9 +59,19 @@ export async function exportRequestAuditLogs(
 
         const data = { ...parsedQuery.data, ...parsedParams.data };
 
+        const [{ count }] = await countRequestQuery(data);
+        if (count > MAX_EXPORT_LIMIT) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    `Export limit exceeded. Your selection contains ${count} rows, but the maximum is [${MAX_EXPORT_LIMIT}] rows. Please select a shorter time range to reduce the data.`
+                )
+            );
+        }
+
         const baseQuery = queryRequest(data);
 
-        const log = await baseQuery.limit(data.limit).offset(data.offset);
+        const log = await baseQuery.limit(MAX_EXPORT_LIMIT);
 
         const csvData = generateCSV(log);
 

src/app/[orgId]/settings/logs/request/page.tsx

@@ -11,14 +11,14 @@ import { Button } from "@app/components/ui/button";
 import { useEnvContext } from "@app/hooks/useEnvContext";
 import { toast } from "@app/hooks/useToast";
 import { createApiClient } from "@app/lib/api";
-import { useParams, useRouter, useSearchParams } from "next/navigation";
 import { useTranslations } from "next-intl";
 import { getSevenDaysAgo } from "@app/lib/getSevenDaysAgo";
 import { ColumnDef } from "@tanstack/react-table";
+import axios from "axios";
 import { ArrowUpRight, Key, Lock, Unlock, User } from "lucide-react";
 import Link from "next/link";
-
-import { useEffect, useState } from "react";
+import { useParams, useRouter, useSearchParams } from "next/navigation";
+import { useEffect, useState, useTransition } from "react";
 
 export default function GeneralPage() {
     const router = useRouter();
@@ -29,7 +29,7 @@ export default function GeneralPage() {
 
     const [rows, setRows] = useState<any[]>([]);
     const [isRefreshing, setIsRefreshing] = useState(false);
-    const [isExporting, setIsExporting] = useState(false);
+    const [isExporting, startTransition] = useTransition();
 
     // Pagination state
     const [totalCount, setTotalCount] = useState<number>(0);
@@ -303,8 +303,6 @@ export default function GeneralPage() {
 
     const exportData = async () => {
         try {
-            setIsExporting(true);
-
             // Prepare query params for export
             const params: any = {
                 timeStart: dateRange.startDate?.date
@@ -336,11 +334,21 @@ export default function GeneralPage() {
             document.body.appendChild(link);
             link.click();
             link.parentNode?.removeChild(link);
-            setIsExporting(false);
         } catch (error) {
+            let apiErrorMessage: string | null = null;
+            if (axios.isAxiosError(error) && error.response) {
+                const data = error.response.data;
+
+                if (data instanceof Blob && data.type === "application/json") {
+                    // Parse the Blob as JSON
+                    const text = await data.text();
+                    const errorData = JSON.parse(text);
+                    apiErrorMessage = errorData.message;
+                }
+            }
             toast({
                 title: t("error"),
-                description: t("exportError"),
+                description: apiErrorMessage ?? t("exportError"),
                 variant: "destructive"
             });
         }
@@ -774,7 +782,7 @@ export default function GeneralPage() {
                 searchColumn="host"
                 onRefresh={refreshData}
                 isRefreshing={isRefreshing}
-                onExport={exportData}
+                onExport={() => startTransition(exportData)}
                 isExporting={isExporting}
                 onDateRangeChange={handleDateRangeChange}
                 dateRange={{

src/components/LogDataTable.tsx

@@ -42,6 +42,12 @@ import {
 import { Tabs, TabsList, TabsTrigger } from "@app/components/ui/tabs";
 import { useTranslations } from "next-intl";
 import { DateRangePicker, DateTimeValue } from "@app/components/DateTimePicker";
+import {
+    Tooltip,
+    TooltipContent,
+    TooltipProvider,
+    TooltipTrigger
+} from "./ui/tooltip";
 
 const STORAGE_KEYS = {
     PAGE_SIZE: "datatable-page-size",
@@ -51,6 +57,7 @@ const STORAGE_KEYS = {
 
 export const getStoredPageSize = (
     tableId?: string,
+
     defaultSize = 20
 ): number => {
     if (typeof window === "undefined") return defaultSize;