Implement s3 streaming destination

This commit is contained in:
Owen
2026-05-07 21:09:21 -07:00
parent 10f95896aa
commit cbdc74768f
6 changed files with 895 additions and 37 deletions

View File

@@ -1,6 +1,6 @@
"use client";
import { useState, useEffect, useCallback } from "react";
import React, { useState, useEffect, useCallback } from "react";
import { useParams } from "next/navigation";
import { createApiClient, formatAxiosError } from "@app/lib/api";
import { useEnvContext } from "@app/hooks/useEnvContext";
@@ -38,7 +38,10 @@ import {
HttpDestinationCredenza,
parseHttpConfig
} from "@app/components/HttpDestinationCredenza";
import { S3DestinationCredenza } from "@app/components/S3DestinationCredenza";
import {
S3DestinationCredenza,
parseS3Config
} from "@app/components/S3DestinationCredenza";
import { DatadogDestinationCredenza } from "@app/components/DatadogDestinationCredenza";
import { useTranslations } from "next-intl";
@@ -64,6 +67,42 @@ interface DestinationCardProps {
disabled?: boolean;
}
function getDestinationDisplay(destination: Destination): {
name: string;
typeLabel: string;
detail: string;
icon: React.ReactNode;
} {
if (destination.type === "s3") {
const cfg = parseS3Config(destination.config);
const detail = cfg.bucket
? `s3://${cfg.bucket}${cfg.prefix ? `/${cfg.prefix.replace(/^\/+/, "")}` : ""}`
: "";
return {
name: cfg.name,
typeLabel: "Amazon S3",
detail,
icon: (
<Image
src="/third-party/s3.png"
alt="Amazon S3"
width={16}
height={16}
className="rounded-sm"
/>
)
};
}
// Default: HTTP
const cfg = parseHttpConfig(destination.config);
return {
name: cfg.name,
typeLabel: "HTTP",
detail: cfg.url,
icon: <Globe className="h-3.5 w-3.5 text-black" />
};
}
function DestinationCard({
destination,
onToggle,
@@ -73,25 +112,25 @@ function DestinationCard({
disabled = false
}: DestinationCardProps) {
const t = useTranslations();
const cfg = parseHttpConfig(destination.config);
const { name, typeLabel, detail, icon } =
getDestinationDisplay(destination);
return (
<div className="relative flex flex-col rounded-lg border bg-card text-card-foreground p-5 gap-3">
{/* Top row: icon + name/type + toggle */}
<div className="flex items-start justify-between gap-3">
<div className="flex items-center gap-3 min-w-0">
{/* Squirkle icon: gray outer → white inner → black globe */}
<div className="shrink-0 flex items-center justify-center w-10 h-10 rounded-2xl bg-muted">
<div className="flex items-center justify-center w-6 h-6 rounded-xl bg-white shadow-sm">
<Globe className="h-3.5 w-3.5 text-black" />
{icon}
</div>
</div>
<div className="min-w-0">
<p className="font-semibold text-sm leading-tight truncate">
{cfg.name || t("streamingUnnamedDestination")}
{name || t("streamingUnnamedDestination")}
</p>
<p className="text-xs text-muted-foreground truncate mt-0.5">
HTTP
{typeLabel}
</p>
</div>
</div>
@@ -105,9 +144,9 @@ function DestinationCard({
/>
</div>
{/* URL preview */}
{/* Detail preview (URL for HTTP, s3:// path for S3) */}
<p className="text-xs text-muted-foreground truncate">
{cfg.url || (
{detail || (
<span className="italic">
{t("streamingNoUrlConfigured")}
</span>
@@ -485,7 +524,7 @@ export default function StreamingDestinationsPage() {
if (!v) setDeleteTarget(null);
}}
string={
parseHttpConfig(deleteTarget.config).name ||
getDestinationDisplay(deleteTarget).name ||
t("streamingDeleteDialogThisDestination")
}
title={t("streamingDeleteTitle")}
@@ -493,7 +532,7 @@ export default function StreamingDestinationsPage() {
<p>
{t("streamingDeleteDialogAreYouSure")}{" "}
<span>
{parseHttpConfig(deleteTarget.config).name ||
{getDestinationDisplay(deleteTarget).name ||
t("streamingDeleteDialogThisDestination")}
</span>
{t("streamingDeleteDialogPermanentlyRemoved")}

View File

@@ -1,6 +1,6 @@
"use client";
import { useState, useEffect } from "react";
import {
Credenza,
CredenzaBody,
@@ -12,13 +12,62 @@ import {
CredenzaTitle
} from "@app/components/Credenza";
import { Button } from "@app/components/ui/button";
import { ContactSalesBanner } from "@app/components/ContactSalesBanner";
import { Input } from "@app/components/ui/input";
import { Label } from "@app/components/ui/label";
import { Switch } from "@app/components/ui/switch";
import { HorizontalTabs } from "@app/components/HorizontalTabs";
import { RadioGroup, RadioGroupItem } from "@app/components/ui/radio-group";
import { Checkbox } from "@app/components/ui/checkbox";
import { createApiClient, formatAxiosError } from "@app/lib/api";
import { useEnvContext } from "@app/hooks/useEnvContext";
import { toast } from "@app/hooks/useToast";
import { useTranslations } from "next-intl";
import { Destination } from "@app/components/HttpDestinationCredenza";
// ── Types ──────────────────────────────────────────────────────────────────────
export type S3PayloadFormat = "json_array" | "ndjson" | "csv";
export interface S3Config {
name: string;
accessKeyId: string;
secretAccessKey: string;
region: string;
bucket: string;
prefix: string;
endpoint: string;
format: S3PayloadFormat;
gzip: boolean;
}
// ── Helpers ────────────────────────────────────────────────────────────────────
export const defaultS3Config = (): S3Config => ({
name: "",
accessKeyId: "",
secretAccessKey: "",
region: "us-east-1",
bucket: "",
prefix: "",
endpoint: "",
format: "json_array",
gzip: false
});
export function parseS3Config(raw: string): S3Config {
try {
return { ...defaultS3Config(), ...JSON.parse(raw) };
} catch {
return defaultS3Config();
}
}
// ── Component ──────────────────────────────────────────────────────────────────
export interface S3DestinationCredenzaProps {
open: boolean;
onOpenChange: (open: boolean) => void;
editing: any;
editing: Destination | null;
orgId: string;
onSaved: () => void;
}
@@ -28,18 +77,84 @@ export function S3DestinationCredenza({
onOpenChange,
editing,
orgId,
onSaved,
onSaved
}: S3DestinationCredenzaProps) {
const api = createApiClient(useEnvContext());
const t = useTranslations();
const [saving, setSaving] = useState(false);
const [cfg, setCfg] = useState<S3Config>(defaultS3Config());
const [sendAccessLogs, setSendAccessLogs] = useState(false);
const [sendActionLogs, setSendActionLogs] = useState(false);
const [sendConnectionLogs, setSendConnectionLogs] = useState(false);
const [sendRequestLogs, setSendRequestLogs] = useState(false);
useEffect(() => {
if (open) {
setCfg(editing ? parseS3Config(editing.config) : defaultS3Config());
setSendAccessLogs(editing?.sendAccessLogs ?? false);
setSendActionLogs(editing?.sendActionLogs ?? false);
setSendConnectionLogs(editing?.sendConnectionLogs ?? false);
setSendRequestLogs(editing?.sendRequestLogs ?? false);
}
}, [open, editing]);
const update = (patch: Partial<S3Config>) =>
setCfg((prev) => ({ ...prev, ...patch }));
const isValid =
cfg.name.trim() !== "" &&
cfg.accessKeyId.trim() !== "" &&
cfg.secretAccessKey.trim() !== "" &&
cfg.region.trim() !== "" &&
cfg.bucket.trim() !== "";
async function handleSave() {
if (!isValid) return;
setSaving(true);
try {
const payload = {
type: "s3",
config: JSON.stringify(cfg),
sendAccessLogs,
sendActionLogs,
sendConnectionLogs,
sendRequestLogs
};
if (editing) {
await api.post(
`/org/${orgId}/event-streaming-destination/${editing.destinationId}`,
payload
);
toast({ title: t("s3DestUpdatedSuccess") });
} else {
await api.put(
`/org/${orgId}/event-streaming-destination`,
payload
);
toast({ title: t("s3DestCreatedSuccess") });
}
onSaved();
onOpenChange(false);
} catch (e) {
toast({
variant: "destructive",
title: editing
? t("s3DestUpdateFailed")
: t("s3DestCreateFailed"),
description: formatAxiosError(e, t("streamingUnexpectedError"))
});
} finally {
setSaving(false);
}
}
return (
<Credenza open={open} onOpenChange={onOpenChange}>
<CredenzaContent className="sm:max-w-2xl">
<CredenzaHeader>
<CredenzaTitle>
{editing
? t("S3DestEditTitle")
: t("S3DestAddTitle")}
{editing ? t("S3DestEditTitle") : t("S3DestAddTitle")}
</CredenzaTitle>
<CredenzaDescription>
{editing
@@ -49,13 +164,367 @@ export function S3DestinationCredenza({
</CredenzaHeader>
<CredenzaBody>
<ContactSalesBanner />
<HorizontalTabs
clientSide
items={[
{ title: t("s3DestTabSettings"), href: "" },
{ title: t("s3DestTabFormat"), href: "" },
{ title: t("httpDestTabLogs"), href: "" }
]}
>
{/* ── Settings tab ────────────────────────────── */}
<div className="space-y-6 mt-4 p-1">
{/* Name */}
<div className="space-y-2">
<Label htmlFor="s3-name">
{t("s3DestNameLabel")}
</Label>
<Input
id="s3-name"
placeholder={t("s3DestNamePlaceholder")}
value={cfg.name}
onChange={(e) =>
update({ name: e.target.value })
}
/>
</div>
{/* AWS Access Key ID */}
<div className="space-y-2">
<Label htmlFor="s3-access-key-id">
{t("s3DestAccessKeyIdLabel")}
</Label>
<Input
id="s3-access-key-id"
placeholder="AKIAIOSFODNN7EXAMPLE"
value={cfg.accessKeyId}
onChange={(e) =>
update({
accessKeyId: e.target.value
})
}
autoComplete="off"
/>
</div>
{/* AWS Secret Access Key */}
<div className="space-y-2">
<Label htmlFor="s3-secret-key">
{t("s3DestSecretAccessKeyLabel")}
</Label>
<Input
id="s3-secret-key"
type="password"
placeholder={t(
"s3DestSecretAccessKeyPlaceholder"
)}
value={cfg.secretAccessKey}
onChange={(e) =>
update({
secretAccessKey: e.target.value
})
}
autoComplete="new-password"
/>
</div>
{/* Region */}
<div className="space-y-2">
<Label htmlFor="s3-region">
{t("s3DestRegionLabel")}
</Label>
<Input
id="s3-region"
placeholder="us-east-1"
value={cfg.region}
onChange={(e) =>
update({ region: e.target.value })
}
/>
</div>
{/* Bucket */}
<div className="space-y-2">
<Label htmlFor="s3-bucket">
{t("s3DestBucketLabel")}
</Label>
<Input
id="s3-bucket"
placeholder="my-logs-bucket"
value={cfg.bucket}
onChange={(e) =>
update({ bucket: e.target.value })
}
/>
</div>
{/* Prefix */}
<div className="space-y-2">
<Label htmlFor="s3-prefix">
{t("s3DestPrefixLabel")}
</Label>
<Input
id="s3-prefix"
placeholder="pangolin/logs"
value={cfg.prefix}
onChange={(e) =>
update({ prefix: e.target.value })
}
/>
<p className="text-xs text-muted-foreground">
{t("s3DestPrefixDescription")}
</p>
</div>
{/* Custom endpoint (optional for S3-compatible storage) */}
<div className="space-y-2">
<Label htmlFor="s3-endpoint">
{t("s3DestEndpointLabel")}
</Label>
<Input
id="s3-endpoint"
placeholder="https://s3.example.com"
value={cfg.endpoint}
onChange={(e) =>
update({ endpoint: e.target.value })
}
/>
<p className="text-xs text-muted-foreground">
{t("s3DestEndpointDescription")}
</p>
</div>
</div>
{/* ── Format tab ───────────────────────────────── */}
<div className="space-y-6 mt-4 p-1">
{/* Gzip compression toggle */}
<div className="flex items-start gap-3 rounded-md border p-3">
<Switch
id="s3-gzip"
checked={cfg.gzip}
onCheckedChange={(v) => update({ gzip: v })}
className="mt-0.5"
/>
<div>
<Label
htmlFor="s3-gzip"
className="cursor-pointer font-medium"
>
{t("s3DestGzipLabel")}
</Label>
<p className="text-xs text-muted-foreground mt-0.5">
{t("s3DestGzipDescription")}
</p>
</div>
</div>
{/* Payload format selector */}
<div className="space-y-3">
<div>
<label className="font-medium block">
{t("s3DestFormatTitle")}
</label>
<p className="text-sm text-muted-foreground mt-0.5">
{t("s3DestFormatDescription")}
</p>
</div>
<RadioGroup
value={cfg.format}
onValueChange={(v) =>
update({
format: v as S3PayloadFormat
})
}
className="gap-2"
>
{/* JSON Array */}
<label className="flex items-start gap-3 rounded-md border p-3 cursor-pointer has-[:checked]:border-primary has-[:checked]:bg-primary/5">
<RadioGroupItem
value="json_array"
className="mt-0.5"
/>
<div>
<p className="text-sm font-medium leading-none">
{t(
"httpDestFormatJsonArrayTitle"
)}
</p>
<p className="text-xs text-muted-foreground mt-1">
{t(
"s3DestFormatJsonArrayDescription"
)}
</p>
</div>
</label>
{/* NDJSON */}
<label className="flex items-start gap-3 rounded-md border p-3 cursor-pointer has-[:checked]:border-primary has-[:checked]:bg-primary/5">
<RadioGroupItem
value="ndjson"
className="mt-0.5"
/>
<div>
<p className="text-sm font-medium leading-none">
{t("httpDestFormatNdjsonTitle")}
</p>
<p className="text-xs text-muted-foreground mt-1">
{t(
"s3DestFormatNdjsonDescription"
)}
</p>
</div>
</label>
{/* CSV */}
<label className="flex items-start gap-3 rounded-md border p-3 cursor-pointer has-[:checked]:border-primary has-[:checked]:bg-primary/5">
<RadioGroupItem
value="csv"
className="mt-0.5"
/>
<div>
<p className="text-sm font-medium leading-none">
{t("s3DestFormatCsvTitle")}
</p>
<p className="text-xs text-muted-foreground mt-1">
{t(
"s3DestFormatCsvDescription"
)}
</p>
</div>
</label>
</RadioGroup>
</div>
</div>
{/* ── Logs tab ──────────────────────────────────── */}
<div className="space-y-6 mt-4 p-1">
<div>
<label className="font-medium block">
{t("httpDestLogTypesTitle")}
</label>
<p className="text-sm text-muted-foreground mt-0.5">
{t("httpDestLogTypesDescription")}
</p>
</div>
<div className="space-y-3">
<div className="flex items-start gap-3 rounded-md border p-3">
<Checkbox
id="s3-log-access"
checked={sendAccessLogs}
onCheckedChange={(v) =>
setSendAccessLogs(v === true)
}
className="mt-0.5"
/>
<div>
<Label
htmlFor="s3-log-access"
className="cursor-pointer font-medium"
>
{t("httpDestAccessLogsTitle")}
</Label>
<p className="text-xs text-muted-foreground mt-0.5">
{t("httpDestAccessLogsDescription")}
</p>
</div>
</div>
<div className="flex items-start gap-3 rounded-md border p-3">
<Checkbox
id="s3-log-action"
checked={sendActionLogs}
onCheckedChange={(v) =>
setSendActionLogs(v === true)
}
className="mt-0.5"
/>
<div>
<Label
htmlFor="s3-log-action"
className="cursor-pointer font-medium"
>
{t("httpDestActionLogsTitle")}
</Label>
<p className="text-xs text-muted-foreground mt-0.5">
{t("httpDestActionLogsDescription")}
</p>
</div>
</div>
<div className="flex items-start gap-3 rounded-md border p-3">
<Checkbox
id="s3-log-connection"
checked={sendConnectionLogs}
onCheckedChange={(v) =>
setSendConnectionLogs(v === true)
}
className="mt-0.5"
/>
<div>
<Label
htmlFor="s3-log-connection"
className="cursor-pointer font-medium"
>
{t("httpDestConnectionLogsTitle")}
</Label>
<p className="text-xs text-muted-foreground mt-0.5">
{t(
"httpDestConnectionLogsDescription"
)}
</p>
</div>
</div>
<div className="flex items-start gap-3 rounded-md border p-3">
<Checkbox
id="s3-log-request"
checked={sendRequestLogs}
onCheckedChange={(v) =>
setSendRequestLogs(v === true)
}
className="mt-0.5"
/>
<div>
<Label
htmlFor="s3-log-request"
className="cursor-pointer font-medium"
>
{t("httpDestRequestLogsTitle")}
</Label>
<p className="text-xs text-muted-foreground mt-0.5">
{t(
"httpDestRequestLogsDescription"
)}
</p>
</div>
</div>
</div>
</div>
</HorizontalTabs>
</CredenzaBody>
<CredenzaFooter>
<CredenzaClose asChild>
<Button variant="outline">{t("cancel")}</Button>
<Button
type="button"
variant="outline"
disabled={saving}
>
{t("cancel")}
</Button>
</CredenzaClose>
<Button
type="button"
onClick={handleSave}
loading={saving}
disabled={!isValid || saving}
>
{editing
? t("s3DestSaveChanges")
: t("s3DestCreateDestination")}
</Button>
</CredenzaFooter>
</CredenzaContent>
</Credenza>