Merge pull request #2962 from fosrl/dev

1.18.1-s.6

messages/en-US.json

@@ -763,6 +763,7 @@
     "newtEndpoint": "Endpoint",
     "newtId": "ID",
     "newtSecretKey": "Secret",
+    "newtVersion": "Version",
     "architecture": "Architecture",
     "sites": "Sites",
     "siteWgAnyClients": "Use any WireGuard client to connect. You will have to address internal resources using the peer IP.",
@@ -3201,5 +3202,6 @@
     "domainPickerWildcardSubdomainNotAllowed": "Wildcard subdomains are not allowed.",
     "domainPickerWildcardCertWarning": "Wildcard resources may require additional configuration to work properly.",
     "domainPickerWildcardCertWarningLink": "Learn more",
-    "health": "Health"
+    "health": "Health",
+    "domainPendingErrorTitle": "Verification Issue"
 }

server/private/lib/acmeCertSync.ts

@@ -12,6 +12,7 @@
  */
 
 import fs from "fs";
+import path from "path";
 import crypto from "crypto";
 import {
     certificates,
@@ -274,12 +275,244 @@ function detectWildcard(
     return { wildcard: false, wildcardSan: null };
 }
 
+interface HttpCert {
+    wildcard: boolean;
+    altName: string;
+    certName: string;
+    commonName: string;
+    certFile: string;
+    keyFile: string;
+}
+
+async function syncAcmeCertsFromHttp(endpoint: string): Promise<void> {
+    let response: Response;
+    try {
+        response = await fetch(endpoint);
+    } catch (err) {
+        logger.debug(
+            `acmeCertSync: could not reach HTTP endpoint ${endpoint}: ${err}`
+        );
+        return;
+    }
+
+    if (!response.ok) {
+        logger.debug(
+            `acmeCertSync: HTTP endpoint returned status ${response.status}`
+        );
+        return;
+    }
+
+    let httpCerts: HttpCert[];
+    try {
+        httpCerts = await response.json();
+    } catch (err) {
+        logger.debug(
+            `acmeCertSync: could not parse JSON from HTTP endpoint: ${err}`
+        );
+        return;
+    }
+
+    if (!Array.isArray(httpCerts) || httpCerts.length === 0) {
+        logger.debug(
+            `acmeCertSync: no certificates returned from HTTP endpoint`
+        );
+        return;
+    }
+
+    for (const cert of httpCerts) {
+        const domain = cert?.certName;
+
+        if (!domain || typeof domain !== "string") {
+            logger.debug(
+                `acmeCertSync: skipping HTTP cert with missing certName`
+            );
+            continue;
+        }
+
+        const certPem = cert.certFile;
+        const keyPem = cert.keyFile;
+
+        if (!certPem?.trim() || !keyPem?.trim()) {
+            logger.debug(
+                `acmeCertSync: skipping HTTP cert for ${domain} - empty certFile or keyFile`
+            );
+            continue;
+        }
+
+        const firstCertPemForValidation = extractFirstCert(certPem);
+        if (!firstCertPemForValidation) {
+            logger.debug(
+                `acmeCertSync: skipping HTTP cert for ${domain} - no PEM certificate block found`
+            );
+            continue;
+        }
+
+        let validatedX509: crypto.X509Certificate;
+        try {
+            validatedX509 = new crypto.X509Certificate(
+                firstCertPemForValidation
+            );
+        } catch (err) {
+            logger.debug(
+                `acmeCertSync: skipping HTTP cert for ${domain} - invalid X.509 certificate: ${err}`
+            );
+            continue;
+        }
+
+        try {
+            crypto.createPrivateKey(keyPem);
+        } catch (err) {
+            logger.debug(
+                `acmeCertSync: skipping HTTP cert for ${domain} - invalid private key: ${err}`
+            );
+            continue;
+        }
+
+        const wildcard = cert.wildcard ?? false;
+
+        const existing = await db
+            .select()
+            .from(certificates)
+            .where(eq(certificates.domain, domain))
+            .limit(1);
+
+        let oldCertPem: string | null = null;
+        let oldKeyPem: string | null = null;
+
+        if (existing.length > 0 && existing[0].certFile) {
+            try {
+                const storedCertPem = decrypt(
+                    existing[0].certFile,
+                    config.getRawConfig().server.secret!
+                );
+                const wildcardUnchanged = existing[0].wildcard === wildcard;
+                if (storedCertPem === certPem && wildcardUnchanged) {
+                    continue;
+                }
+                oldCertPem = storedCertPem;
+                if (existing[0].keyFile) {
+                    try {
+                        oldKeyPem = decrypt(
+                            existing[0].keyFile,
+                            config.getRawConfig().server.secret!
+                        );
+                    } catch (keyErr) {
+                        logger.debug(
+                            `acmeCertSync: could not decrypt stored key for ${domain}: ${keyErr}`
+                        );
+                    }
+                }
+            } catch (err) {
+                logger.debug(
+                    `acmeCertSync: could not decrypt stored cert for ${domain}, will update: ${err}`
+                );
+            }
+        }
+
+        let expiresAt: number | null = null;
+        try {
+            expiresAt = Math.floor(
+                new Date(validatedX509.validTo).getTime() / 1000
+            );
+        } catch (err) {
+            logger.debug(
+                `acmeCertSync: could not parse cert expiry for ${domain}: ${err}`
+            );
+        }
+
+        const encryptedCert = encrypt(
+            certPem,
+            config.getRawConfig().server.secret!
+        );
+        const encryptedKey = encrypt(
+            keyPem,
+            config.getRawConfig().server.secret!
+        );
+        const now = Math.floor(Date.now() / 1000);
+
+        const domainId = await findDomainId(domain);
+        if (domainId) {
+            logger.debug(
+                `acmeCertSync: resolved domainId "${domainId}" for HTTP cert domain "${domain}"`
+            );
+        } else {
+            logger.debug(
+                `acmeCertSync: no matching domain record found for HTTP cert domain "${domain}"`
+            );
+        }
+
+        if (existing.length > 0) {
+            logger.debug(
+                `acmeCertSync: updating existing certificate (HTTP) for ${domain} (expires ${expiresAt ? new Date(expiresAt * 1000).toISOString() : "unknown"})`
+            );
+            await db
+                .update(certificates)
+                .set({
+                    certFile: encryptedCert,
+                    keyFile: encryptedKey,
+                    status: "valid",
+                    expiresAt,
+                    updatedAt: now,
+                    wildcard,
+                    ...(domainId !== null && { domainId })
+                })
+                .where(eq(certificates.domain, domain));
+
+            await pushCertUpdateToAffectedNewts(
+                domain,
+                domainId,
+                oldCertPem,
+                oldKeyPem
+            );
+        } else {
+            logger.debug(
+                `acmeCertSync: inserting new certificate (HTTP) for ${domain} (expires ${expiresAt ? new Date(expiresAt * 1000).toISOString() : "unknown"})`
+            );
+            await db.insert(certificates).values({
+                domain,
+                domainId,
+                certFile: encryptedCert,
+                keyFile: encryptedKey,
+                status: "valid",
+                expiresAt,
+                createdAt: now,
+                updatedAt: now,
+                wildcard
+            });
+
+            await pushCertUpdateToAffectedNewts(domain, domainId, null, null);
+        }
+    }
+}
+
+function findAcmeJsonFiles(dirPath: string): string[] {
+    const results: string[] = [];
+    let entries: fs.Dirent[];
+    try {
+        entries = fs.readdirSync(dirPath, { withFileTypes: true });
+    } catch (err) {
+        logger.warn(
+            `acmeCertSync: could not read directory "${dirPath}": ${err}`
+        );
+        return results;
+    }
+    for (const entry of entries) {
+        const fullPath = path.join(dirPath, entry.name);
+        if (entry.isDirectory()) {
+            results.push(...findAcmeJsonFiles(fullPath));
+        } else if (entry.isFile() && entry.name === "acme.json") {
+            results.push(fullPath);
+        }
+    }
+    return results;
+}
+
 async function syncAcmeCerts(acmeJsonPath: string): Promise<void> {
     let raw: string;
     try {
         raw = fs.readFileSync(acmeJsonPath, "utf8");
     } catch (err) {
-        logger.debug(`acmeCertSync: could not read ${acmeJsonPath}: ${err}`);
+        logger.warn(`acmeCertSync: could not read "${acmeJsonPath}": ${err}`);
         return;
     }
 
@@ -287,7 +520,9 @@ async function syncAcmeCerts(acmeJsonPath: string): Promise<void> {
     try {
         acmeJson = JSON.parse(raw);
     } catch (err) {
-        logger.debug(`acmeCertSync: could not parse acme.json: ${err}`);
+        logger.warn(
+            `acmeCertSync: could not parse "${acmeJsonPath}" as JSON: ${err}`
+        );
         return;
     }
 
@@ -389,11 +624,7 @@ async function syncAcmeCerts(acmeJsonPath: string): Promise<void> {
         const existing = await db
             .select()
             .from(certificates)
-            .where(
+            .where(and(eq(certificates.domain, domain)))
-                and(
-                    eq(certificates.domain, domain)
-                )
-            )
             .limit(1);
 
         let oldCertPem: string | null = null;
@@ -547,19 +778,62 @@ export function initAcmeCertSync(): void {
         privateConfigData.acme?.acme_json_path ??
         "config/letsencrypt/acme.json";
     const intervalMs = privateConfigData.acme?.sync_interval_ms ?? 5000;
+    const httpEndpoint = privateConfigData.acme?.acme_http_endpoint;
 
     logger.debug(
         `acmeCertSync: starting ACME cert sync from "${acmeJsonPath}" across all resolvers every ${intervalMs}ms`
     );
+    if (httpEndpoint) {
+        logger.debug(
+            `acmeCertSync: also syncing from HTTP endpoint "${httpEndpoint}" every ${intervalMs}ms`
+        );
+    }
 
-    // Run immediately on init, then on the configured interval
+    const runSync = () => {
-    syncAcmeCerts(acmeJsonPath).catch((err) => {
+        if (httpEndpoint) {
-        logger.error(`acmeCertSync: error during initial sync: ${err}`);
+            syncAcmeCertsFromHttp(httpEndpoint).catch((err) => {
+                logger.error(`acmeCertSync: error during HTTP sync: ${err}`);
             });
+        } else {
+            // only run the file-based sync if the HTTP endpoint is not configured, to avoid doubling up
+            let stat: fs.Stats | null = null;
+            try {
+                stat = fs.statSync(acmeJsonPath);
+            } catch (err) {
+                logger.warn(
+                    `acmeCertSync: cannot stat path "${acmeJsonPath}": ${err}`
+                );
+                return;
+            }
 
-    setInterval(() => {
+            if (stat.isDirectory()) {
+                const files = findAcmeJsonFiles(acmeJsonPath);
+                if (files.length === 0) {
+                    logger.debug(
+                        `acmeCertSync: no acme.json files found in directory "${acmeJsonPath}"`
+                    );
+                    return;
+                }
+                logger.debug(
+                    `acmeCertSync: found ${files.length} acme.json file(s) in directory "${acmeJsonPath}"`
+                );
+                for (const file of files) {
+                    syncAcmeCerts(file).catch((err) => {
+                        logger.error(
+                            `acmeCertSync: error during sync of "${file}": ${err}`
+                        );
+                    });
+                }
+            } else {
                 syncAcmeCerts(acmeJsonPath).catch((err) => {
                     logger.error(`acmeCertSync: error during sync: ${err}`);
                 });
-    }, intervalMs);
+            }
+        }
+    };
+
+    // Run immediately on init, then on the configured interval
+    runSync();
+
+    setInterval(runSync, intervalMs);
 }

server/private/lib/readConfigFile.ts

@@ -21,7 +21,8 @@ import { getEnvOrYaml } from "@server/lib/getEnvOrYaml";
 
 const portSchema = z.number().positive().gt(0).lte(65535);
 
-export const privateConfigSchema = z.object({
+export const privateConfigSchema = z
+    .object({
         app: z
             .object({
                 region: z.string().optional().default("default"),
@@ -70,10 +71,7 @@ export const privateConfigSchema = z.object({
                     .optional(),
                 tls: z
                     .object({
-                    rejectUnauthorized: z
+                        rejectUnauthorized: z.boolean().optional().default(true)
-                        .boolean()
-                        .optional()
-                        .default(true)
                     })
                     .optional()
             })
@@ -102,6 +100,7 @@ export const privateConfigSchema = z.object({
                     .string()
                     .optional()
                     .default("config/letsencrypt/acme.json"),
+                acme_http_endpoint: z.string().optional(),
                 sync_interval_ms: z.number().optional().default(5000)
             })
             .optional(),
@@ -181,13 +180,13 @@ export const privateConfigSchema = z.object({
                 webhook_secret: z
                     .string()
                     .optional()
-                .transform(getEnvOrYaml("STRIPE_WEBHOOK_SECRET")),
+                    .transform(getEnvOrYaml("STRIPE_WEBHOOK_SECRET"))
                 // s3Bucket: z.string(),
                 // s3Region: z.string().default("us-east-1"),
                 // localFilePath: z.string().optional()
             })
             .optional()
-})
+    })
     .transform((data) => {
         // this to maintain backwards compatibility with the old config file
         const identityProviderMode = data.app?.identity_provider_mode;

server/private/routers/certificates/createCertificate.ts

@@ -79,7 +79,7 @@ export async function createCertificate(
 
     let domainToWrite = domain;
     if (
-        domainRecord.type == "wildcard" &&
+        domainRecord.type == "wildcard" && // this is to fix the wildcard certs for traefik in self hosted NOT ON THE CLOUD
         domainRecord.preferWildcardCert &&
         !domain.startsWith("*.")
     ) {
@@ -89,6 +89,16 @@ export async function createCertificate(
             domainToWrite = parts.slice(1).join(".");
             domainToWrite = `*.${domainToWrite}`;
         }
+    } else if (domainRecord.type == "ns") {
+        // first if we have a * in the domain for this case we dont want to include it because it will mess with the cert generator so remove it
+        if (domain.startsWith("*.")) {
+            domain = domain.slice(2);
+        }
+
+        const parts = domain.split(".");
+        if (parts.length > 2) {
+            domainToWrite = parts.slice(1).join(".");
+        }
     }
 
     // No cert found, create a new one in pending state

server/routers/site/getSite.ts

@@ -42,9 +42,12 @@ async function query(siteId?: number, niceId?: string, orgId?: string) {
     }
 }
 
-export type GetSiteResponse = NonNullable<
+type SiteQueryRow = NonNullable<Awaited<ReturnType<typeof query>>>;
-    Awaited<ReturnType<typeof query>>
+
->["sites"] & { newtId: string | null };
+export type GetSiteResponse = SiteQueryRow["sites"] & {
+    newtId: string | null;
+    newtVersion: string | null;
+};
 
 registry.registerPath({
     method: "get",
@@ -100,7 +103,8 @@ export async function getSite(
 
         const data: GetSiteResponse = {
             ...site.sites,
-            newtId: site.newt ? site.newt.newtId : null
+            newtId: site.newt ? site.newt.newtId : null,
+            newtVersion: site.newt?.version ?? null
         };
 
         return response<GetSiteResponse>(res, {

src/components/ProductUpdates.tsx

@@ -103,10 +103,12 @@ export default function ProductUpdates({
             )}
         >
             <div className="flex flex-col gap-1">
+                {filteredUpdates.length > 0 && (
+                    <div className="mt-3 flex flex-col gap-2">
                         {filteredUpdates.length > 1 && (
                             <small
                                 className={cn(
-                            "text-xs text-muted-foreground flex items-center gap-1 mt-2",
+                                    "text-xs text-muted-foreground flex items-center gap-1",
                                     showMoreUpdatesText
                                         ? "animate-in fade-in duration-300"
                                         : "opacity-0"
@@ -116,10 +118,12 @@ export default function ProductUpdates({
                                 <span>
                                     {showNewVersionPopup
                                         ? t("productUpdateMoreInfo", {
-                                      noOfUpdates: filteredUpdates.length
+                                              noOfUpdates:
+                                                  filteredUpdates.length
                                           })
                                         : t("productUpdateInfo", {
-                                      noOfUpdates: filteredUpdates.length
+                                              noOfUpdates:
+                                                  filteredUpdates.length
                                           })}
                                 </span>
                             </small>
@@ -130,14 +134,21 @@ export default function ProductUpdates({
                             onDimissAll={() =>
                                 setProductUpdatesRead([
                                     ...productUpdatesRead,
-                            ...filteredUpdates.map((update) => update.id)
+                                    ...filteredUpdates.map(
+                                        (update) => update.id
+                                    )
                                 ])
                             }
                             onDimiss={(id) =>
-                        setProductUpdatesRead([...productUpdatesRead, id])
+                                setProductUpdatesRead([
+                                    ...productUpdatesRead,
+                                    id
+                                ])
                             }
                         />
                     </div>
+                )}
+            </div>
 
             <NewVersionAvailable
                 version={data.latestVersion?.data}

src/components/SiteInfoCard.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
+import { Alert, AlertDescription } from "@/components/ui/alert";
 import { useSiteContext } from "@app/hooks/useSiteContext";
 import {
     InfoSection,
@@ -9,41 +9,29 @@ import {
     InfoSectionTitle
 } from "@app/components/InfoSection";
 import { useTranslations } from "next-intl";
-import { useEnvContext } from "@app/hooks/useEnvContext";
 
 type SiteInfoCardProps = {};
 
+function formatPublicEndpoint(endpoint: string) {
+    return endpoint.includes(":")
+        ? endpoint.substring(0, endpoint.lastIndexOf(":"))
+        : endpoint;
+}
+
 export default function SiteInfoCard({}: SiteInfoCardProps) {
-    const { site, updateSite } = useSiteContext();
+    const { site } = useSiteContext();
     const t = useTranslations();
-    const { env } = useEnvContext();
 
-    const getConnectionTypeString = (type: string) => {
+    const identifierSection = (
-        if (type === "newt") {
-            return "Newt";
-        } else if (type === "wireguard") {
-            return "WireGuard";
-        } else if (type === "local") {
-            return t("local");
-        } else {
-            return t("unknown");
-        }
-    };
-
-    return (
-        <Alert>
-            <AlertDescription>
-                <InfoSections cols={site.endpoint ? 4 : 3}>
         <InfoSection>
             <InfoSectionTitle>{t("identifier")}</InfoSectionTitle>
             <InfoSectionContent>{site.niceId}</InfoSectionContent>
         </InfoSection>
-                    {(site.type == "newt" || site.type == "wireguard") && (
+    );
-                        <>
+
+    const statusSection = (
         <InfoSection>
-                                <InfoSectionTitle>
+            <InfoSectionTitle>{t("status")}</InfoSectionTitle>
-                                    {t("status")}
-                                </InfoSectionTitle>
             <InfoSectionContent>
                 {site.online ? (
                     <div className="text-green-500 flex items-center space-x-2">
@@ -58,28 +46,100 @@ export default function SiteInfoCard({}: SiteInfoCardProps) {
                 )}
             </InfoSectionContent>
         </InfoSection>
-                        </>
+    );
-                    )}
+
+    const endpointSection = site.endpoint ? (
+        <InfoSection>
+            <InfoSectionTitle>{t("publicIpEndpoint")}</InfoSectionTitle>
+            <InfoSectionContent>
+                {formatPublicEndpoint(site.endpoint)}
+            </InfoSectionContent>
+        </InfoSection>
+    ) : null;
+
+    if (site.type === "newt") {
+        return (
+            <Alert>
+                <AlertDescription>
+                    <InfoSections cols={site.endpoint ? 5 : 4}>
+                        {identifierSection}
+                        {statusSection}
+                        <InfoSection>
+                            <InfoSectionTitle>
+                                {t("connectionType")}
+                            </InfoSectionTitle>
+                            <InfoSectionContent>Newt</InfoSectionContent>
+                        </InfoSection>
+                        <InfoSection>
+                            <InfoSectionTitle>
+                                {t("newtVersion")}
+                            </InfoSectionTitle>
+                            <InfoSectionContent>
+                                {site.newtVersion
+                                    ? `v${site.newtVersion}`
+                                    : "-"}
+                            </InfoSectionContent>
+                        </InfoSection>
+                        {endpointSection}
+                    </InfoSections>
+                </AlertDescription>
+            </Alert>
+        );
+    }
+
+    if (site.type === "wireguard") {
+        return (
+            <Alert>
+                <AlertDescription>
+                    <InfoSections cols={site.endpoint ? 4 : 3}>
+                        {identifierSection}
+                        {statusSection}
+                        <InfoSection>
+                            <InfoSectionTitle>
+                                {t("connectionType")}
+                            </InfoSectionTitle>
+                            <InfoSectionContent>WireGuard</InfoSectionContent>
+                        </InfoSection>
+                        {endpointSection}
+                    </InfoSections>
+                </AlertDescription>
+            </Alert>
+        );
+    }
+
+    if (site.type === "local") {
+        return (
+            <Alert>
+                <AlertDescription>
+                    <InfoSections cols={site.endpoint ? 3 : 2}>
+                        {identifierSection}
                         <InfoSection>
                             <InfoSectionTitle>
                                 {t("connectionType")}
                             </InfoSectionTitle>
                             <InfoSectionContent>
-                            {getConnectionTypeString(site.type)}
+                                {t("local")}
                             </InfoSectionContent>
                         </InfoSection>
-                    {site.endpoint && (
+                        {endpointSection}
+                    </InfoSections>
+                </AlertDescription>
+            </Alert>
+        );
+    }
+
+    return (
+        <Alert>
+            <AlertDescription>
+                <InfoSections cols={site.endpoint ? 3 : 2}>
+                    {identifierSection}
                     <InfoSection>
                         <InfoSectionTitle>
-                                {t("publicIpEndpoint")}
+                            {t("connectionType")}
                         </InfoSectionTitle>
-                            <InfoSectionContent>
+                        <InfoSectionContent>{t("unknown")}</InfoSectionContent>
-                                {site.endpoint.includes(":")
-                                    ? site.endpoint.substring(0, site.endpoint.lastIndexOf(":"))
-                                    : site.endpoint}
-                            </InfoSectionContent>
                     </InfoSection>
-                    )}
+                    {endpointSection}
                 </InfoSections>
             </AlertDescription>
         </Alert>

src/components/resource-target-address-item.tsx

@@ -113,10 +113,10 @@ export function ResourceTargetAddressItem({
                                     ? selectedSite?.name
                                     : t("siteSelect")}
                             </span>
-                            <CaretSortIcon className="ml-2h-4 w-4 shrink-0 opacity-50" />
+                            <CaretSortIcon className="ml-2 h-4 w-4 shrink-0 opacity-50" />
                         </Button>
                     </PopoverTrigger>
-                    <PopoverContent className="p-0 w-45">
+                    <PopoverContent className="p-0">
                         <SitesSelector
                             orgId={orgId}
                             selectedSite={selectedSite}
@@ -225,7 +225,6 @@ export function ResourceTargetAddressItem({
                         }
                     }}
                 />
-
             </div>
         </div>
     );