Merge branch 'main' into dev

.github/workflows/cicd.yml

@@ -329,20 +329,89 @@ jobs:
                   skopeo login ghcr.io -u "${{ github.actor }}" -p "${{ secrets.GITHUB_TOKEN }}"
               shell: bash
 
-            - name: Copy tag from Docker Hub to GHCR
+            - name: Copy tags from Docker Hub to GHCR
-              # Mirror the already-built image (all architectures) to GHCR so we can sign it
+              # Mirror the already-built images (all architectures) to GHCR so we can sign them
               # Wait a bit for both architectures to be available in Docker Hub manifest
               env:
                   REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json
               run: |
                   set -euo pipefail
                   TAG=${{ env.TAG }}
-                  echo "Waiting for multi-arch manifest to be ready..."
+                  MAJOR_TAG=$(echo $TAG | cut -d. -f1)
+                  MINOR_TAG=$(echo $TAG | cut -d. -f1,2)
+                  
+                  echo "Waiting for multi-arch manifests to be ready..."
                   sleep 30
+                  
+                  # Determine if this is an RC release
+                  IS_RC="false"
+                  if echo "$TAG" | grep -qE "rc[0-9]+$"; then
+                      IS_RC="true"
+                  fi
+                  
+                  if [ "$IS_RC" = "true" ]; then
+                      echo "RC release detected - copying version-specific tags only"
+                      
+                      # SQLite OSS
                       echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG} -> ${{ env.GHCR_IMAGE }}:${TAG}"
                       skopeo copy --all --retry-times 3 \
                         docker://$DOCKERHUB_IMAGE:$TAG \
                         docker://$GHCR_IMAGE:$TAG
+                      
+                      # PostgreSQL OSS
+                      echo "Copying ${{ env.DOCKERHUB_IMAGE }}:postgresql-${TAG} -> ${{ env.GHCR_IMAGE }}:postgresql-${TAG}"
+                      skopeo copy --all --retry-times 3 \
+                        docker://$DOCKERHUB_IMAGE:postgresql-$TAG \
+                        docker://$GHCR_IMAGE:postgresql-$TAG
+                      
+                      # SQLite Enterprise
+                      echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-${TAG} -> ${{ env.GHCR_IMAGE }}:ee-${TAG}"
+                      skopeo copy --all --retry-times 3 \
+                        docker://$DOCKERHUB_IMAGE:ee-$TAG \
+                        docker://$GHCR_IMAGE:ee-$TAG
+                      
+                      # PostgreSQL Enterprise
+                      echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-postgresql-${TAG} -> ${{ env.GHCR_IMAGE }}:ee-postgresql-${TAG}"
+                      skopeo copy --all --retry-times 3 \
+                        docker://$DOCKERHUB_IMAGE:ee-postgresql-$TAG \
+                        docker://$GHCR_IMAGE:ee-postgresql-$TAG
+                  else
+                      echo "Regular release detected - copying all tags (latest, major, minor, full version)"
+                      
+                      # SQLite OSS - all tags
+                      for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
+                          echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:${TAG_SUFFIX}"
+                          skopeo copy --all --retry-times 3 \
+                            docker://$DOCKERHUB_IMAGE:$TAG_SUFFIX \
+                            docker://$GHCR_IMAGE:$TAG_SUFFIX
+                      done
+                      
+                      # PostgreSQL OSS - all tags
+                      for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
+                          echo "Copying ${{ env.DOCKERHUB_IMAGE }}:postgresql-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:postgresql-${TAG_SUFFIX}"
+                          skopeo copy --all --retry-times 3 \
+                            docker://$DOCKERHUB_IMAGE:postgresql-$TAG_SUFFIX \
+                            docker://$GHCR_IMAGE:postgresql-$TAG_SUFFIX
+                      done
+                      
+                      # SQLite Enterprise - all tags
+                      for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
+                          echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:ee-${TAG_SUFFIX}"
+                          skopeo copy --all --retry-times 3 \
+                            docker://$DOCKERHUB_IMAGE:ee-$TAG_SUFFIX \
+                            docker://$GHCR_IMAGE:ee-$TAG_SUFFIX
+                      done
+                      
+                      # PostgreSQL Enterprise - all tags
+                      for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
+                          echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-postgresql-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:ee-postgresql-${TAG_SUFFIX}"
+                          skopeo copy --all --retry-times 3 \
+                            docker://$DOCKERHUB_IMAGE:ee-postgresql-$TAG_SUFFIX \
+                            docker://$GHCR_IMAGE:ee-postgresql-$TAG_SUFFIX
+                      done
+                  fi
+                  
+                  echo "All images copied successfully to GHCR!"
               shell: bash
 
             - name: Login to GitHub Container Registry (for cosign)
@@ -371,11 +440,40 @@ jobs:
                   issuer="https://token.actions.githubusercontent.com"
                   id_regex="^https://github.com/${{ github.repository }}/.+"  # accept this repo (all workflows/refs)
 
-                  for IMAGE in "${GHCR_IMAGE}" "${DOCKERHUB_IMAGE}"; do
+                  # Determine if this is an RC release
-                    echo "Processing ${IMAGE}:${TAG}"
+                  IS_RC="false"
+                  if echo "$TAG" | grep -qE "rc[0-9]+$"; then
+                      IS_RC="true"
+                  fi
 
-                    DIGEST="$(skopeo inspect --retry-times 3 docker://${IMAGE}:${TAG} | jq -r '.Digest')"
+                  # Define image variants to sign
-                    REF="${IMAGE}@${DIGEST}"
+                  if [ "$IS_RC" = "true" ]; then
+                      echo "RC release - signing version-specific tags only"
+                      IMAGE_TAGS=(
+                          "${TAG}"
+                          "postgresql-${TAG}"
+                          "ee-${TAG}"
+                          "ee-postgresql-${TAG}"
+                      )
+                  else
+                      echo "Regular release - signing all tags"
+                      MAJOR_TAG=$(echo $TAG | cut -d. -f1)
+                      MINOR_TAG=$(echo $TAG | cut -d. -f1,2)
+                      IMAGE_TAGS=(
+                          "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"
+                          "postgresql-latest" "postgresql-$MAJOR_TAG" "postgresql-$MINOR_TAG" "postgresql-$TAG"
+                          "ee-latest" "ee-$MAJOR_TAG" "ee-$MINOR_TAG" "ee-$TAG"
+                          "ee-postgresql-latest" "ee-postgresql-$MAJOR_TAG" "ee-postgresql-$MINOR_TAG" "ee-postgresql-$TAG"
+                      )
+                  fi
+
+                  # Sign each image variant for both registries
+                  for BASE_IMAGE in "${GHCR_IMAGE}" "${DOCKERHUB_IMAGE}"; do
+                    for IMAGE_TAG in "${IMAGE_TAGS[@]}"; do
+                      echo "Processing ${BASE_IMAGE}:${IMAGE_TAG}"
+
+                      DIGEST="$(skopeo inspect --retry-times 3 docker://${BASE_IMAGE}:${IMAGE_TAG} | jq -r '.Digest')"
+                      REF="${BASE_IMAGE}@${DIGEST}"
                       echo "Resolved digest: ${REF}"
 
                       echo "==> cosign sign (keyless) --recursive ${REF}"
@@ -392,7 +490,12 @@ jobs:
                         --certificate-oidc-issuer "${issuer}" \
                         --certificate-identity-regexp "${id_regex}" \
                         "${REF}" -o text
+                      
+                      echo "✓ Successfully signed and verified ${BASE_IMAGE}:${IMAGE_TAG}"
                     done
+                  done
+                  
+                  echo "All images signed and verified successfully!"
               shell: bash
 
     post-run:

.github/workflows/cicd.yml.backup

@@ -0,0 +1,426 @@
+name: CI/CD Pipeline
+
+# CI/CD workflow for building, publishing, mirroring, signing container images and building release binaries.
+# Actions are pinned to specific SHAs to reduce supply-chain risk. This workflow triggers on tag push events.
+
+permissions:
+  contents: read
+  packages: write      # for GHCR push
+  id-token: write      # for Cosign Keyless (OIDC) Signing
+
+# Required secrets:
+# - DOCKER_HUB_USERNAME / DOCKER_HUB_ACCESS_TOKEN: push to Docker Hub
+# - GITHUB_TOKEN: used for GHCR login and OIDC keyless signing
+# - COSIGN_PRIVATE_KEY / COSIGN_PASSWORD / COSIGN_PUBLIC_KEY: for key-based signing
+
+on:
+    push:
+        tags:
+            - "[0-9]+.[0-9]+.[0-9]+"
+            - "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+    pre-run:
+        runs-on: ubuntu-latest
+        permissions: write-all
+        steps:
+            - name: Configure AWS credentials
+              uses: aws-actions/configure-aws-credentials@v2
+              with:
+                  role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }}
+                  role-duration-seconds: 3600
+                  aws-region: ${{ secrets.AWS_REGION }}
+
+            - name: Verify AWS identity
+              run: aws sts get-caller-identity
+
+            - name: Start EC2 instances
+              run: |
+                  aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }}
+                  aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }}
+                  echo "EC2 instances started"
+
+
+    release-arm:
+        name: Build and Release (ARM64)
+        runs-on: [self-hosted, linux, arm64, us-east-1]
+        needs: [pre-run]
+        if: >-
+            ${{
+                needs.pre-run.result == 'success'
+            }}
+        # Job-level timeout to avoid runaway or stuck runs
+        timeout-minutes: 120
+        env:
+          # Target images
+          DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
+          GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}
+
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
+            - name: Monitor storage space
+              run: |
+                  THRESHOLD=75
+                  USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g')
+                  echo "Used space: $USED_SPACE%"
+                  if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then
+                      echo "Used space is below the threshold of 75% free. Running Docker system prune."
+                      echo y | docker system prune -a
+                  else
+                      echo "Storage space is above the threshold. No action needed."
+                  fi
+
+            - name: Log in to Docker Hub
+              uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+              with:
+                  registry: docker.io
+                  username: ${{ secrets.DOCKER_HUB_USERNAME }}
+                  password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+            - name: Extract tag name
+              id: get-tag
+              run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+              shell: bash
+
+            - name: Update version in package.json
+              run: |
+                  TAG=${{ env.TAG }}
+                  sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
+                  cat server/lib/consts.ts
+              shell: bash
+
+            - name: Check if release candidate
+              id: check-rc
+              run: |
+                  TAG=${{ env.TAG }}
+                  if [[ "$TAG" == *"-rc."* ]]; then
+                      echo "IS_RC=true" >> $GITHUB_ENV
+                  else
+                      echo "IS_RC=false" >> $GITHUB_ENV
+                  fi
+              shell: bash
+
+            - name: Build and push Docker images (Docker Hub - ARM64)
+              run: |
+                  TAG=${{ env.TAG }}
+                  if [ "$IS_RC" = "true" ]; then
+                      make build-rc-arm tag=$TAG
+                  else
+                      make build-release-arm tag=$TAG
+                  fi
+                  echo "Built & pushed ARM64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}"
+              shell: bash
+
+    release-amd:
+        name: Build and Release (AMD64)
+        runs-on: [self-hosted, linux, x64, us-east-1]
+        needs: [pre-run]
+        if: >-
+            ${{
+                needs.pre-run.result == 'success'
+            }}
+        # Job-level timeout to avoid runaway or stuck runs
+        timeout-minutes: 120
+        env:
+          # Target images
+          DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
+          GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}
+
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
+            - name: Monitor storage space
+              run: |
+                  THRESHOLD=75
+                  USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g')
+                  echo "Used space: $USED_SPACE%"
+                  if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then
+                      echo "Used space is below the threshold of 75% free. Running Docker system prune."
+                      echo y | docker system prune -a
+                  else
+                      echo "Storage space is above the threshold. No action needed."
+                  fi
+
+            - name: Log in to Docker Hub
+              uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+              with:
+                  registry: docker.io
+                  username: ${{ secrets.DOCKER_HUB_USERNAME }}
+                  password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+            - name: Extract tag name
+              id: get-tag
+              run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+              shell: bash
+
+            - name: Update version in package.json
+              run: |
+                  TAG=${{ env.TAG }}
+                  sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
+                  cat server/lib/consts.ts
+              shell: bash
+
+            - name: Check if release candidate
+              id: check-rc
+              run: |
+                  TAG=${{ env.TAG }}
+                  if [[ "$TAG" == *"-rc."* ]]; then
+                      echo "IS_RC=true" >> $GITHUB_ENV
+                  else
+                      echo "IS_RC=false" >> $GITHUB_ENV
+                  fi
+              shell: bash
+
+            - name: Build and push Docker images (Docker Hub - AMD64)
+              run: |
+                  TAG=${{ env.TAG }}
+                  if [ "$IS_RC" = "true" ]; then
+                      make build-rc-amd tag=$TAG
+                  else
+                      make build-release-amd tag=$TAG
+                  fi
+                  echo "Built & pushed AMD64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}"
+              shell: bash
+
+    create-manifest:
+        name: Create Multi-Arch Manifests
+        runs-on: [self-hosted, linux, x64, us-east-1]
+        needs: [release-arm, release-amd]
+        if: >-
+            ${{
+                needs.release-arm.result == 'success' &&
+                needs.release-amd.result == 'success'
+            }}
+        timeout-minutes: 30
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
+            - name: Log in to Docker Hub
+              uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+              with:
+                  registry: docker.io
+                  username: ${{ secrets.DOCKER_HUB_USERNAME }}
+                  password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+            - name: Extract tag name
+              id: get-tag
+              run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+              shell: bash
+
+            - name: Check if release candidate
+              id: check-rc
+              run: |
+                  TAG=${{ env.TAG }}
+                  if [[ "$TAG" == *"-rc."* ]]; then
+                      echo "IS_RC=true" >> $GITHUB_ENV
+                  else
+                      echo "IS_RC=false" >> $GITHUB_ENV
+                  fi
+              shell: bash
+
+            - name: Create multi-arch manifests
+              run: |
+                  TAG=${{ env.TAG }}
+                  if [ "$IS_RC" = "true" ]; then
+                      make create-manifests-rc tag=$TAG
+                  else
+                      make create-manifests tag=$TAG
+                  fi
+                  echo "Created multi-arch manifests for tag: ${TAG}"
+              shell: bash
+
+    sign-and-package:
+        name: Sign and Package
+        runs-on: [self-hosted, linux, x64, us-east-1]
+        needs: [release-arm, release-amd, create-manifest]
+        if: >-
+            ${{
+                needs.release-arm.result == 'success' &&
+                needs.release-amd.result == 'success' &&
+                needs.create-manifest.result == 'success'
+            }}
+        # Job-level timeout to avoid runaway or stuck runs
+        timeout-minutes: 120
+        env:
+          # Target images
+          DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
+          GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}
+
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
+            - name: Extract tag name
+              id: get-tag
+              run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+              shell: bash
+
+            - name: Install Go
+              uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
+              with:
+                  go-version: 1.24
+
+            - name: Update version in package.json
+              run: |
+                  TAG=${{ env.TAG }}
+                  sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
+                  cat server/lib/consts.ts
+              shell: bash
+
+            - name: Pull latest Gerbil version
+              id: get-gerbil-tag
+              run: |
+                  LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/gerbil/tags | jq -r '.[0].name')
+                  echo "LATEST_GERBIL_TAG=$LATEST_TAG" >> $GITHUB_ENV
+              shell: bash
+
+            - name: Pull latest Badger version
+              id: get-badger-tag
+              run: |
+                  LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/badger/tags | jq -r '.[0].name')
+                  echo "LATEST_BADGER_TAG=$LATEST_TAG" >> $GITHUB_ENV
+              shell: bash
+
+            - name: Update install/main.go
+              run: |
+                  PANGOLIN_VERSION=${{ env.TAG }}
+                  GERBIL_VERSION=${{ env.LATEST_GERBIL_TAG }}
+                  BADGER_VERSION=${{ env.LATEST_BADGER_TAG }}
+                  sed -i "s/config.PangolinVersion = \".*\"/config.PangolinVersion = \"$PANGOLIN_VERSION\"/" install/main.go
+                  sed -i "s/config.GerbilVersion = \".*\"/config.GerbilVersion = \"$GERBIL_VERSION\"/" install/main.go
+                  sed -i "s/config.BadgerVersion = \".*\"/config.BadgerVersion = \"$BADGER_VERSION\"/" install/main.go
+                  echo "Updated install/main.go with Pangolin version $PANGOLIN_VERSION, Gerbil version $GERBIL_VERSION, and Badger version $BADGER_VERSION"
+                  cat install/main.go
+              shell: bash
+
+            - name: Build installer
+              working-directory: install
+              run: |
+                  make go-build-release
+
+            - name: Upload artifacts from /install/bin
+              uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+              with:
+                  name: install-bin
+                  path: install/bin/
+
+            - name: Install skopeo + jq
+              # skopeo: copy/inspect images between registries
+              # jq: JSON parsing tool used to extract digest values
+              run: |
+                  sudo apt-get update -y
+                  sudo apt-get install -y skopeo jq
+                  skopeo --version
+              shell: bash
+
+            - name: Login to GHCR
+              env:
+                  REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json
+              run: |
+                  mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")"
+                  skopeo login ghcr.io -u "${{ github.actor }}" -p "${{ secrets.GITHUB_TOKEN }}"
+              shell: bash
+
+            - name: Copy tag from Docker Hub to GHCR
+              # Mirror the already-built image (all architectures) to GHCR so we can sign it
+              # Wait a bit for both architectures to be available in Docker Hub manifest
+              env:
+                  REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json
+              run: |
+                  set -euo pipefail
+                  TAG=${{ env.TAG }}
+                  echo "Waiting for multi-arch manifest to be ready..."
+                  sleep 30
+                  echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG} -> ${{ env.GHCR_IMAGE }}:${TAG}"
+                  skopeo copy --all --retry-times 3 \
+                    docker://$DOCKERHUB_IMAGE:$TAG \
+                    docker://$GHCR_IMAGE:$TAG
+              shell: bash
+
+            - name: Login to GitHub Container Registry (for cosign)
+              uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+              with:
+                registry: ghcr.io
+                username: ${{ github.actor }}
+                password: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Install cosign
+              # cosign is used to sign and verify container images (key and keyless)
+              uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0
+
+            - name: Dual-sign and verify (GHCR & Docker Hub)
+              # Sign each image by digest using keyless (OIDC) and key-based signing,
+              # then verify both the public key signature and the keyless OIDC signature.
+              env:
+                  TAG: ${{ env.TAG }}
+                  COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
+                  COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
+                  COSIGN_PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }}
+                  COSIGN_YES: "true"
+              run: |
+                  set -euo pipefail
+
+                  issuer="https://token.actions.githubusercontent.com"
+                  id_regex="^https://github.com/${{ github.repository }}/.+"  # accept this repo (all workflows/refs)
+
+                  for IMAGE in "${GHCR_IMAGE}" "${DOCKERHUB_IMAGE}"; do
+                    echo "Processing ${IMAGE}:${TAG}"
+
+                    DIGEST="$(skopeo inspect --retry-times 3 docker://${IMAGE}:${TAG} | jq -r '.Digest')"
+                    REF="${IMAGE}@${DIGEST}"
+                    echo "Resolved digest: ${REF}"
+
+                    echo "==> cosign sign (keyless) --recursive ${REF}"
+                    cosign sign --recursive "${REF}"
+
+                    echo "==> cosign sign (key) --recursive ${REF}"
+                    cosign sign --key env://COSIGN_PRIVATE_KEY --recursive "${REF}"
+
+                    echo "==> cosign verify (public key) ${REF}"
+                    cosign verify --key env://COSIGN_PUBLIC_KEY "${REF}" -o text
+
+                    echo "==> cosign verify (keyless policy) ${REF}"
+                    cosign verify \
+                      --certificate-oidc-issuer "${issuer}" \
+                      --certificate-identity-regexp "${id_regex}" \
+                      "${REF}" -o text
+                  done
+              shell: bash
+
+    post-run:
+        needs: [pre-run, release-arm, release-amd, create-manifest, sign-and-package]
+        if: >-
+            ${{
+                always() &&
+                needs.pre-run.result == 'success' &&
+                (needs.release-arm.result == 'success' || needs.release-arm.result == 'skipped' || needs.release-arm.result == 'failure') &&
+                (needs.release-amd.result == 'success' || needs.release-amd.result == 'skipped' || needs.release-amd.result == 'failure') &&
+                (needs.create-manifest.result == 'success' || needs.create-manifest.result == 'skipped' || needs.create-manifest.result == 'failure') &&
+                (needs.sign-and-package.result == 'success' || needs.sign-and-package.result == 'skipped' || needs.sign-and-package.result == 'failure')
+            }}
+        runs-on: ubuntu-latest
+        permissions: write-all
+        steps:
+            - name: Configure AWS credentials
+              uses: aws-actions/configure-aws-credentials@v2
+              with:
+                  role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }}
+                  role-duration-seconds: 3600
+                  aws-region: ${{ secrets.AWS_REGION }}
+
+            - name: Verify AWS identity
+              run: aws sts get-caller-identity
+
+            - name: Stop EC2 instances
+              run: |
+                  aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }}
+                  aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }}
+                  echo "EC2 instances stopped"

.github/workflows/saas.yml

@@ -0,0 +1,125 @@
+name: CI/CD Pipeline
+
+# CI/CD workflow for building, publishing, mirroring, signing container images and building release binaries.
+# Actions are pinned to specific SHAs to reduce supply-chain risk. This workflow triggers on tag push events.
+
+permissions:
+  contents: read
+  packages: write      # for GHCR push
+  id-token: write      # for Cosign Keyless (OIDC) Signing
+
+on:
+    push:
+        tags:
+            - "[0-9]+.[0-9]+.[0-9]+-s.[0-9]+"
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+    pre-run:
+        runs-on: ubuntu-latest
+        permissions: write-all
+        steps:
+            - name: Configure AWS credentials
+              uses: aws-actions/configure-aws-credentials@v2
+              with:
+                  role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }}
+                  role-duration-seconds: 3600
+                  aws-region: ${{ secrets.AWS_REGION }}
+
+            - name: Verify AWS identity
+              run: aws sts get-caller-identity
+
+            - name: Start EC2 instances
+              run: |
+                  aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }}
+                  echo "EC2 instances started"
+
+
+    release-arm:
+        name: Build and Release (ARM64)
+        runs-on: [self-hosted, linux, arm64, us-east-1]
+        needs: [pre-run]
+        if: >-
+            ${{
+                needs.pre-run.result == 'success'
+            }}
+        # Job-level timeout to avoid runaway or stuck runs
+        timeout-minutes: 120
+        env:
+          # Target images
+          AWS_IMAGE: ${{ secrets.aws_account_id }}.dkr.ecr.us-east-1.amazonaws.com/${{ github.event.repository.name }}
+
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
+            - name: Monitor storage space
+              run: |
+                  THRESHOLD=75
+                  USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g')
+                  echo "Used space: $USED_SPACE%"
+                  if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then
+                      echo "Used space is below the threshold of 75% free. Running Docker system prune."
+                      echo y | docker system prune -a
+                  else
+                      echo "Storage space is above the threshold. No action needed."
+                  fi
+
+            - name: Configure AWS credentials
+              uses: aws-actions/configure-aws-credentials@v2
+              with:
+                  role-to-assume: arn:aws:iam::${{ secrets.aws_account_id }}:role/${{ secrets.AWS_ROLE_NAME }}
+                  role-duration-seconds: 3600
+                  aws-region: ${{ secrets.AWS_REGION }}
+
+            - name: Login to Amazon ECR
+              id: login-ecr
+              uses: aws-actions/amazon-ecr-login@v2
+
+            - name: Extract tag name
+              id: get-tag
+              run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+              shell: bash
+
+            - name: Update version in package.json
+              run: |
+                  TAG=${{ env.TAG }}
+                  sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
+                  cat server/lib/consts.ts
+              shell: bash
+
+            - name: Build and push Docker images (Docker Hub - ARM64)
+              run: |
+                  TAG=${{ env.TAG }}
+                  make build-saas tag=$TAG
+                  echo "Built & pushed ARM64 images to: ${{ env.AWS_IMAGE }}:${TAG}"
+              shell: bash
+
+    post-run:
+        needs: [pre-run, release-arm]
+        if: >-
+            ${{
+                always() &&
+                needs.pre-run.result == 'success' &&
+                (needs.release-arm.result == 'success' || needs.release-arm.result == 'skipped' || needs.release-arm.result == 'failure')
+            }}
+        runs-on: ubuntu-latest
+        permissions: write-all
+        steps:
+            - name: Configure AWS credentials
+              uses: aws-actions/configure-aws-credentials@v2
+              with:
+                  role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }}
+                  role-duration-seconds: 3600
+                  aws-region: ${{ secrets.AWS_REGION }}
+
+            - name: Verify AWS identity
+              run: aws sts get-caller-identity
+
+            - name: Stop EC2 instances
+              run: |
+                  aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }}
+                  echo "EC2 instances stopped"

Makefile

@@ -90,6 +90,18 @@ build-ee-postgresql:
 		--tag fosrl/pangolin:ee-postgresql-$(tag) \
 		--push .
 
+build-saas:
+	@if [ -z "$(tag)" ]; then \
+		echo "Error: tag is required. Usage: make build-release tag=<tag>"; \
+		exit 1; \
+	fi
+	docker buildx build \
+		--build-arg BUILD=saas \
+		--build-arg DATABASE=pg \
+		--platform linux/arm64 \
+		--tag $(AWS_IMAGE):$(tag) \
+		--push .
+
 build-release-arm:
 	@if [ -z "$(tag)" ]; then \
 		echo "Error: tag is required. Usage: make build-release-arm tag=<tag>"; \

install/main.go

@@ -340,7 +340,7 @@ func collectUserInput(reader *bufio.Reader) Config {
 	// Basic configuration
 	fmt.Println("\n=== Basic Configuration ===")
 
-	config.IsEnterprise = readBoolNoDefault(reader, "Do you want to install the Enterprise version of Pangolin? The EE is free for persoal use or for businesses making less than 100k USD annually.")
+	config.IsEnterprise = readBoolNoDefault(reader, "Do you want to install the Enterprise version of Pangolin? The EE is free for personal use or for businesses making less than 100k USD annually.")
 
 	config.BaseDomain = readString(reader, "Enter your base domain (no subdomain e.g. example.com)", "")
 

messages/en-US.json

@@ -2244,7 +2244,7 @@
     "deviceOrganizationsAccess": "Access to all organizations your account has access to",
     "deviceAuthorize": "Authorize {applicationName}",
     "deviceConnected": "Device Connected!",
-    "deviceAuthorizedMessage": "Device is authorized to access your account.",
+    "deviceAuthorizedMessage": "Device is authorized to access your account. Please return to the client application.",
     "pangolinCloud": "Pangolin Cloud",
     "viewDevices": "View Devices",
     "viewDevicesDescription": "Manage your connected devices",

server/lib/blueprints/types.ts

@@ -290,8 +290,8 @@ export const ClientResourceSchema = z
         alias: z
             .string()
             .regex(
-                /^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/,
+                /^(?:[a-zA-Z0-9*?](?:[a-zA-Z0-9*?-]{0,61}[a-zA-Z0-9*?])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/,
-                "Alias must be a fully qualified domain name (e.g., example.com)"
+                "Alias must be a fully qualified domain name with optional wildcards (e.g., example.com, *.example.com, host-0?.example.internal)"
             )
             .optional(),
         roles: z

server/private/lib/exitNodes/exitNodes.ts

@@ -288,7 +288,7 @@ export function selectBestExitNode(
     const validNodes = pingResults.filter((n) => !n.error && n.weight > 0);
 
     if (validNodes.length === 0) {
-        logger.error("No valid exit nodes available");
+        logger.debug("No valid exit nodes available");
         return null;
     }
 

server/private/lib/lock.ts

@@ -24,7 +24,9 @@ export class LockManager {
      */
     async acquireLock(
         lockKey: string,
-        ttlMs: number = 30000
+        ttlMs: number = 30000,
+        maxRetries: number = 3,
+        retryDelayMs: number = 100
     ): Promise<boolean> {
         if (!redis || !redis.status || redis.status !== "ready") {
             return true;
@@ -35,6 +37,7 @@ export class LockManager {
         }:${Date.now()}`;
         const redisKey = `lock:${lockKey}`;
 
+        for (let attempt = 0; attempt < maxRetries; attempt++) {
             try {
                 // Use SET with NX (only set if not exists) and PX (expire in milliseconds)
                 // This is atomic and handles both setting and expiration
@@ -73,11 +76,28 @@ export class LockManager {
                     return true;
                 }
 
-            return false;
+                // If this isn't our last attempt, wait before retrying with exponential backoff
-        } catch (error) {
+                if (attempt < maxRetries - 1) {
-            logger.error(`Failed to acquire lock ${lockKey}:`, error);
+                    const delay = retryDelayMs * Math.pow(2, attempt);
-            return false;
+                    logger.debug(
+                        `Lock ${lockKey} not available, retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries})`
+                    );
+                    await new Promise((resolve) => setTimeout(resolve, delay));
                 }
+            } catch (error) {
+                logger.error(`Failed to acquire lock ${lockKey} (attempt ${attempt + 1}/${maxRetries}):`, error);
+                // On error, still retry if we have attempts left
+                if (attempt < maxRetries - 1) {
+                    const delay = retryDelayMs * Math.pow(2, attempt);
+                    await new Promise((resolve) => setTimeout(resolve, delay));
+                }
+            }
+        }
+
+        logger.debug(
+            `Failed to acquire lock ${lockKey} after ${maxRetries} attempts`
+        );
+        return false;
     }
 
     /**

server/private/lib/traefik/getTraefikConfig.ts

@@ -456,11 +456,11 @@ export async function getTraefikConfig(
                     // );
                 } else if (resource.maintenanceModeType === "automatic") {
                     showMaintenancePage = !hasHealthyServers;
-                    if (showMaintenancePage) {
+                    // if (showMaintenancePage) {
-                        logger.warn(
+                    //     logger.warn(
-                            `Resource ${resource.name} (${fullDomain}) has no healthy servers - showing maintenance page (AUTOMATIC mode)`
+                    //         `Resource ${resource.name} (${fullDomain}) has no healthy servers - showing maintenance page (AUTOMATIC mode)`
-                        );
+                    //     );
-                    }
+                    // }
                 }
             }
 

server/private/middlewares/verifySubscription.ts

@@ -27,7 +27,18 @@ export async function verifyValidSubscription(
             return next();
         }
 
-        const tier = await getOrgTierData(req.params.orgId);
+        const orgId = req.params.orgId || req.body.orgId || req.query.orgId || req.userOrgId;
+
+        if (!orgId) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    "Organization ID is required to verify subscription"
+                )
+            );
+        }
+
+        const tier = await getOrgTierData(orgId);
 
         if (!tier.active) {
             return next(

server/private/routers/external.ts

@@ -436,18 +436,18 @@ authenticated.get(
 
 authenticated.post(
     "/re-key/:clientId/regenerate-client-secret",
+    verifyClientAccess, // this is first to set the org id
     verifyValidLicense,
     verifyValidSubscription,
-    verifyClientAccess,
     verifyUserHasAction(ActionsEnum.reGenerateSecret),
     reKey.reGenerateClientSecret
 );
 
 authenticated.post(
     "/re-key/:siteId/regenerate-site-secret",
+    verifySiteAccess, // this is first to set the org id
     verifyValidLicense,
     verifyValidSubscription,
-    verifySiteAccess,
     verifyUserHasAction(ActionsEnum.reGenerateSecret),
     reKey.reGenerateSiteSecret
 );

src/app/auth/login/device/success/page.tsx

@@ -7,6 +7,7 @@ import { useLicenseStatusContext } from "@app/hooks/useLicenseStatusContext";
 import { CheckCircle2 } from "lucide-react";
 import { useTranslations } from "next-intl";
 import Link from "next/link";
+import { useEffect } from "react";
 
 export default function DeviceAuthSuccessPage() {
     const { env } = useEnvContext();
@@ -20,6 +21,32 @@ export default function DeviceAuthSuccessPage() {
         ? env.branding.logo?.authPage?.height || 58
         : 58;
 
+    useEffect(() => {
+        // Detect if we're on iOS or Android
+        const userAgent = navigator.userAgent || navigator.vendor || (window as any).opera;
+        const isIOS = /iPad|iPhone|iPod/.test(userAgent) && !(window as any).MSStream;
+        const isAndroid = /android/i.test(userAgent);
+
+        if (isAndroid) {
+            // For Android Chrome Custom Tabs, use intent:// scheme which works more reliably
+            // This explicitly tells Chrome to send an intent to the app, which will bring
+            // SignInCodeActivity back to the foreground (it has launchMode="singleTop")
+            setTimeout(() => {
+                window.location.href = "intent://auth-success#Intent;scheme=pangolin;package=net.pangolin.Pangolin;end";
+            }, 500);
+        } else if (isIOS) {
+            // Wait 500ms then attempt to open the app
+            setTimeout(() => {
+                // Try to open the app using deep link
+                window.location.href = "pangolin://";
+
+                setTimeout(() => {
+                    window.location.href = "https://apps.apple.com/app/pangolin/net.pangolin.Pangolin.PangoliniOS";
+                }, 2000);
+            }, 500);
+        }
+    }, []);
+
     return (
         <>
             <Card>

src/components/ConfirmDeleteDialog.tsx

@@ -63,6 +63,8 @@ export default function ConfirmDeleteDialog({
         }
     });
 
+    const isConfirmed = form.watch("string") === string;
+
     async function onSubmit() {
         try {
             await onConfirm();
@@ -139,7 +141,8 @@ export default function ConfirmDeleteDialog({
                             type="submit"
                             form="confirm-delete-form"
                             loading={loading}
-                            disabled={loading}
+                            disabled={loading || !isConfirmed}
+                            className={!isConfirmed && !loading ? "opacity-50" : ""}
                         >
                             {buttonText}
                         </Button>