diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 09e406ad..38f55482 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -329,20 +329,89 @@ jobs: skopeo login ghcr.io -u "${{ github.actor }}" -p "${{ secrets.GITHUB_TOKEN }}" shell: bash - - name: Copy tag from Docker Hub to GHCR - # Mirror the already-built image (all architectures) to GHCR so we can sign it + - name: Copy tags from Docker Hub to GHCR + # Mirror the already-built images (all architectures) to GHCR so we can sign them # Wait a bit for both architectures to be available in Docker Hub manifest env: REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json run: | set -euo pipefail TAG=${{ env.TAG }} - echo "Waiting for multi-arch manifest to be ready..." + MAJOR_TAG=$(echo $TAG | cut -d. -f1) + MINOR_TAG=$(echo $TAG | cut -d. -f1,2) + + echo "Waiting for multi-arch manifests to be ready..." sleep 30 - echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG} -> ${{ env.GHCR_IMAGE }}:${TAG}" - skopeo copy --all --retry-times 3 \ - docker://$DOCKERHUB_IMAGE:$TAG \ - docker://$GHCR_IMAGE:$TAG + + # Determine if this is an RC release + IS_RC="false" + if echo "$TAG" | grep -qE "rc[0-9]+$"; then + IS_RC="true" + fi + + if [ "$IS_RC" = "true" ]; then + echo "RC release detected - copying version-specific tags only" + + # SQLite OSS + echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG} -> ${{ env.GHCR_IMAGE }}:${TAG}" + skopeo copy --all --retry-times 3 \ + docker://$DOCKERHUB_IMAGE:$TAG \ + docker://$GHCR_IMAGE:$TAG + + # PostgreSQL OSS + echo "Copying ${{ env.DOCKERHUB_IMAGE }}:postgresql-${TAG} -> ${{ env.GHCR_IMAGE }}:postgresql-${TAG}" + skopeo copy --all --retry-times 3 \ + docker://$DOCKERHUB_IMAGE:postgresql-$TAG \ + docker://$GHCR_IMAGE:postgresql-$TAG + + # SQLite Enterprise + echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-${TAG} -> ${{ env.GHCR_IMAGE }}:ee-${TAG}" + skopeo copy --all --retry-times 3 \ + docker://$DOCKERHUB_IMAGE:ee-$TAG \ + docker://$GHCR_IMAGE:ee-$TAG + + # PostgreSQL Enterprise + echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-postgresql-${TAG} -> ${{ env.GHCR_IMAGE }}:ee-postgresql-${TAG}" + skopeo copy --all --retry-times 3 \ + docker://$DOCKERHUB_IMAGE:ee-postgresql-$TAG \ + docker://$GHCR_IMAGE:ee-postgresql-$TAG + else + echo "Regular release detected - copying all tags (latest, major, minor, full version)" + + # SQLite OSS - all tags + for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do + echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:${TAG_SUFFIX}" + skopeo copy --all --retry-times 3 \ + docker://$DOCKERHUB_IMAGE:$TAG_SUFFIX \ + docker://$GHCR_IMAGE:$TAG_SUFFIX + done + + # PostgreSQL OSS - all tags + for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do + echo "Copying ${{ env.DOCKERHUB_IMAGE }}:postgresql-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:postgresql-${TAG_SUFFIX}" + skopeo copy --all --retry-times 3 \ + docker://$DOCKERHUB_IMAGE:postgresql-$TAG_SUFFIX \ + docker://$GHCR_IMAGE:postgresql-$TAG_SUFFIX + done + + # SQLite Enterprise - all tags + for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do + echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:ee-${TAG_SUFFIX}" + skopeo copy --all --retry-times 3 \ + docker://$DOCKERHUB_IMAGE:ee-$TAG_SUFFIX \ + docker://$GHCR_IMAGE:ee-$TAG_SUFFIX + done + + # PostgreSQL Enterprise - all tags + for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do + echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-postgresql-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:ee-postgresql-${TAG_SUFFIX}" + skopeo copy --all --retry-times 3 \ + docker://$DOCKERHUB_IMAGE:ee-postgresql-$TAG_SUFFIX \ + docker://$GHCR_IMAGE:ee-postgresql-$TAG_SUFFIX + done + fi + + echo "All images copied successfully to GHCR!" shell: bash - name: Login to GitHub Container Registry (for cosign) @@ -371,28 +440,62 @@ jobs: issuer="https://token.actions.githubusercontent.com" id_regex="^https://github.com/${{ github.repository }}/.+" # accept this repo (all workflows/refs) - for IMAGE in "${GHCR_IMAGE}" "${DOCKERHUB_IMAGE}"; do - echo "Processing ${IMAGE}:${TAG}" + # Determine if this is an RC release + IS_RC="false" + if echo "$TAG" | grep -qE "rc[0-9]+$"; then + IS_RC="true" + fi - DIGEST="$(skopeo inspect --retry-times 3 docker://${IMAGE}:${TAG} | jq -r '.Digest')" - REF="${IMAGE}@${DIGEST}" - echo "Resolved digest: ${REF}" + # Define image variants to sign + if [ "$IS_RC" = "true" ]; then + echo "RC release - signing version-specific tags only" + IMAGE_TAGS=( + "${TAG}" + "postgresql-${TAG}" + "ee-${TAG}" + "ee-postgresql-${TAG}" + ) + else + echo "Regular release - signing all tags" + MAJOR_TAG=$(echo $TAG | cut -d. -f1) + MINOR_TAG=$(echo $TAG | cut -d. -f1,2) + IMAGE_TAGS=( + "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG" + "postgresql-latest" "postgresql-$MAJOR_TAG" "postgresql-$MINOR_TAG" "postgresql-$TAG" + "ee-latest" "ee-$MAJOR_TAG" "ee-$MINOR_TAG" "ee-$TAG" + "ee-postgresql-latest" "ee-postgresql-$MAJOR_TAG" "ee-postgresql-$MINOR_TAG" "ee-postgresql-$TAG" + ) + fi - echo "==> cosign sign (keyless) --recursive ${REF}" - cosign sign --recursive "${REF}" + # Sign each image variant for both registries + for BASE_IMAGE in "${GHCR_IMAGE}" "${DOCKERHUB_IMAGE}"; do + for IMAGE_TAG in "${IMAGE_TAGS[@]}"; do + echo "Processing ${BASE_IMAGE}:${IMAGE_TAG}" - echo "==> cosign sign (key) --recursive ${REF}" - cosign sign --key env://COSIGN_PRIVATE_KEY --recursive "${REF}" + DIGEST="$(skopeo inspect --retry-times 3 docker://${BASE_IMAGE}:${IMAGE_TAG} | jq -r '.Digest')" + REF="${BASE_IMAGE}@${DIGEST}" + echo "Resolved digest: ${REF}" - echo "==> cosign verify (public key) ${REF}" - cosign verify --key env://COSIGN_PUBLIC_KEY "${REF}" -o text + echo "==> cosign sign (keyless) --recursive ${REF}" + cosign sign --recursive "${REF}" - echo "==> cosign verify (keyless policy) ${REF}" - cosign verify \ - --certificate-oidc-issuer "${issuer}" \ - --certificate-identity-regexp "${id_regex}" \ - "${REF}" -o text + echo "==> cosign sign (key) --recursive ${REF}" + cosign sign --key env://COSIGN_PRIVATE_KEY --recursive "${REF}" + + echo "==> cosign verify (public key) ${REF}" + cosign verify --key env://COSIGN_PUBLIC_KEY "${REF}" -o text + + echo "==> cosign verify (keyless policy) ${REF}" + cosign verify \ + --certificate-oidc-issuer "${issuer}" \ + --certificate-identity-regexp "${id_regex}" \ + "${REF}" -o text + + echo "✓ Successfully signed and verified ${BASE_IMAGE}:${IMAGE_TAG}" + done done + + echo "All images signed and verified successfully!" shell: bash post-run: diff --git a/.github/workflows/cicd.yml.backup b/.github/workflows/cicd.yml.backup new file mode 100644 index 00000000..09e406ad --- /dev/null +++ b/.github/workflows/cicd.yml.backup @@ -0,0 +1,426 @@ +name: CI/CD Pipeline + +# CI/CD workflow for building, publishing, mirroring, signing container images and building release binaries. +# Actions are pinned to specific SHAs to reduce supply-chain risk. This workflow triggers on tag push events. + +permissions: + contents: read + packages: write # for GHCR push + id-token: write # for Cosign Keyless (OIDC) Signing + +# Required secrets: +# - DOCKER_HUB_USERNAME / DOCKER_HUB_ACCESS_TOKEN: push to Docker Hub +# - GITHUB_TOKEN: used for GHCR login and OIDC keyless signing +# - COSIGN_PRIVATE_KEY / COSIGN_PASSWORD / COSIGN_PUBLIC_KEY: for key-based signing + +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+" + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + pre-run: + runs-on: ubuntu-latest + permissions: write-all + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} + role-duration-seconds: 3600 + aws-region: ${{ secrets.AWS_REGION }} + + - name: Verify AWS identity + run: aws sts get-caller-identity + + - name: Start EC2 instances + run: | + aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }} + aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }} + echo "EC2 instances started" + + + release-arm: + name: Build and Release (ARM64) + runs-on: [self-hosted, linux, arm64, us-east-1] + needs: [pre-run] + if: >- + ${{ + needs.pre-run.result == 'success' + }} + # Job-level timeout to avoid runaway or stuck runs + timeout-minutes: 120 + env: + # Target images + DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + + steps: + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Monitor storage space + run: | + THRESHOLD=75 + USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g') + echo "Used space: $USED_SPACE%" + if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then + echo "Used space is below the threshold of 75% free. Running Docker system prune." + echo y | docker system prune -a + else + echo "Storage space is above the threshold. No action needed." + fi + + - name: Log in to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: docker.io + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Extract tag name + id: get-tag + run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + shell: bash + + - name: Update version in package.json + run: | + TAG=${{ env.TAG }} + sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts + cat server/lib/consts.ts + shell: bash + + - name: Check if release candidate + id: check-rc + run: | + TAG=${{ env.TAG }} + if [[ "$TAG" == *"-rc."* ]]; then + echo "IS_RC=true" >> $GITHUB_ENV + else + echo "IS_RC=false" >> $GITHUB_ENV + fi + shell: bash + + - name: Build and push Docker images (Docker Hub - ARM64) + run: | + TAG=${{ env.TAG }} + if [ "$IS_RC" = "true" ]; then + make build-rc-arm tag=$TAG + else + make build-release-arm tag=$TAG + fi + echo "Built & pushed ARM64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}" + shell: bash + + release-amd: + name: Build and Release (AMD64) + runs-on: [self-hosted, linux, x64, us-east-1] + needs: [pre-run] + if: >- + ${{ + needs.pre-run.result == 'success' + }} + # Job-level timeout to avoid runaway or stuck runs + timeout-minutes: 120 + env: + # Target images + DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + + steps: + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Monitor storage space + run: | + THRESHOLD=75 + USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g') + echo "Used space: $USED_SPACE%" + if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then + echo "Used space is below the threshold of 75% free. Running Docker system prune." + echo y | docker system prune -a + else + echo "Storage space is above the threshold. No action needed." + fi + + - name: Log in to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: docker.io + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Extract tag name + id: get-tag + run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + shell: bash + + - name: Update version in package.json + run: | + TAG=${{ env.TAG }} + sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts + cat server/lib/consts.ts + shell: bash + + - name: Check if release candidate + id: check-rc + run: | + TAG=${{ env.TAG }} + if [[ "$TAG" == *"-rc."* ]]; then + echo "IS_RC=true" >> $GITHUB_ENV + else + echo "IS_RC=false" >> $GITHUB_ENV + fi + shell: bash + + - name: Build and push Docker images (Docker Hub - AMD64) + run: | + TAG=${{ env.TAG }} + if [ "$IS_RC" = "true" ]; then + make build-rc-amd tag=$TAG + else + make build-release-amd tag=$TAG + fi + echo "Built & pushed AMD64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}" + shell: bash + + create-manifest: + name: Create Multi-Arch Manifests + runs-on: [self-hosted, linux, x64, us-east-1] + needs: [release-arm, release-amd] + if: >- + ${{ + needs.release-arm.result == 'success' && + needs.release-amd.result == 'success' + }} + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Log in to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: docker.io + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Extract tag name + id: get-tag + run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + shell: bash + + - name: Check if release candidate + id: check-rc + run: | + TAG=${{ env.TAG }} + if [[ "$TAG" == *"-rc."* ]]; then + echo "IS_RC=true" >> $GITHUB_ENV + else + echo "IS_RC=false" >> $GITHUB_ENV + fi + shell: bash + + - name: Create multi-arch manifests + run: | + TAG=${{ env.TAG }} + if [ "$IS_RC" = "true" ]; then + make create-manifests-rc tag=$TAG + else + make create-manifests tag=$TAG + fi + echo "Created multi-arch manifests for tag: ${TAG}" + shell: bash + + sign-and-package: + name: Sign and Package + runs-on: [self-hosted, linux, x64, us-east-1] + needs: [release-arm, release-amd, create-manifest] + if: >- + ${{ + needs.release-arm.result == 'success' && + needs.release-amd.result == 'success' && + needs.create-manifest.result == 'success' + }} + # Job-level timeout to avoid runaway or stuck runs + timeout-minutes: 120 + env: + # Target images + DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + + steps: + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Extract tag name + id: get-tag + run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + shell: bash + + - name: Install Go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 + with: + go-version: 1.24 + + - name: Update version in package.json + run: | + TAG=${{ env.TAG }} + sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts + cat server/lib/consts.ts + shell: bash + + - name: Pull latest Gerbil version + id: get-gerbil-tag + run: | + LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/gerbil/tags | jq -r '.[0].name') + echo "LATEST_GERBIL_TAG=$LATEST_TAG" >> $GITHUB_ENV + shell: bash + + - name: Pull latest Badger version + id: get-badger-tag + run: | + LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/badger/tags | jq -r '.[0].name') + echo "LATEST_BADGER_TAG=$LATEST_TAG" >> $GITHUB_ENV + shell: bash + + - name: Update install/main.go + run: | + PANGOLIN_VERSION=${{ env.TAG }} + GERBIL_VERSION=${{ env.LATEST_GERBIL_TAG }} + BADGER_VERSION=${{ env.LATEST_BADGER_TAG }} + sed -i "s/config.PangolinVersion = \".*\"/config.PangolinVersion = \"$PANGOLIN_VERSION\"/" install/main.go + sed -i "s/config.GerbilVersion = \".*\"/config.GerbilVersion = \"$GERBIL_VERSION\"/" install/main.go + sed -i "s/config.BadgerVersion = \".*\"/config.BadgerVersion = \"$BADGER_VERSION\"/" install/main.go + echo "Updated install/main.go with Pangolin version $PANGOLIN_VERSION, Gerbil version $GERBIL_VERSION, and Badger version $BADGER_VERSION" + cat install/main.go + shell: bash + + - name: Build installer + working-directory: install + run: | + make go-build-release + + - name: Upload artifacts from /install/bin + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: install-bin + path: install/bin/ + + - name: Install skopeo + jq + # skopeo: copy/inspect images between registries + # jq: JSON parsing tool used to extract digest values + run: | + sudo apt-get update -y + sudo apt-get install -y skopeo jq + skopeo --version + shell: bash + + - name: Login to GHCR + env: + REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json + run: | + mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")" + skopeo login ghcr.io -u "${{ github.actor }}" -p "${{ secrets.GITHUB_TOKEN }}" + shell: bash + + - name: Copy tag from Docker Hub to GHCR + # Mirror the already-built image (all architectures) to GHCR so we can sign it + # Wait a bit for both architectures to be available in Docker Hub manifest + env: + REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json + run: | + set -euo pipefail + TAG=${{ env.TAG }} + echo "Waiting for multi-arch manifest to be ready..." + sleep 30 + echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG} -> ${{ env.GHCR_IMAGE }}:${TAG}" + skopeo copy --all --retry-times 3 \ + docker://$DOCKERHUB_IMAGE:$TAG \ + docker://$GHCR_IMAGE:$TAG + shell: bash + + - name: Login to GitHub Container Registry (for cosign) + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Install cosign + # cosign is used to sign and verify container images (key and keyless) + uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0 + + - name: Dual-sign and verify (GHCR & Docker Hub) + # Sign each image by digest using keyless (OIDC) and key-based signing, + # then verify both the public key signature and the keyless OIDC signature. + env: + TAG: ${{ env.TAG }} + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} + COSIGN_PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }} + COSIGN_YES: "true" + run: | + set -euo pipefail + + issuer="https://token.actions.githubusercontent.com" + id_regex="^https://github.com/${{ github.repository }}/.+" # accept this repo (all workflows/refs) + + for IMAGE in "${GHCR_IMAGE}" "${DOCKERHUB_IMAGE}"; do + echo "Processing ${IMAGE}:${TAG}" + + DIGEST="$(skopeo inspect --retry-times 3 docker://${IMAGE}:${TAG} | jq -r '.Digest')" + REF="${IMAGE}@${DIGEST}" + echo "Resolved digest: ${REF}" + + echo "==> cosign sign (keyless) --recursive ${REF}" + cosign sign --recursive "${REF}" + + echo "==> cosign sign (key) --recursive ${REF}" + cosign sign --key env://COSIGN_PRIVATE_KEY --recursive "${REF}" + + echo "==> cosign verify (public key) ${REF}" + cosign verify --key env://COSIGN_PUBLIC_KEY "${REF}" -o text + + echo "==> cosign verify (keyless policy) ${REF}" + cosign verify \ + --certificate-oidc-issuer "${issuer}" \ + --certificate-identity-regexp "${id_regex}" \ + "${REF}" -o text + done + shell: bash + + post-run: + needs: [pre-run, release-arm, release-amd, create-manifest, sign-and-package] + if: >- + ${{ + always() && + needs.pre-run.result == 'success' && + (needs.release-arm.result == 'success' || needs.release-arm.result == 'skipped' || needs.release-arm.result == 'failure') && + (needs.release-amd.result == 'success' || needs.release-amd.result == 'skipped' || needs.release-amd.result == 'failure') && + (needs.create-manifest.result == 'success' || needs.create-manifest.result == 'skipped' || needs.create-manifest.result == 'failure') && + (needs.sign-and-package.result == 'success' || needs.sign-and-package.result == 'skipped' || needs.sign-and-package.result == 'failure') + }} + runs-on: ubuntu-latest + permissions: write-all + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} + role-duration-seconds: 3600 + aws-region: ${{ secrets.AWS_REGION }} + + - name: Verify AWS identity + run: aws sts get-caller-identity + + - name: Stop EC2 instances + run: | + aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }} + aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }} + echo "EC2 instances stopped" diff --git a/.github/workflows/saas.yml b/.github/workflows/saas.yml new file mode 100644 index 00000000..0c36de25 --- /dev/null +++ b/.github/workflows/saas.yml @@ -0,0 +1,125 @@ +name: CI/CD Pipeline + +# CI/CD workflow for building, publishing, mirroring, signing container images and building release binaries. +# Actions are pinned to specific SHAs to reduce supply-chain risk. This workflow triggers on tag push events. + +permissions: + contents: read + packages: write # for GHCR push + id-token: write # for Cosign Keyless (OIDC) Signing + +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+-s.[0-9]+" + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + pre-run: + runs-on: ubuntu-latest + permissions: write-all + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} + role-duration-seconds: 3600 + aws-region: ${{ secrets.AWS_REGION }} + + - name: Verify AWS identity + run: aws sts get-caller-identity + + - name: Start EC2 instances + run: | + aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }} + echo "EC2 instances started" + + + release-arm: + name: Build and Release (ARM64) + runs-on: [self-hosted, linux, arm64, us-east-1] + needs: [pre-run] + if: >- + ${{ + needs.pre-run.result == 'success' + }} + # Job-level timeout to avoid runaway or stuck runs + timeout-minutes: 120 + env: + # Target images + AWS_IMAGE: ${{ secrets.aws_account_id }}.dkr.ecr.us-east-1.amazonaws.com/${{ github.event.repository.name }} + + steps: + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Monitor storage space + run: | + THRESHOLD=75 + USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g') + echo "Used space: $USED_SPACE%" + if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then + echo "Used space is below the threshold of 75% free. Running Docker system prune." + echo y | docker system prune -a + else + echo "Storage space is above the threshold. No action needed." + fi + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.aws_account_id }}:role/${{ secrets.AWS_ROLE_NAME }} + role-duration-seconds: 3600 + aws-region: ${{ secrets.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Extract tag name + id: get-tag + run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + shell: bash + + - name: Update version in package.json + run: | + TAG=${{ env.TAG }} + sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts + cat server/lib/consts.ts + shell: bash + + - name: Build and push Docker images (Docker Hub - ARM64) + run: | + TAG=${{ env.TAG }} + make build-saas tag=$TAG + echo "Built & pushed ARM64 images to: ${{ env.AWS_IMAGE }}:${TAG}" + shell: bash + + post-run: + needs: [pre-run, release-arm] + if: >- + ${{ + always() && + needs.pre-run.result == 'success' && + (needs.release-arm.result == 'success' || needs.release-arm.result == 'skipped' || needs.release-arm.result == 'failure') + }} + runs-on: ubuntu-latest + permissions: write-all + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} + role-duration-seconds: 3600 + aws-region: ${{ secrets.AWS_REGION }} + + - name: Verify AWS identity + run: aws sts get-caller-identity + + - name: Stop EC2 instances + run: | + aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }} + echo "EC2 instances stopped" diff --git a/Makefile b/Makefile index 13dc601f..da31b6e2 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,18 @@ build-ee-postgresql: --tag fosrl/pangolin:ee-postgresql-$(tag) \ --push . +build-saas: + @if [ -z "$(tag)" ]; then \ + echo "Error: tag is required. Usage: make build-release tag="; \ + exit 1; \ + fi + docker buildx build \ + --build-arg BUILD=saas \ + --build-arg DATABASE=pg \ + --platform linux/arm64 \ + --tag $(AWS_IMAGE):$(tag) \ + --push . + build-release-arm: @if [ -z "$(tag)" ]; then \ echo "Error: tag is required. Usage: make build-release-arm tag="; \ diff --git a/install/main.go b/install/main.go index de001df2..a231da2d 100644 --- a/install/main.go +++ b/install/main.go @@ -340,7 +340,7 @@ func collectUserInput(reader *bufio.Reader) Config { // Basic configuration fmt.Println("\n=== Basic Configuration ===") - config.IsEnterprise = readBoolNoDefault(reader, "Do you want to install the Enterprise version of Pangolin? The EE is free for persoal use or for businesses making less than 100k USD annually.") + config.IsEnterprise = readBoolNoDefault(reader, "Do you want to install the Enterprise version of Pangolin? The EE is free for personal use or for businesses making less than 100k USD annually.") config.BaseDomain = readString(reader, "Enter your base domain (no subdomain e.g. example.com)", "") diff --git a/messages/en-US.json b/messages/en-US.json index 35b7e925..2aa312b7 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -2244,7 +2244,7 @@ "deviceOrganizationsAccess": "Access to all organizations your account has access to", "deviceAuthorize": "Authorize {applicationName}", "deviceConnected": "Device Connected!", - "deviceAuthorizedMessage": "Device is authorized to access your account.", + "deviceAuthorizedMessage": "Device is authorized to access your account. Please return to the client application.", "pangolinCloud": "Pangolin Cloud", "viewDevices": "View Devices", "viewDevicesDescription": "Manage your connected devices", diff --git a/server/lib/blueprints/types.ts b/server/lib/blueprints/types.ts index 650d5b18..cba9bfa7 100644 --- a/server/lib/blueprints/types.ts +++ b/server/lib/blueprints/types.ts @@ -290,8 +290,8 @@ export const ClientResourceSchema = z alias: z .string() .regex( - /^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/, - "Alias must be a fully qualified domain name (e.g., example.com)" + /^(?:[a-zA-Z0-9*?](?:[a-zA-Z0-9*?-]{0,61}[a-zA-Z0-9*?])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/, + "Alias must be a fully qualified domain name with optional wildcards (e.g., example.com, *.example.com, host-0?.example.internal)" ) .optional(), roles: z diff --git a/server/private/lib/exitNodes/exitNodes.ts b/server/private/lib/exitNodes/exitNodes.ts index 556fdcf7..97c89614 100644 --- a/server/private/lib/exitNodes/exitNodes.ts +++ b/server/private/lib/exitNodes/exitNodes.ts @@ -288,7 +288,7 @@ export function selectBestExitNode( const validNodes = pingResults.filter((n) => !n.error && n.weight > 0); if (validNodes.length === 0) { - logger.error("No valid exit nodes available"); + logger.debug("No valid exit nodes available"); return null; } diff --git a/server/private/lib/lock.ts b/server/private/lib/lock.ts index 08496f65..7e68565e 100644 --- a/server/private/lib/lock.ts +++ b/server/private/lib/lock.ts @@ -24,7 +24,9 @@ export class LockManager { */ async acquireLock( lockKey: string, - ttlMs: number = 30000 + ttlMs: number = 30000, + maxRetries: number = 3, + retryDelayMs: number = 100 ): Promise { if (!redis || !redis.status || redis.status !== "ready") { return true; @@ -35,49 +37,67 @@ export class LockManager { }:${Date.now()}`; const redisKey = `lock:${lockKey}`; - try { - // Use SET with NX (only set if not exists) and PX (expire in milliseconds) - // This is atomic and handles both setting and expiration - const result = await redis.set( - redisKey, - lockValue, - "PX", - ttlMs, - "NX" - ); - - if (result === "OK") { - logger.debug( - `Lock acquired: ${lockKey} by ${ - config.getRawConfig().gerbil.exit_node_name - }` + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + // Use SET with NX (only set if not exists) and PX (expire in milliseconds) + // This is atomic and handles both setting and expiration + const result = await redis.set( + redisKey, + lockValue, + "PX", + ttlMs, + "NX" ); - return true; - } - // Check if the existing lock is from this worker (reentrant behavior) - const existingValue = await redis.get(redisKey); - if ( - existingValue && - existingValue.startsWith( - `${config.getRawConfig().gerbil.exit_node_name}:` - ) - ) { - // Extend the lock TTL since it's the same worker - await redis.pexpire(redisKey, ttlMs); - logger.debug( - `Lock extended: ${lockKey} by ${ - config.getRawConfig().gerbil.exit_node_name - }` - ); - return true; - } + if (result === "OK") { + logger.debug( + `Lock acquired: ${lockKey} by ${ + config.getRawConfig().gerbil.exit_node_name + }` + ); + return true; + } - return false; - } catch (error) { - logger.error(`Failed to acquire lock ${lockKey}:`, error); - return false; + // Check if the existing lock is from this worker (reentrant behavior) + const existingValue = await redis.get(redisKey); + if ( + existingValue && + existingValue.startsWith( + `${config.getRawConfig().gerbil.exit_node_name}:` + ) + ) { + // Extend the lock TTL since it's the same worker + await redis.pexpire(redisKey, ttlMs); + logger.debug( + `Lock extended: ${lockKey} by ${ + config.getRawConfig().gerbil.exit_node_name + }` + ); + return true; + } + + // If this isn't our last attempt, wait before retrying with exponential backoff + if (attempt < maxRetries - 1) { + const delay = retryDelayMs * Math.pow(2, attempt); + logger.debug( + `Lock ${lockKey} not available, retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries})` + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } catch (error) { + logger.error(`Failed to acquire lock ${lockKey} (attempt ${attempt + 1}/${maxRetries}):`, error); + // On error, still retry if we have attempts left + if (attempt < maxRetries - 1) { + const delay = retryDelayMs * Math.pow(2, attempt); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } } + + logger.debug( + `Failed to acquire lock ${lockKey} after ${maxRetries} attempts` + ); + return false; } /** diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index 18410e62..f0343c5d 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -456,11 +456,11 @@ export async function getTraefikConfig( // ); } else if (resource.maintenanceModeType === "automatic") { showMaintenancePage = !hasHealthyServers; - if (showMaintenancePage) { - logger.warn( - `Resource ${resource.name} (${fullDomain}) has no healthy servers - showing maintenance page (AUTOMATIC mode)` - ); - } + // if (showMaintenancePage) { + // logger.warn( + // `Resource ${resource.name} (${fullDomain}) has no healthy servers - showing maintenance page (AUTOMATIC mode)` + // ); + // } } } diff --git a/server/private/middlewares/verifySubscription.ts b/server/private/middlewares/verifySubscription.ts index 5249c026..8cda737e 100644 --- a/server/private/middlewares/verifySubscription.ts +++ b/server/private/middlewares/verifySubscription.ts @@ -27,7 +27,18 @@ export async function verifyValidSubscription( return next(); } - const tier = await getOrgTierData(req.params.orgId); + const orgId = req.params.orgId || req.body.orgId || req.query.orgId || req.userOrgId; + + if (!orgId) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Organization ID is required to verify subscription" + ) + ); + } + + const tier = await getOrgTierData(orgId); if (!tier.active) { return next( diff --git a/server/private/routers/external.ts b/server/private/routers/external.ts index d9608e21..97c6db9f 100644 --- a/server/private/routers/external.ts +++ b/server/private/routers/external.ts @@ -436,18 +436,18 @@ authenticated.get( authenticated.post( "/re-key/:clientId/regenerate-client-secret", + verifyClientAccess, // this is first to set the org id verifyValidLicense, verifyValidSubscription, - verifyClientAccess, verifyUserHasAction(ActionsEnum.reGenerateSecret), reKey.reGenerateClientSecret ); authenticated.post( "/re-key/:siteId/regenerate-site-secret", + verifySiteAccess, // this is first to set the org id verifyValidLicense, verifyValidSubscription, - verifySiteAccess, verifyUserHasAction(ActionsEnum.reGenerateSecret), reKey.reGenerateSiteSecret ); diff --git a/src/app/auth/login/device/success/page.tsx b/src/app/auth/login/device/success/page.tsx index f725a867..6ee49587 100644 --- a/src/app/auth/login/device/success/page.tsx +++ b/src/app/auth/login/device/success/page.tsx @@ -7,6 +7,7 @@ import { useLicenseStatusContext } from "@app/hooks/useLicenseStatusContext"; import { CheckCircle2 } from "lucide-react"; import { useTranslations } from "next-intl"; import Link from "next/link"; +import { useEffect } from "react"; export default function DeviceAuthSuccessPage() { const { env } = useEnvContext(); @@ -20,6 +21,32 @@ export default function DeviceAuthSuccessPage() { ? env.branding.logo?.authPage?.height || 58 : 58; + useEffect(() => { + // Detect if we're on iOS or Android + const userAgent = navigator.userAgent || navigator.vendor || (window as any).opera; + const isIOS = /iPad|iPhone|iPod/.test(userAgent) && !(window as any).MSStream; + const isAndroid = /android/i.test(userAgent); + + if (isAndroid) { + // For Android Chrome Custom Tabs, use intent:// scheme which works more reliably + // This explicitly tells Chrome to send an intent to the app, which will bring + // SignInCodeActivity back to the foreground (it has launchMode="singleTop") + setTimeout(() => { + window.location.href = "intent://auth-success#Intent;scheme=pangolin;package=net.pangolin.Pangolin;end"; + }, 500); + } else if (isIOS) { + // Wait 500ms then attempt to open the app + setTimeout(() => { + // Try to open the app using deep link + window.location.href = "pangolin://"; + + setTimeout(() => { + window.location.href = "https://apps.apple.com/app/pangolin/net.pangolin.Pangolin.PangoliniOS"; + }, 2000); + }, 500); + } + }, []); + return ( <> @@ -55,4 +82,4 @@ export default function DeviceAuthSuccessPage() {

); -} +} \ No newline at end of file diff --git a/src/components/ConfirmDeleteDialog.tsx b/src/components/ConfirmDeleteDialog.tsx index 86b25bb2..4c5c6ad6 100644 --- a/src/components/ConfirmDeleteDialog.tsx +++ b/src/components/ConfirmDeleteDialog.tsx @@ -63,6 +63,8 @@ export default function ConfirmDeleteDialog({ } }); + const isConfirmed = form.watch("string") === string; + async function onSubmit() { try { await onConfirm(); @@ -139,7 +141,8 @@ export default function ConfirmDeleteDialog({ type="submit" form="confirm-delete-form" loading={loading} - disabled={loading} + disabled={loading || !isConfirmed} + className={!isConfirmed && !loading ? "opacity-50" : ""} > {buttonText}