name: CI/CD Pipeline # CI/CD workflow for building, publishing, mirroring, signing container images and building release binaries. # Actions are pinned to specific SHAs to reduce supply-chain risk. This workflow triggers on tag push events. permissions: contents: read packages: write # for GHCR push id-token: write # for Cosign Keyless (OIDC) Signing # Required secrets: # - DOCKER_HUB_USERNAME / DOCKER_HUB_ACCESS_TOKEN: push to Docker Hub # - GITHUB_TOKEN: used for GHCR login and OIDC keyless signing # - COSIGN_PRIVATE_KEY / COSIGN_PASSWORD / COSIGN_PUBLIC_KEY: for key-based signing on: push: tags: - "[0-9]+.[0-9]+.[0-9]+" - "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+" concurrency: group: ${{ github.ref }} cancel-in-progress: true jobs: pre-run: runs-on: ubuntu-latest permissions: write-all steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v5 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} role-duration-seconds: 3600 aws-region: ${{ secrets.AWS_REGION }} - name: Verify AWS identity run: aws sts get-caller-identity - name: Start EC2 instances run: | aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }} aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }} echo "EC2 instances started" release-arm: name: Build and Release (ARM64) runs-on: [self-hosted, linux, arm64, us-east-1] needs: [pre-run] if: >- ${{ needs.pre-run.result == 'success' }} # Job-level timeout to avoid runaway or stuck runs timeout-minutes: 120 env: # Target images DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} steps: - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Monitor storage space run: | THRESHOLD=75 USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g') echo "Used space: $USED_SPACE%" if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then echo "Used space is below the threshold of 75% free. Running Docker system prune." echo y | docker system prune -a else echo "Storage space is above the threshold. No action needed." fi - name: Log in to Docker Hub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: docker.io username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Extract tag name id: get-tag run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV shell: bash - name: Update version in package.json run: | TAG=${{ env.TAG }} sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts cat server/lib/consts.ts shell: bash - name: Check if release candidate id: check-rc run: | TAG=${{ env.TAG }} if [[ "$TAG" == *"-rc."* ]]; then echo "IS_RC=true" >> $GITHUB_ENV else echo "IS_RC=false" >> $GITHUB_ENV fi shell: bash - name: Build and push Docker images (Docker Hub - ARM64) run: | TAG=${{ env.TAG }} if [ "$IS_RC" = "true" ]; then make build-rc-arm tag=$TAG else make build-release-arm tag=$TAG fi echo "Built & pushed ARM64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}" shell: bash release-amd: name: Build and Release (AMD64) runs-on: [self-hosted, linux, x64, us-east-1] needs: [pre-run] if: >- ${{ needs.pre-run.result == 'success' }} # Job-level timeout to avoid runaway or stuck runs timeout-minutes: 120 env: # Target images DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} steps: - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Monitor storage space run: | THRESHOLD=75 USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g') echo "Used space: $USED_SPACE%" if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then echo "Used space is below the threshold of 75% free. Running Docker system prune." echo y | docker system prune -a else echo "Storage space is above the threshold. No action needed." fi - name: Log in to Docker Hub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: docker.io username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Extract tag name id: get-tag run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV shell: bash - name: Update version in package.json run: | TAG=${{ env.TAG }} sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts cat server/lib/consts.ts shell: bash - name: Check if release candidate id: check-rc run: | TAG=${{ env.TAG }} if [[ "$TAG" == *"-rc."* ]]; then echo "IS_RC=true" >> $GITHUB_ENV else echo "IS_RC=false" >> $GITHUB_ENV fi shell: bash - name: Build and push Docker images (Docker Hub - AMD64) run: | TAG=${{ env.TAG }} if [ "$IS_RC" = "true" ]; then make build-rc-amd tag=$TAG else make build-release-amd tag=$TAG fi echo "Built & pushed AMD64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}" shell: bash create-manifest: name: Create Multi-Arch Manifests runs-on: [self-hosted, linux, x64, us-east-1] needs: [release-arm, release-amd] if: >- ${{ needs.release-arm.result == 'success' && needs.release-amd.result == 'success' }} timeout-minutes: 30 steps: - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Log in to Docker Hub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: docker.io username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Extract tag name id: get-tag run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV shell: bash - name: Check if release candidate id: check-rc run: | TAG=${{ env.TAG }} if [[ "$TAG" == *"-rc."* ]]; then echo "IS_RC=true" >> $GITHUB_ENV else echo "IS_RC=false" >> $GITHUB_ENV fi shell: bash - name: Create multi-arch manifests run: | TAG=${{ env.TAG }} if [ "$IS_RC" = "true" ]; then make create-manifests-rc tag=$TAG else make create-manifests tag=$TAG fi echo "Created multi-arch manifests for tag: ${TAG}" shell: bash sign-and-package: name: Sign and Package runs-on: [self-hosted, linux, x64, us-east-1] needs: [release-arm, release-amd, create-manifest] if: >- ${{ needs.release-arm.result == 'success' && needs.release-amd.result == 'success' && needs.create-manifest.result == 'success' }} # Job-level timeout to avoid runaway or stuck runs timeout-minutes: 120 env: # Target images DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} steps: - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Extract tag name id: get-tag run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV shell: bash - name: Install Go uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: go-version: 1.24 - name: Update version in package.json run: | TAG=${{ env.TAG }} sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts cat server/lib/consts.ts shell: bash - name: Pull latest Gerbil version id: get-gerbil-tag run: | LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/gerbil/tags | jq -r '.[0].name') echo "LATEST_GERBIL_TAG=$LATEST_TAG" >> $GITHUB_ENV shell: bash - name: Pull latest Badger version id: get-badger-tag run: | LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/badger/tags | jq -r '.[0].name') echo "LATEST_BADGER_TAG=$LATEST_TAG" >> $GITHUB_ENV shell: bash - name: Update install/main.go run: | PANGOLIN_VERSION=${{ env.TAG }} GERBIL_VERSION=${{ env.LATEST_GERBIL_TAG }} BADGER_VERSION=${{ env.LATEST_BADGER_TAG }} sed -i "s/config.PangolinVersion = \".*\"/config.PangolinVersion = \"$PANGOLIN_VERSION\"/" install/main.go sed -i "s/config.GerbilVersion = \".*\"/config.GerbilVersion = \"$GERBIL_VERSION\"/" install/main.go sed -i "s/config.BadgerVersion = \".*\"/config.BadgerVersion = \"$BADGER_VERSION\"/" install/main.go echo "Updated install/main.go with Pangolin version $PANGOLIN_VERSION, Gerbil version $GERBIL_VERSION, and Badger version $BADGER_VERSION" cat install/main.go shell: bash - name: Build installer working-directory: install run: | make go-build-release - name: Upload artifacts from /install/bin uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: install-bin path: install/bin/ - name: Install skopeo + jq # skopeo: copy/inspect images between registries # jq: JSON parsing tool used to extract digest values run: | sudo apt-get update -y sudo apt-get install -y skopeo jq skopeo --version shell: bash - name: Login to GHCR env: REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json run: | mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")" skopeo login ghcr.io -u "${{ github.actor }}" -p "${{ secrets.GITHUB_TOKEN }}" shell: bash - name: Copy tags from Docker Hub to GHCR # Mirror the already-built images (all architectures) to GHCR so we can sign them # Wait a bit for both architectures to be available in Docker Hub manifest env: REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json run: | set -euo pipefail TAG=${{ env.TAG }} MAJOR_TAG=$(echo $TAG | cut -d. -f1) MINOR_TAG=$(echo $TAG | cut -d. -f1,2) echo "Waiting for multi-arch manifests to be ready..." sleep 30 # Determine if this is an RC release IS_RC="false" if [[ "$TAG" == *"-rc."* ]]; then IS_RC="true" fi if [ "$IS_RC" = "true" ]; then echo "RC release detected - copying version-specific tags only" # SQLite OSS echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG} -> ${{ env.GHCR_IMAGE }}:${TAG}" skopeo copy --all --retry-times 3 \ docker://$DOCKERHUB_IMAGE:$TAG \ docker://$GHCR_IMAGE:$TAG # PostgreSQL OSS echo "Copying ${{ env.DOCKERHUB_IMAGE }}:postgresql-${TAG} -> ${{ env.GHCR_IMAGE }}:postgresql-${TAG}" skopeo copy --all --retry-times 3 \ docker://$DOCKERHUB_IMAGE:postgresql-$TAG \ docker://$GHCR_IMAGE:postgresql-$TAG # SQLite Enterprise echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-${TAG} -> ${{ env.GHCR_IMAGE }}:ee-${TAG}" skopeo copy --all --retry-times 3 \ docker://$DOCKERHUB_IMAGE:ee-$TAG \ docker://$GHCR_IMAGE:ee-$TAG # PostgreSQL Enterprise echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-postgresql-${TAG} -> ${{ env.GHCR_IMAGE }}:ee-postgresql-${TAG}" skopeo copy --all --retry-times 3 \ docker://$DOCKERHUB_IMAGE:ee-postgresql-$TAG \ docker://$GHCR_IMAGE:ee-postgresql-$TAG else echo "Regular release detected - copying all tags (latest, major, minor, full version)" # SQLite OSS - all tags for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:${TAG_SUFFIX}" skopeo copy --all --retry-times 3 \ docker://$DOCKERHUB_IMAGE:$TAG_SUFFIX \ docker://$GHCR_IMAGE:$TAG_SUFFIX done # PostgreSQL OSS - all tags for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do echo "Copying ${{ env.DOCKERHUB_IMAGE }}:postgresql-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:postgresql-${TAG_SUFFIX}" skopeo copy --all --retry-times 3 \ docker://$DOCKERHUB_IMAGE:postgresql-$TAG_SUFFIX \ docker://$GHCR_IMAGE:postgresql-$TAG_SUFFIX done # SQLite Enterprise - all tags for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:ee-${TAG_SUFFIX}" skopeo copy --all --retry-times 3 \ docker://$DOCKERHUB_IMAGE:ee-$TAG_SUFFIX \ docker://$GHCR_IMAGE:ee-$TAG_SUFFIX done # PostgreSQL Enterprise - all tags for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-postgresql-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:ee-postgresql-${TAG_SUFFIX}" skopeo copy --all --retry-times 3 \ docker://$DOCKERHUB_IMAGE:ee-postgresql-$TAG_SUFFIX \ docker://$GHCR_IMAGE:ee-postgresql-$TAG_SUFFIX done fi echo "All images copied successfully to GHCR!" shell: bash - name: Login to GitHub Container Registry (for cosign) uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Install cosign # cosign is used to sign and verify container images (key and keyless) uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0 - name: Dual-sign and verify (GHCR & Docker Hub) # Sign each image by digest using keyless (OIDC) and key-based signing, # then verify both the public key signature and the keyless OIDC signature. env: TAG: ${{ env.TAG }} COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} COSIGN_PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }} COSIGN_YES: "true" run: | set -euo pipefail issuer="https://token.actions.githubusercontent.com" id_regex="^https://github.com/${{ github.repository }}/.+" # accept this repo (all workflows/refs) # Determine if this is an RC release IS_RC="false" if [[ "$TAG" == *"-rc."* ]]; then IS_RC="true" fi # Define image variants to sign if [ "$IS_RC" = "true" ]; then echo "RC release - signing version-specific tags only" IMAGE_TAGS=( "${TAG}" "postgresql-${TAG}" "ee-${TAG}" "ee-postgresql-${TAG}" ) else echo "Regular release - signing all tags" MAJOR_TAG=$(echo $TAG | cut -d. -f1) MINOR_TAG=$(echo $TAG | cut -d. -f1,2) IMAGE_TAGS=( "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG" "postgresql-latest" "postgresql-$MAJOR_TAG" "postgresql-$MINOR_TAG" "postgresql-$TAG" "ee-latest" "ee-$MAJOR_TAG" "ee-$MINOR_TAG" "ee-$TAG" "ee-postgresql-latest" "ee-postgresql-$MAJOR_TAG" "ee-postgresql-$MINOR_TAG" "ee-postgresql-$TAG" ) fi # Sign each image variant for both registries for BASE_IMAGE in "${GHCR_IMAGE}" "${DOCKERHUB_IMAGE}"; do for IMAGE_TAG in "${IMAGE_TAGS[@]}"; do echo "Processing ${BASE_IMAGE}:${IMAGE_TAG}" DIGEST="$(skopeo inspect --retry-times 3 docker://${BASE_IMAGE}:${IMAGE_TAG} | jq -r '.Digest')" REF="${BASE_IMAGE}@${DIGEST}" echo "Resolved digest: ${REF}" echo "==> cosign sign (keyless) --recursive ${REF}" cosign sign --recursive "${REF}" echo "==> cosign sign (key) --recursive ${REF}" cosign sign --key env://COSIGN_PRIVATE_KEY --recursive "${REF}" # Retry wrapper for verification to handle registry propagation delays retry_verify() { local cmd="$1" local attempts=6 local delay=5 local i=1 until eval "$cmd"; do if [ $i -ge $attempts ]; then echo "Verification failed after $attempts attempts" return 1 fi echo "Verification not yet available. Retry $i/$attempts after ${delay}s..." sleep $delay i=$((i+1)) delay=$((delay*2)) # Cap the delay to avoid very long waits if [ $delay -gt 60 ]; then delay=60; fi done return 0 } echo "==> cosign verify (public key) ${REF}" if retry_verify "cosign verify --key env://COSIGN_PUBLIC_KEY '${REF}' -o text"; then VERIFIED_INDEX=true else VERIFIED_INDEX=false fi echo "==> cosign verify (keyless policy) ${REF}" if retry_verify "cosign verify --certificate-oidc-issuer '${issuer}' --certificate-identity-regexp '${id_regex}' '${REF}' -o text"; then VERIFIED_INDEX_KEYLESS=true else VERIFIED_INDEX_KEYLESS=false fi # If index verification fails, attempt to verify child platform manifests if [ "${VERIFIED_INDEX}" != "true" ] || [ "${VERIFIED_INDEX_KEYLESS}" != "true" ]; then echo "Index verification not available; attempting child manifest verification for ${BASE_IMAGE}:${IMAGE_TAG}" CHILD_VERIFIED=false for ARCH in arm64 amd64; do CHILD_TAG="${IMAGE_TAG}-${ARCH}" echo "Resolving child digest for ${BASE_IMAGE}:${CHILD_TAG}" CHILD_DIGEST="$(skopeo inspect --retry-times 3 docker://${BASE_IMAGE}:${CHILD_TAG} | jq -r '.Digest' || true)" if [ -n "${CHILD_DIGEST}" ] && [ "${CHILD_DIGEST}" != "null" ]; then CHILD_REF="${BASE_IMAGE}@${CHILD_DIGEST}" echo "==> cosign verify (public key) child ${CHILD_REF}" if retry_verify "cosign verify --key env://COSIGN_PUBLIC_KEY '${CHILD_REF}' -o text"; then CHILD_VERIFIED=true echo "Public key verification succeeded for child ${CHILD_REF}" else echo "Public key verification failed for child ${CHILD_REF}" fi echo "==> cosign verify (keyless policy) child ${CHILD_REF}" if retry_verify "cosign verify --certificate-oidc-issuer '${issuer}' --certificate-identity-regexp '${id_regex}' '${CHILD_REF}' -o text"; then CHILD_VERIFIED=true echo "Keyless verification succeeded for child ${CHILD_REF}" else echo "Keyless verification failed for child ${CHILD_REF}" fi else echo "No child digest found for ${BASE_IMAGE}:${CHILD_TAG}; skipping" fi done if [ "${CHILD_VERIFIED}" != "true" ]; then echo "Failed to verify index and no child manifests verified for ${BASE_IMAGE}:${IMAGE_TAG}" exit 10 fi fi echo "✓ Successfully signed and verified ${BASE_IMAGE}:${IMAGE_TAG}" done done echo "All images signed and verified successfully!" shell: bash post-run: needs: [pre-run, release-arm, release-amd, create-manifest, sign-and-package] if: >- ${{ always() && needs.pre-run.result == 'success' && (needs.release-arm.result == 'success' || needs.release-arm.result == 'skipped' || needs.release-arm.result == 'failure') && (needs.release-amd.result == 'success' || needs.release-amd.result == 'skipped' || needs.release-amd.result == 'failure') && (needs.create-manifest.result == 'success' || needs.create-manifest.result == 'skipped' || needs.create-manifest.result == 'failure') && (needs.sign-and-package.result == 'success' || needs.sign-and-package.result == 'skipped' || needs.sign-and-package.result == 'failure') }} runs-on: ubuntu-latest permissions: write-all steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v5 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} role-duration-seconds: 3600 aws-region: ${{ secrets.AWS_REGION }} - name: Verify AWS identity run: aws sts get-caller-identity - name: Stop EC2 instances run: | aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }} aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }} echo "EC2 instances stopped"