diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 8af8625d..e95cd2e5 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -24,9 +24,35 @@ concurrency: cancel-in-progress: true jobs: - release: - name: Build and Release - runs-on: [self-hosted, linux, x64] + pre-run: + runs-on: ubuntu-latest + permissions: write-all + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} + role-duration-seconds: 3600 + aws-region: ${{ secrets.AWS_REGION }} + + - name: Verify AWS identity + run: aws sts get-caller-identity + + - name: Start EC2 instances + run: | + aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }} + aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }} + echo "EC2 instances started" + + + release-arm: + name: Build and Release (ARM64) + runs-on: [self-hosted, linux, arm64, us-east-1] + needs: [pre-run] + if: >- + ${{ + needs.pre-run.result == 'success' + }} # Job-level timeout to avoid runaway or stuck runs timeout-minutes: 120 env: @@ -38,11 +64,17 @@ jobs: - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Set up QEMU - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + - name: Monitor storage space + run: | + THRESHOLD=75 + USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g') + echo "Used space: $USED_SPACE%" + if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then + echo "Used space is below the threshold of 75% free. Running Docker system prune." + echo y | docker system prune -a + else + echo "Storage space is above the threshold. No action needed." + fi - name: Log in to Docker Hub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 @@ -50,6 +82,103 @@ jobs: registry: docker.io username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Extract tag name + id: get-tag + run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + shell: bash + + - name: Update version in package.json + run: | + TAG=${{ env.TAG }} + sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts + cat server/lib/consts.ts + shell: bash + + - name: Build and push Docker images (Docker Hub - ARM64) + run: | + TAG=${{ env.TAG }} + make build-release-arm tag=$TAG + echo "Built & pushed ARM64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}" + shell: bash + + release-amd: + name: Build and Release (AMD64) + runs-on: [self-hosted, linux, x64, us-east-1] + needs: [pre-run] + if: >- + ${{ + needs.pre-run.result == 'success' + }} + # Job-level timeout to avoid runaway or stuck runs + timeout-minutes: 120 + env: + # Target images + DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + + steps: + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Monitor storage space + run: | + THRESHOLD=75 + USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g') + echo "Used space: $USED_SPACE%" + if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then + echo "Used space is below the threshold of 75% free. Running Docker system prune." + echo y | docker system prune -a + else + echo "Storage space is above the threshold. No action needed." + fi + + - name: Log in to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: docker.io + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Extract tag name + id: get-tag + run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + shell: bash + + - name: Update version in package.json + run: | + TAG=${{ env.TAG }} + sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts + cat server/lib/consts.ts + shell: bash + + - name: Build and push Docker images (Docker Hub - AMD64) + run: | + TAG=${{ env.TAG }} + make build-release-amd tag=$TAG + echo "Built & pushed AMD64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}" + shell: bash + + sign-and-package: + name: Sign and Package + runs-on: [self-hosted, linux, x64, us-east-1] + needs: [release-arm, release-amd] + if: >- + ${{ + needs.release-arm.result == 'success' && + needs.release-amd.result == 'success' + }} + # Job-level timeout to avoid runaway or stuck runs + timeout-minutes: 120 + env: + # Target images + DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + + steps: + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Extract tag name id: get-tag run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV @@ -96,7 +225,7 @@ jobs: - name: Build installer working-directory: install run: | - make go-build-release + make go-build-release - name: Upload artifacts from /install/bin uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 @@ -104,13 +233,6 @@ jobs: name: install-bin path: install/bin/ - - name: Build and push Docker images (Docker Hub) - run: | - TAG=${{ env.TAG }} - make build-release tag=$TAG - echo "Built & pushed to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}" - shell: bash - - name: Install skopeo + jq # skopeo: copy/inspect images between registries # jq: JSON parsing tool used to extract digest values @@ -127,15 +249,18 @@ jobs: - name: Copy tag from Docker Hub to GHCR # Mirror the already-built image (all architectures) to GHCR so we can sign it + # Wait a bit for both architectures to be available in Docker Hub manifest run: | set -euo pipefail TAG=${{ env.TAG }} + echo "Waiting for multi-arch manifest to be ready..." + sleep 30 echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG} -> ${{ env.GHCR_IMAGE }}:${TAG}" skopeo copy --all --retry-times 3 \ docker://$DOCKERHUB_IMAGE:$TAG \ docker://$GHCR_IMAGE:$TAG shell: bash - + - name: Login to GitHub Container Registry (for cosign) uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: @@ -185,3 +310,32 @@ jobs: "${REF}" -o text done shell: bash + + post-run: + needs: [pre-run, release-arm, release-amd, sign-and-package] + if: >- + ${{ + always() && + needs.pre-run.result == 'success' && + (needs.release-arm.result == 'success' || needs.release-arm.result == 'skipped' || needs.release-arm.result == 'failure') && + (needs.release-amd.result == 'success' || needs.release-amd.result == 'skipped' || needs.release-amd.result == 'failure') && + (needs.sign-and-package.result == 'success' || needs.sign-and-package.result == 'skipped' || needs.sign-and-package.result == 'failure') + }} + runs-on: ubuntu-latest + permissions: write-all + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} + role-duration-seconds: 3600 + aws-region: ${{ secrets.AWS_REGION }} + + - name: Verify AWS identity + run: aws sts get-caller-identity + + - name: Stop EC2 instances + run: | + aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }} + aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }} + echo "EC2 instances stopped" diff --git a/Makefile b/Makefile index 6c538a47..ffd89bb5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build build-pg build-release build-arm build-x86 test clean +.PHONY: build build-pg build-release build-release-arm build-release-amd build-arm build-x86 test clean major_tag := $(shell echo $(tag) | cut -d. -f1) minor_tag := $(shell echo $(tag) | cut -d. -f1,2) @@ -44,6 +44,94 @@ build-release: --tag fosrl/pangolin:ee-postgresql-$(tag) \ --push . +build-release-arm: + @if [ -z "$(tag)" ]; then \ + echo "Error: tag is required. Usage: make build-release-arm tag="; \ + exit 1; \ + fi + @MAJOR_TAG=$$(echo $(tag) | cut -d. -f1); \ + MINOR_TAG=$$(echo $(tag) | cut -d. -f1,2); \ + docker buildx build \ + --build-arg BUILD=oss \ + --build-arg DATABASE=sqlite \ + --platform linux/arm64 \ + --tag fosrl/pangolin:latest \ + --tag fosrl/pangolin:$$MAJOR_TAG \ + --tag fosrl/pangolin:$$MINOR_TAG \ + --tag fosrl/pangolin:$(tag) \ + --push . && \ + docker buildx build \ + --build-arg BUILD=oss \ + --build-arg DATABASE=pg \ + --platform linux/arm64 \ + --tag fosrl/pangolin:postgresql-latest \ + --tag fosrl/pangolin:postgresql-$$MAJOR_TAG \ + --tag fosrl/pangolin:postgresql-$$MINOR_TAG \ + --tag fosrl/pangolin:postgresql-$(tag) \ + --push . && \ + docker buildx build \ + --build-arg BUILD=enterprise \ + --build-arg DATABASE=sqlite \ + --platform linux/arm64 \ + --tag fosrl/pangolin:ee-latest \ + --tag fosrl/pangolin:ee-$$MAJOR_TAG \ + --tag fosrl/pangolin:ee-$$MINOR_TAG \ + --tag fosrl/pangolin:ee-$(tag) \ + --push . && \ + docker buildx build \ + --build-arg BUILD=enterprise \ + --build-arg DATABASE=pg \ + --platform linux/arm64 \ + --tag fosrl/pangolin:ee-postgresql-latest \ + --tag fosrl/pangolin:ee-postgresql-$$MAJOR_TAG \ + --tag fosrl/pangolin:ee-postgresql-$$MINOR_TAG \ + --tag fosrl/pangolin:ee-postgresql-$(tag) \ + --push . + +build-release-amd: + @if [ -z "$(tag)" ]; then \ + echo "Error: tag is required. Usage: make build-release-amd tag="; \ + exit 1; \ + fi + @MAJOR_TAG=$$(echo $(tag) | cut -d. -f1); \ + MINOR_TAG=$$(echo $(tag) | cut -d. -f1,2); \ + docker buildx build \ + --build-arg BUILD=oss \ + --build-arg DATABASE=sqlite \ + --platform linux/amd64 \ + --tag fosrl/pangolin:latest \ + --tag fosrl/pangolin:$$MAJOR_TAG \ + --tag fosrl/pangolin:$$MINOR_TAG \ + --tag fosrl/pangolin:$(tag) \ + --push . && \ + docker buildx build \ + --build-arg BUILD=oss \ + --build-arg DATABASE=pg \ + --platform linux/amd64 \ + --tag fosrl/pangolin:postgresql-latest \ + --tag fosrl/pangolin:postgresql-$$MAJOR_TAG \ + --tag fosrl/pangolin:postgresql-$$MINOR_TAG \ + --tag fosrl/pangolin:postgresql-$(tag) \ + --push . && \ + docker buildx build \ + --build-arg BUILD=enterprise \ + --build-arg DATABASE=sqlite \ + --platform linux/amd64 \ + --tag fosrl/pangolin:ee-latest \ + --tag fosrl/pangolin:ee-$$MAJOR_TAG \ + --tag fosrl/pangolin:ee-$$MINOR_TAG \ + --tag fosrl/pangolin:ee-$(tag) \ + --push . && \ + docker buildx build \ + --build-arg BUILD=enterprise \ + --build-arg DATABASE=pg \ + --platform linux/amd64 \ + --tag fosrl/pangolin:ee-postgresql-latest \ + --tag fosrl/pangolin:ee-postgresql-$$MAJOR_TAG \ + --tag fosrl/pangolin:ee-postgresql-$$MINOR_TAG \ + --tag fosrl/pangolin:ee-postgresql-$(tag) \ + --push . + build-rc: @if [ -z "$(tag)" ]; then \ echo "Error: tag is required. Usage: make build-release tag="; \