CI Release Builder (#455)

.github/workflows/build-in-docker.yml

@@ -2,37 +2,104 @@ name: Build using Docker
 
 on:
   push:
-    branches: [ "dev", "candidate", "release", "jshooks" ]
+    branches: ["dev", "candidate", "release", "jshooks"]
   pull_request:
-    branches: [ "dev", "candidate", "release", "jshooks" ]
+    branches: ["dev", "candidate", "release", "jshooks"]
 
 concurrency:
-  group: ${{ github.workflow }}
-  cancel-in-progress: false
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  DEBUG_BUILD_CONTAINERS_AFTER_CLEANUP: 1
 
 jobs:
   checkout:
     runs-on: [self-hosted, vanity]
+    outputs:
+      checkout_path: ${{ steps.vars.outputs.checkout_path }}
     steps:
-    - uses: actions/checkout@v4
-      with:
-        clean: false
+      - name: Prepare checkout path
+        id: vars
+        run: |
+          SAFE_BRANCH=$(echo "${{ github.ref_name }}" | sed -e 's/[^a-zA-Z0-9._-]/-/g')
+          CHECKOUT_PATH="${SAFE_BRANCH}-${{ github.sha }}"
+          echo "checkout_path=${CHECKOUT_PATH}" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/checkout@v4
+        with:
+          path: ${{ steps.vars.outputs.checkout_path }}
+          clean: true
+          fetch-depth: 2 # Only get the last 2 commits, to avoid fetching all history
+
   checkpatterns:
     runs-on: [self-hosted, vanity]
     needs: checkout
+    defaults:
+      run:
+        working-directory: ${{ needs.checkout.outputs.checkout_path }}
     steps:
-    - name: Check for suspicious patterns
-      run: /bin/bash suspicious_patterns.sh
+      - name: Check for suspicious patterns
+        run: /bin/bash suspicious_patterns.sh
+
   build:
     runs-on: [self-hosted, vanity]
-    needs: checkpatterns
+    needs: [checkpatterns, checkout]
+    defaults:
+      run:
+        working-directory: ${{ needs.checkout.outputs.checkout_path }}
     steps:
-    - name: Build using Docker
-      run: /bin/bash release-builder.sh
+      - name: Set Cleanup Script Path
+        run: |
+          echo "JOB_CLEANUP_SCRIPT=$(mktemp)" >> $GITHUB_ENV
+
+      - name: Build using Docker
+        run: /bin/bash release-builder.sh
+
+      - name: Stop Container (Cleanup)
+        if: always()
+        run: |
+          echo "Running cleanup script: $JOB_CLEANUP_SCRIPT"
+          /bin/bash -e -x "$JOB_CLEANUP_SCRIPT"
+          CLEANUP_EXIT_CODE=$?
+
+          if [[ "$CLEANUP_EXIT_CODE" -eq 0 ]]; then
+            echo "Cleanup script succeeded."
+            rm -f "$JOB_CLEANUP_SCRIPT"
+            echo "Cleanup script removed."
+          else
+            echo "⚠️ Cleanup script failed! Keeping for debugging: $JOB_CLEANUP_SCRIPT"
+          fi
+
+          if [[ "${DEBUG_BUILD_CONTAINERS_AFTER_CLEANUP}" == "1" ]]; then
+            echo "🔍 Checking for leftover containers..."
+            BUILD_CONTAINERS=$(docker ps --format '{{.Names}}' | grep '^xahaud_cached_builder' || echo "")
+
+            if [[ -n "$BUILD_CONTAINERS" ]]; then
+              echo "⚠️ WARNING: Some build containers are still running"
+              echo "$BUILD_CONTAINERS"
+            else
+              echo "✅ No build containers found"
+            fi
+          fi
+
   tests:
     runs-on: [self-hosted, vanity]
-    needs: build
+    needs: [build, checkout]
+    defaults:
+      run:
+        working-directory: ${{ needs.checkout.outputs.checkout_path }}
     steps:
-    - name: Unit tests
-      run: /bin/bash docker-unit-tests.sh
+      - name: Unit tests
+        run: /bin/bash docker-unit-tests.sh
 
+  cleanup:
+    runs-on: [self-hosted, vanity]
+    needs: [tests, checkout]
+    if: always()
+    steps:
+      - name: Cleanup workspace
+        run: |
+          CHECKOUT_PATH="${{ needs.checkout.outputs.checkout_path }}"
+          echo "Cleaning workspace for ${CHECKOUT_PATH}"
+          rm -rf "${{ github.workspace }}/${CHECKOUT_PATH}"

docker-unit-tests.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
+echo "Mounting $(pwd)/io in ubuntu and running unit tests"
 docker run --rm -i -v $(pwd):/io ubuntu sh -c '/io/release-build/xahaud -u'
-

release-builder.sh

@@ -5,8 +5,6 @@
 # debugging.
 set -ex
 
-set -e
-
 echo "START BUILDING (HOST)"
 
 echo "Cleaning previously built binary"
@@ -19,7 +17,26 @@ if [[ "$GITHUB_REPOSITORY" == "" ]]; then
   BUILD_CORES=8
 fi
 
-CONTAINER_NAME=xahaud_cached_builder_$(echo "$GITHUB_ACTOR" | awk '{print tolower($0)}')
+EXIT_IF_CONTAINER_RUNNING=${EXIT_IF_CONTAINER_RUNNING:-1}
+# Ensure still works outside of GH Actions by setting these to /dev/null
+# GA will run this script and then delete it at the end of the job
+JOB_CLEANUP_SCRIPT=${JOB_CLEANUP_SCRIPT:-/dev/null}
+NORMALIZED_WORKFLOW=$(echo "$GITHUB_WORKFLOW" | tr -c 'a-zA-Z0-9' '-')
+NORMALIZED_REF=$(echo "$GITHUB_REF" | tr -c 'a-zA-Z0-9' '-')
+CONTAINER_NAME="xahaud_cached_builder_${NORMALIZED_WORKFLOW}-${NORMALIZED_REF}"
+
+# Check if the container is already running
+if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
+    echo "⚠️ A running container (${CONTAINER_NAME}) was detected."
+
+    if [[ "$EXIT_IF_CONTAINER_RUNNING" -eq 1 ]]; then
+        echo "❌ EXIT_IF_CONTAINER_RUNNING is set. Exiting."
+        exit 1
+    else
+        echo "🛑 Stopping the running container: ${CONTAINER_NAME}"
+        docker stop "${CONTAINER_NAME}"
+    fi
+fi
 
 echo "-- BUILD CORES:       $BUILD_CORES"
 echo "-- GITHUB_REPOSITORY: $GITHUB_REPOSITORY"
@@ -62,6 +79,8 @@ else
     # GH Action, runner
     echo "GH Action, runner, clean & re-create create persistent container"
     docker rm -f $CONTAINER_NAME
+    echo "echo 'Stopping container: $CONTAINER_NAME'" >> "$JOB_CLEANUP_SCRIPT"
+    echo "docker stop --time=15 \"$CONTAINER_NAME\" || echo 'Failed to stop container or container not running'" >> "$JOB_CLEANUP_SCRIPT"
     docker run -di --user 0:$(id -g) --name $CONTAINER_NAME -v /data/builds:/data/builds -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash
     docker exec -i $CONTAINER_NAME /hbb_exe/activate-exec bash -x /io/build-full.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER"
     docker stop $CONTAINER_NAME