⚡ Deploy workflows - Skip the disk cleanup, build during the infrastructure job, send less to Docker, and crawl at the full rate

.dockerignore

@@ -3,6 +3,15 @@
 .github
 
 content-temp
+# build-artifacts.yml puts the SSW.Rules.Content checkout in the content folder. Its git data and its images are
+# approximately 3 GB, and the image build does not read them. The MDX and JSON files stay in the build context, because
+# Tailwind reads the text files of the build folder to find the class names for the CSS.
+content/**
+!content/**/*.mdx
+!content/**/*.json
+# A git ref is a file with the name of its branch, and that name can end in .json or .mdx.
+content/.git/**
+
 # scripts are needed during build (postbuild runs generate-archived-rules.js)
 # so keep scripts excluded by default, but include the one required file
 scripts

.github/workflows/build-and-deploy.yml

@@ -52,13 +52,15 @@ jobs:
     # ============================================================================
     # BUILD AND PUSH CONTAINER
     # ============================================================================
+    # The build runs at the same time as the infrastructure job, so it cannot read the registry name from the outputs
+    # of that job. The acr_name value follows the naming rule in infra/deploy.ps1. The deploy job stops if the image is
+    # not in the registry that the infrastructure job reports.
     build:
         name: Build and Push Container
-        needs: infrastructure
         uses: ./.github/workflows/build-artifacts.yml
         with:
             tag: ${{ github.event.inputs.environment || 'staging' }}
-            acr_name: ${{ needs.infrastructure.outputs.acrName }}
+            acr_name: acrsswrules${{ github.event.inputs.environment == 'production' && 'prod' || github.event.inputs.environment || 'staging' }}
             image_name: ssw-rules
             tina_branch: ${{ github.event.inputs.tina_branch }}
             source_branch: ${{ github.event.inputs.branch_name }}
@@ -91,6 +93,17 @@ jobs:
         needs: [infrastructure, build]
         environment: ${{ github.event.inputs.environment || 'staging' }}
         steps:
+            - name: Check that the image is in the registry of the infrastructure
+              env:
+                  IMAGE: ${{ needs.build.outputs.image_tag }}
+                  ACR_LOGIN_SERVER: ${{ needs.infrastructure.outputs.acrLoginServer }}
+              run: |
+                  if [[ -z "$ACR_LOGIN_SERVER" || "$IMAGE" != "$ACR_LOGIN_SERVER/"* ]]; then
+                      echo "::error::The build job pushed the image '$IMAGE', but the registry of the infrastructure is '$ACR_LOGIN_SERVER'. Make the acr_name value of the build job agree with the registry name in infra/deploy.ps1."
+                      exit 1
+                  fi
+                  echo "The image $IMAGE is in the registry of the infrastructure ($ACR_LOGIN_SERVER)"
+
             - name: Use build info from build job
               id: build_info
               run: |
@@ -258,6 +271,11 @@ jobs:
         uses: ./.github/workflows/crawl-sitemap.yml
         with:
             sitemap_url: https://www.ssw.com.au/rules/sitemap-index-0.xml
+            # This crawl starts minutes after the build. Each page is then newer than its revalidate period, so the
+            # server answers from its cache and a request starts no render. The crawler permits 20 requests each
+            # second at most.
+            batch_size: '20'
+            delay_ms: '0'
 
     check-content-freshness:
         name: Check Content Freshness & Retrigger if Stale

.github/workflows/build-artifacts.yml

@@ -352,6 +352,9 @@ jobs:
       - name: Set up Docker Buildx (no cache)
         uses: docker/setup-buildx-action@v3
         with:
+          # GitHub discards the runner after the job. The removal of the builder deletes its build cache, which takes
+          # time and has no benefit on such a runner.
+          cleanup: false
           driver-opts: |
             env.BUILDKIT_STEP_LOG_MAX_SIZE=10485760
             env.BUILDKIT_STEP_LOG_MAX_SPEED=10485760
@@ -393,7 +396,35 @@ jobs:
           echo "=== Home directory usage ==="
           du -h -d 2 /home/runner 2>/dev/null | head -20 || true
 
+      # The image build writes less than 20 GB to the Docker data directory. The cleanup takes 1 to 3 minutes, so it runs
+      # only when the file system of that directory has less free space than MIN_FREE_GB. It also runs when this step
+      # cannot read the free space.
+      - name: Check if the disk cleanup is necessary
+        id: disk_space
+        env:
+          MIN_FREE_GB: 30
+        run: |
+          DOCKER_ROOT=$(timeout 30 docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)
+          DOCKER_ROOT=${DOCKER_ROOT:-/var/lib/docker}
+          FREE_KB=$(df --output=avail -k "$DOCKER_ROOT" 2>/dev/null | awk 'NR==2 {print $1}' || true)
+
+          if [[ "$FREE_KB" =~ ^[0-9]+$ ]]; then
+            FREE_GB=$((FREE_KB / 1024 / 1024))
+            echo "Free disk space for Docker ($DOCKER_ROOT): ${FREE_GB} GB"
+            echo "Minimum to build with no cleanup: ${MIN_FREE_GB} GB"
+            if [ "$FREE_GB" -ge "$MIN_FREE_GB" ]; then
+              echo "Result: the disk cleanup is not necessary"
+              echo "enough_free_space=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "::notice::Free disk space for Docker is ${FREE_GB} GB, which is less than ${MIN_FREE_GB} GB. The disk cleanup runs."
+          else
+            echo "::warning::Could not read the free disk space of $DOCKER_ROOT. The disk cleanup runs."
+          fi
+          echo "enough_free_space=false" >> "$GITHUB_OUTPUT"
+
       - name: Aggressive disk space cleanup (pre-build)
+        if: steps.disk_space.outputs.enough_free_space != 'true'
         timeout-minutes: 5
         run: |
           echo "=== Starting aggressive disk cleanup ==="
@@ -451,6 +482,7 @@ jobs:
           echo "=== Cleanup completed ==="
 
       - name: Check disk usage after cleanup
+        if: steps.disk_space.outputs.enough_free_space != 'true'
         timeout-minutes: 2
         run: |
           echo "=== Disk Usage After Cleanup ==="
@@ -515,32 +547,10 @@ jobs:
           # Clean up intermediate containers after build
           pull: true
 
-      - name: Post-build cleanup
+      # Shows how much disk space the image build used, to check the minimum of the cleanup decision.
+      - name: Check disk usage after build
         if: always()
-        run: |
-          echo "=== Post-build cleanup ==="
-
-          # Clean Docker buildx cache immediately after build
-          echo "Cleaning Docker buildx cache..."
-          docker buildx prune -af || true
-
-          # Remove intermediate build containers and images
-          echo "Removing intermediate Docker resources..."
-          docker system prune -af --volumes || true
-          docker builder prune -af || true
-
-          # Clean runner diagnostic logs that may have grown large
-          echo "Cleaning runner diagnostic logs..."
-          sudo find /home/runner/actions-runner/cached/_diag -name "*.log" -type f -size +100M -delete || true
-          sudo find /home/runner/actions-runner/cached/_diag -name "*.log" -type f -mtime +0 -delete || true
-
-          # Final disk usage check
-          echo ""
-          echo "=== Final Disk Usage ==="
-          df -h
-          echo ""
-          AVAILABLE=$(df -h / | awk 'NR==2 {print $4}')
-          echo "Available disk space: $AVAILABLE"
+        run: df -h || true
 
       - name: Record build completion and analyze performance
         run: |

.github/workflows/crawl-sitemap.yml

@@ -7,6 +7,19 @@ on:
                 type: string
                 required: true
                 description: 'URL of the sitemap to crawl'
+            # Only a caller can set the request rate. A crawl that a person starts uses the default rate of the crawler
+            # (5 requests, then 250 ms). A request for a page that is older than its revalidate period starts a render
+            # that reads TinaCloud, and a fast crawl of such pages can cause TinaCloud throttling.
+            batch_size:
+                type: string
+                required: false
+                default: ''
+                description: 'Number of requests at the same time (SITEMAP_BATCH_SIZE). Empty: the default of the crawler.'
+            delay_ms:
+                type: string
+                required: false
+                default: ''
+                description: 'Delay between batches in milliseconds (SITEMAP_DELAY_MS). Empty: the default of the crawler.'
         outputs:
             total_pages:
                 description: 'Total number of pages crawled'
@@ -62,29 +75,23 @@ jobs:
 
             - name: Checkout code
               uses: actions/checkout@v4
-              with:
-                  fetch-depth: 0
 
             - name: Setup Node.js
               uses: actions/setup-node@v4
               with:
                   node-version-file: .nvmrc
 
-            - name: Install pnpm
-              uses: pnpm/action-setup@v4
-              with:
-                  version: 10.10.0
-
-            - name: Install dependencies
-              run: pnpm install --frozen-lockfile
-
+            # The crawler imports only modules that are part of Node.js, so this job does not install dependencies.
             - name: Run sitemap crawler
               id: crawl_sitemap
               continue-on-error: true
+              env:
+                  SITEMAP_BATCH_SIZE: ${{ inputs.batch_size }}
+                  SITEMAP_DELAY_MS: ${{ inputs.delay_ms }}
               run: |
                   echo "🔍 Running sitemap crawler..."
                   echo "Sitemap URL: ${{ steps.set_url.outputs.sitemap_url }}"
-                  pnpm crawl-sitemap "${{ steps.set_url.outputs.sitemap_url }}" || true
+                  node ./scripts/crawl-sitemap.js "${{ steps.set_url.outputs.sitemap_url }}" || true
 
             - name: Extract statistics
               id: extract_stats

.github/workflows/pr-deploy.yml

@@ -3,8 +3,11 @@ on:
   issue_comment:
     types: [created]
 
+# Each comment starts this workflow, and GitHub applies the concurrency group before the `if` of the validate job.
+# Only a /deploy comment shares the group of its pull request. Another comment gets a group of its own, so it cannot
+# cancel a deploy that is in progress.
 concurrency:
-  group: pr-${{ github.event.issue.number }}
+  group: pr-${{ github.event.issue.number }}-${{ github.event.issue.pull_request && github.event.comment.body == '/deploy' && 'deploy' || github.run_id }}
   cancel-in-progress: true
 
 permissions:
@@ -81,13 +84,16 @@ jobs:
   # ============================================================================
   # BUILD AND PUSH CONTAINER
   # ============================================================================
+  # The build runs at the same time as the infrastructure job, so it cannot read the registry name from the outputs of
+  # that job. A pull request slot uses the staging registry, and acr_name is the name that infra/deploy.ps1 gives it.
+  # The deploy job stops if the image is not in the registry that the infrastructure job reports.
   build:
     name: Build and Push Container
-    needs: [validate, infrastructure]
+    needs: validate
     uses: ./.github/workflows/build-artifacts.yml
     with:
       tag: pr-${{ needs.validate.outputs.pr_number }}
-      acr_name: ${{ needs.infrastructure.outputs.acrName }}
+      acr_name: acrsswrulesstaging
       image_name: ssw-rules
       source_branch: ${{ needs.validate.outputs.head_ref }}
       tina_branch: ${{ needs.validate.outputs.head_ref }}
@@ -121,6 +127,17 @@ jobs:
     needs: [validate, infrastructure, build]
     environment: staging
     steps:
+      - name: Check that the image is in the registry of the infrastructure
+        env:
+          IMAGE: ${{ needs.build.outputs.image_tag }}
+          ACR_LOGIN_SERVER: ${{ needs.infrastructure.outputs.acrLoginServer }}
+        run: |
+          if [[ -z "$ACR_LOGIN_SERVER" || "$IMAGE" != "$ACR_LOGIN_SERVER/"* ]]; then
+            echo "::error::The build job pushed the image '$IMAGE', but the registry of the infrastructure is '$ACR_LOGIN_SERVER'. Make the acr_name value of the build job agree with the registry name in infra/deploy.ps1."
+            exit 1
+          fi
+          echo "The image $IMAGE is in the registry of the infrastructure ($ACR_LOGIN_SERVER)"
+
       - name: Use build info from build job
         id: build_info
         run: |