Merge pull request 'ci: fail a client job fast on a runner too small for it' (#429) from ci/runner-memory-guard into release/7.22

Reviewed-on: #429

.gitea/scripts/check-runner-memory.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Fail a client job before it starts when the runner cannot hold it.
+#
+# Every client job here compiles the frontend from cold: `ng build
+# --configuration production` peaks at 3.8 GB resident on a 4 vCPU box
+# (measured with no .angular/cache present, which is what a fresh runner
+# has), and the Cypress and Lighthouse jobs add a dev server and a browser on
+# top of that. On a 4 GB runner the kernel OOM killer takes the build part way
+# through, which surfaces as `exitcode '137'` at `ng build`, or as Karma
+# disconnecting mid-suite ("no message in 30000 ms"). Both read as a flaky
+# test rather than as an undersized box, and both cost a whole CI slot.
+#
+# 8 GB is the minimum supported runner size: it leaves roughly 2x headroom
+# over that peak. This check runs before the installs, so an undersized runner
+# fails in seconds with the numbers in the log.
+#
+# The floor is 7 GB rather than 8192 MB because an 8 GB box does not report 8
+# GB: the fleet's report 7751 MB total, the rest going to firmware and the
+# kernel. A 4 GB box reports under 4000 MB, so 7 GB separates the two sizes
+# with room for a box that reserves a little more than ours do.
+#
+# What this does not catch is oversubscription: two client jobs landing on one
+# box. A box that passes the floor can still OOM a build when another job is
+# resident, and MemAvailable at start is the only sign of it - that is reported
+# as a warning, not a failure, because a busy box is transient where a small
+# box is not. Keep one client job per runner.
+#
+# Usage: check-runner-memory.sh [min_total_mb]
+#   or:  MIN_TOTAL_MB=4096 check-runner-memory.sh
+set -euo pipefail
+
+MIN_TOTAL_MB="${1:-${MIN_TOTAL_MB:-7000}}"
+
+# A typo here (8GB, a stray flag, an empty variable) would otherwise reach the
+# integer test under `set -e` and abort with "integer expression expected".
+case "$MIN_TOTAL_MB" in
+  ''|*[!0-9]*)
+    echo "usage: check-runner-memory.sh [min_total_mb] - got '$MIN_TOTAL_MB'" >&2
+    exit 2
+    ;;
+esac
+
+meminfo_field() {
+  awk -v key="$1:" '$1 == key { print $2 }' /proc/meminfo
+}
+
+total_kb=$(meminfo_field MemTotal || true)
+avail_kb=$(meminfo_field MemAvailable || true)
+cpus=$(nproc 2>/dev/null || echo '?')
+
+if [ -z "$total_kb" ]; then
+  echo "runner memory: /proc/meminfo has no MemTotal - cannot measure, continuing"
+  exit 0
+fi
+
+total_mb=$(( total_kb / 1024 ))
+avail_mb=$(( ${avail_kb:-0} / 1024 ))
+# Older kernels have no MemAvailable; report that rather than 0, which would
+# read as a permanently exhausted runner.
+if [ -n "$avail_kb" ]; then
+  avail_display="${avail_mb} MB available"
+else
+  avail_display="available unknown"
+fi
+
+# A container usually sees the host's RAM, but may carry its own lower ceiling
+# (cgroup v2 memory.max, v1 memory.limit_in_bytes). That ceiling is what the
+# build is actually killed against, so it counts as the runner's size.
+limit_mb=""
+for f in /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory/memory.limit_in_bytes; do
+  [ -r "$f" ] || continue
+  v=$(cat "$f")
+  case "$v" in
+    ''|max|*[!0-9]*) continue ;;
+  esac
+  # An unlimited v1 cgroup reports a sentinel near 2^63 bytes.
+  [ "$v" -lt 100000000000 ] || continue
+  candidate=$(( v / 1024 / 1024 ))
+  if [ -z "$limit_mb" ] || [ "$candidate" -lt "$limit_mb" ]; then
+    limit_mb="$candidate"
+  fi
+done
+
+echo "runner memory: ${total_mb} MB total, ${avail_display}, ${cpus} vCPU${limit_mb:+, ${limit_mb} MB cgroup limit}"
+
+effective_mb=$total_mb
+if [ -n "$limit_mb" ] && [ "$limit_mb" -lt "$effective_mb" ]; then
+  effective_mb=$limit_mb
+fi
+
+if [ "$effective_mb" -lt "$MIN_TOTAL_MB" ]; then
+  echo "::error::runner has ${effective_mb} MB of memory, a client job needs at least ${MIN_TOTAL_MB} MB (8 GB is the supported minimum; a cold production build alone peaks at about 3.8 GB, and the Cypress and Lighthouse jobs add a dev server and a browser to that)."
+  exit 1
+fi
+
+# Not a gate - a second job landing on the same box shows up here first.
+if [ -n "$avail_kb" ] && [ "$avail_mb" -lt 5000 ]; then
+  echo "::warning::only ${avail_mb} MB is available on this runner; another job may be running on the same box"
+fi

.gitea/workflows/build.yaml

@@ -13,6 +13,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Check runner size
+        # An undersized runner dies inside `ng build` (exitcode 137) or as a
+        # Karma disconnect twenty minutes in. Fail here instead, in seconds,
+        # with the measured numbers in the log - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
@@ -147,6 +152,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Check runner size
+        # Each Cypress job compiles the frontend and runs a browser on top; a
+        # runner too small for that fails as a flaky-looking disconnect - see
+        # the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}

.gitea/workflows/lighthouse.yaml

@@ -14,6 +14,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Check runner size
+        # This job runs a production build, then a SASjs server, a dev server
+        # and Chrome. An undersized runner dies inside the build with exitcode
+        # 137; fail here in seconds instead - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
+
       - name: Use Node.js ${{ env.NODE_VERSION }}
         uses: actions/setup-node@v4
         with:

.gitea/workflows/release-assets.yaml

@@ -34,6 +34,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: Check runner size
+        # Builds the same frontend as the release jobs; an undersized runner
+        # fails inside the build - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}

.gitea/workflows/release.yaml

@@ -18,6 +18,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Check runner size
+        # A release job that OOMs costs the tag as well as the slot; fail in
+        # seconds on an undersized runner instead of twenty minutes in - see
+        # the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
@@ -91,6 +96,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Check runner size
+        # The development build plus a browser: same undersized-runner failure
+        # as the review jobs, on the path that makes the tag - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
@@ -257,6 +266,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Check runner size
+        # This job builds the frontend twice (webdoc and the release check);
+        # both die the same way on an undersized runner - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}

.gitea/workflows/sites.yaml

@@ -38,6 +38,10 @@ jobs:
           # The tag is what the client's build stamp describes (src/version.ts
           # runs git-describe), so the demo build needs the tags in the clone.
           fetch-depth: 0
+      - name: Check runner size
+        # The demo build is the same cold production build; an undersized
+        # runner fails inside it - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}