From ab4f4c9b602709d7c4a4cda309fa95df0b1daaaa Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Sun, 1 Nov 2026 15:19:31 +0000
Subject: [PATCH 1/3] ci: fail a client job fast on a runner too small for it

An undersized runner does not fail honestly: the kernel OOM killer takes
the build, so the job reports exitcode 137 at ng build, or Karma
disconnects mid-suite after twenty minutes. Both read as a flaky test
rather than as a box that cannot hold the job, and both cost a whole CI
slot.

The check runs first in the three client jobs (Build-and-ng-test,
Build-and-test-development, lighthouse), prints total and available RAM,
vCPU count and any cgroup limit, and exits 1 when the effective size is
under 8 GB - the supported minimum. A cold production build measures
3.8 GB peak on its own, so 8 GB leaves roughly 2x headroom.
---
 .gitea/scripts/check-runner-memory.sh | 77 +++++++++++++++++++++++++++
 .gitea/workflows/build.yaml           | 10 ++++
 .gitea/workflows/lighthouse.yaml      |  6 +++
 3 files changed, 93 insertions(+)
 create mode 100755 .gitea/scripts/check-runner-memory.sh

diff --git a/.gitea/scripts/check-runner-memory.sh b/.gitea/scripts/check-runner-memory.sh
new file mode 100755
index 0000000..5f28f94
--- /dev/null
+++ b/.gitea/scripts/check-runner-memory.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Fail a client job before it starts when the runner cannot hold it.
+#
+# Every client job here compiles the frontend from cold: `ng build
+# --configuration production` peaks at 3.8 GB resident on a 4 vCPU box
+# (measured with no .angular/cache present, which is what a fresh runner
+# has), and the Cypress and Lighthouse jobs add a dev server and a browser on
+# top of that. On a 4 GB runner the kernel OOM killer takes the build part way
+# through, which surfaces as `exitcode '137'` at `ng build`, or as Karma
+# disconnecting mid-suite ("no message in 30000 ms"). Both read as a flaky
+# test rather than as an undersized box, and both cost a whole CI slot.
+#
+# 8 GB is the minimum supported runner size: it leaves roughly 2x headroom
+# over that peak. This check runs before the installs, so an undersized runner
+# fails in seconds with the numbers in the log.
+#
+# The floor is 7 GB rather than 8192 MB because an 8 GB box does not report 8
+# GB: the fleet's report 7751 MB total, the rest going to firmware and the
+# kernel. A 4 GB box reports under 4000 MB, so 7 GB separates the two sizes
+# with room for a box that reserves a little more than ours do.
+#
+# Usage: check-runner-memory.sh [min_total_mb]
+#   or:  MIN_TOTAL_MB=4096 check-runner-memory.sh
+set -euo pipefail
+
+MIN_TOTAL_MB="${1:-${MIN_TOTAL_MB:-7000}}"
+
+meminfo_field() {
+  awk -v key="$1:" '$1 == key { print $2 }' /proc/meminfo
+}
+
+total_kb=$(meminfo_field MemTotal || true)
+avail_kb=$(meminfo_field MemAvailable || true)
+cpus=$(nproc 2>/dev/null || echo '?')
+
+if [ -z "$total_kb" ]; then
+  echo "runner memory: /proc/meminfo has no MemTotal - cannot measure, continuing"
+  exit 0
+fi
+
+total_mb=$(( total_kb / 1024 ))
+avail_mb=$(( ${avail_kb:-0} / 1024 ))
+
+# A container usually sees the host's RAM, but may carry its own lower ceiling
+# (cgroup v2 memory.max, v1 memory.limit_in_bytes). That ceiling is what the
+# build is actually killed against, so it counts as the runner's size.
+limit_mb=""
+for f in /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory/memory.limit_in_bytes; do
+  [ -r "$f" ] || continue
+  v=$(cat "$f")
+  case "$v" in
+    ''|max|*[!0-9]*) continue ;;
+  esac
+  # An unlimited v1 cgroup reports a sentinel near 2^63 bytes.
+  [ "$v" -lt 100000000000 ] || continue
+  candidate=$(( v / 1024 / 1024 ))
+  if [ -z "$limit_mb" ] || [ "$candidate" -lt "$limit_mb" ]; then
+    limit_mb="$candidate"
+  fi
+done
+
+echo "runner memory: ${total_mb} MB total, ${avail_mb} MB available, ${cpus} vCPU${limit_mb:+, ${limit_mb} MB cgroup limit}"
+
+effective_mb=$total_mb
+if [ -n "$limit_mb" ] && [ "$limit_mb" -lt "$effective_mb" ]; then
+  effective_mb=$limit_mb
+fi
+
+if [ "$effective_mb" -lt "$MIN_TOTAL_MB" ]; then
+  echo "::error::runner has ${effective_mb} MB of memory, a client job needs at least ${MIN_TOTAL_MB} MB (8 GB is the supported minimum; a cold production build alone peaks at about 3.8 GB, and the Cypress and Lighthouse jobs add a dev server and a browser to that)."
+  exit 1
+fi
+
+# Not a gate - a second job landing on the same box shows up here first.
+if [ "$avail_mb" -lt 5000 ]; then
+  echo "::warning::only ${avail_mb} MB is available on this runner; another job may be running on the same box"
+fi
diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml
index 26a60e4..5a445cc 100644
--- a/.gitea/workflows/build.yaml
+++ b/.gitea/workflows/build.yaml
@@ -13,6 +13,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Check runner size
+        # An undersized runner dies inside `ng build` (exitcode 137) or as a
+        # Karma disconnect twenty minutes in. Fail here instead, in seconds,
+        # with the measured numbers in the log - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
@@ -147,6 +152,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Check runner size
+        # Each Cypress job compiles the frontend and runs a browser on top; a
+        # runner too small for that fails as a flaky-looking disconnect - see
+        # the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
diff --git a/.gitea/workflows/lighthouse.yaml b/.gitea/workflows/lighthouse.yaml
index 35eba28..5507598 100644
--- a/.gitea/workflows/lighthouse.yaml
+++ b/.gitea/workflows/lighthouse.yaml
@@ -14,6 +14,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Check runner size
+        # This job runs a production build, then a SASjs server, a dev server
+        # and Chrome. An undersized runner dies inside the build with exitcode
+        # 137; fail here in seconds instead - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
+
       - name: Use Node.js ${{ env.NODE_VERSION }}
         uses: actions/setup-node@v4
         with:

From b1892ebb986e01200d640bcc83cdd3e728734f9e Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Sun, 1 Nov 2026 16:24:03 +0000
Subject: [PATCH 2/3] ci: gate every job that builds the frontend, and scope
 the check in its header

release.yaml (all three jobs), sites.yaml and release-assets.yaml run the same cold production build that the review jobs do, so they get the same pre-install check.

The header now says what the check rules out (an undersized box) and what it does not (two jobs sharing one box), so the MemAvailable warning is not read as a gate.
---
 .gitea/scripts/check-runner-memory.sh |  6 ++++++
 .gitea/workflows/release-assets.yaml  |  4 ++++
 .gitea/workflows/release.yaml         | 13 +++++++++++++
 .gitea/workflows/sites.yaml           |  4 ++++
 4 files changed, 27 insertions(+)

diff --git a/.gitea/scripts/check-runner-memory.sh b/.gitea/scripts/check-runner-memory.sh
index 5f28f94..c9377f9 100755
--- a/.gitea/scripts/check-runner-memory.sh
+++ b/.gitea/scripts/check-runner-memory.sh
@@ -19,6 +19,12 @@
 # kernel. A 4 GB box reports under 4000 MB, so 7 GB separates the two sizes
 # with room for a box that reserves a little more than ours do.
 #
+# What this does not catch is oversubscription: two client jobs landing on one
+# box. A box that passes the floor can still OOM a build when another job is
+# resident, and MemAvailable at start is the only sign of it - that is reported
+# as a warning, not a failure, because a busy box is transient where a small
+# box is not. Keep one client job per runner.
+#
 # Usage: check-runner-memory.sh [min_total_mb]
 #   or:  MIN_TOTAL_MB=4096 check-runner-memory.sh
 set -euo pipefail
diff --git a/.gitea/workflows/release-assets.yaml b/.gitea/workflows/release-assets.yaml
index 0667221..57b4ccc 100644
--- a/.gitea/workflows/release-assets.yaml
+++ b/.gitea/workflows/release-assets.yaml
@@ -34,6 +34,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: Check runner size
+        # Builds the same frontend as the release jobs; an undersized runner
+        # fails inside the build - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
diff --git a/.gitea/workflows/release.yaml b/.gitea/workflows/release.yaml
index 05310dc..eafa34b 100644
--- a/.gitea/workflows/release.yaml
+++ b/.gitea/workflows/release.yaml
@@ -18,6 +18,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Check runner size
+        # A release job that OOMs costs the tag as well as the slot; fail in
+        # seconds on an undersized runner instead of twenty minutes in - see
+        # the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
@@ -91,6 +96,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Check runner size
+        # The development build plus a browser: same undersized-runner failure
+        # as the review jobs, on the path that makes the tag - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
@@ -257,6 +266,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Check runner size
+        # This job builds the frontend twice (webdoc and the release check);
+        # both die the same way on an undersized runner - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
diff --git a/.gitea/workflows/sites.yaml b/.gitea/workflows/sites.yaml
index 5b9c956..bb2a92f 100644
--- a/.gitea/workflows/sites.yaml
+++ b/.gitea/workflows/sites.yaml
@@ -38,6 +38,10 @@ jobs:
           # The tag is what the client's build stamp describes (src/version.ts
           # runs git-describe), so the demo build needs the tags in the clone.
           fetch-depth: 0
+      - name: Check runner size
+        # The demo build is the same cold production build; an undersized
+        # runner fails inside it - see the script.
+        run: bash .gitea/scripts/check-runner-memory.sh
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}

From f3d85d49bfb6c39d6df244dded54a38b0afa3c61 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Sun, 1 Nov 2026 18:06:28 +0000
Subject: [PATCH 3/3] ci: make the memory check report its own input errors
 clearly

A MIN_TOTAL_MB typo reached the integer test under set -e and aborted with "integer expression expected"; it now prints usage and exits 2. A kernel without MemAvailable no longer reports 0 MB available in the summary or the warning, so the oversubscription warning stays meaningful.
---
 .gitea/scripts/check-runner-memory.sh | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/.gitea/scripts/check-runner-memory.sh b/.gitea/scripts/check-runner-memory.sh
index c9377f9..d8ba00b 100755
--- a/.gitea/scripts/check-runner-memory.sh
+++ b/.gitea/scripts/check-runner-memory.sh
@@ -31,6 +31,15 @@ set -euo pipefail
 
 MIN_TOTAL_MB="${1:-${MIN_TOTAL_MB:-7000}}"
 
+# A typo here (8GB, a stray flag, an empty variable) would otherwise reach the
+# integer test under `set -e` and abort with "integer expression expected".
+case "$MIN_TOTAL_MB" in
+  ''|*[!0-9]*)
+    echo "usage: check-runner-memory.sh [min_total_mb] - got '$MIN_TOTAL_MB'" >&2
+    exit 2
+    ;;
+esac
+
 meminfo_field() {
   awk -v key="$1:" '$1 == key { print $2 }' /proc/meminfo
 }
@@ -46,6 +55,13 @@ fi
 
 total_mb=$(( total_kb / 1024 ))
 avail_mb=$(( ${avail_kb:-0} / 1024 ))
+# Older kernels have no MemAvailable; report that rather than 0, which would
+# read as a permanently exhausted runner.
+if [ -n "$avail_kb" ]; then
+  avail_display="${avail_mb} MB available"
+else
+  avail_display="available unknown"
+fi
 
 # A container usually sees the host's RAM, but may carry its own lower ceiling
 # (cgroup v2 memory.max, v1 memory.limit_in_bytes). That ceiling is what the
@@ -65,7 +81,7 @@ for f in /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory/memory.limit_in_bytes;
   fi
 done
 
-echo "runner memory: ${total_mb} MB total, ${avail_mb} MB available, ${cpus} vCPU${limit_mb:+, ${limit_mb} MB cgroup limit}"
+echo "runner memory: ${total_mb} MB total, ${avail_display}, ${cpus} vCPU${limit_mb:+, ${limit_mb} MB cgroup limit}"
 
 effective_mb=$total_mb
 if [ -n "$limit_mb" ] && [ "$limit_mb" -lt "$effective_mb" ]; then
@@ -78,6 +94,6 @@ if [ "$effective_mb" -lt "$MIN_TOTAL_MB" ]; then
 fi
 
 # Not a gate - a second job landing on the same box shows up here first.
-if [ "$avail_mb" -lt 5000 ]; then
+if [ -n "$avail_kb" ] && [ "$avail_mb" -lt 5000 ]; then
   echo "::warning::only ${avail_mb} MB is available on this runner; another job may be running on the same box"
 fi