#!/usr/bin/env bash # Fail a client job before it starts when the runner cannot hold it. # # Every client job here compiles the frontend from cold: `ng build # --configuration production` peaks at 3.8 GB resident on a 4 vCPU box # (measured with no .angular/cache present, which is what a fresh runner # has), and the Cypress and Lighthouse jobs add a dev server and a browser on # top of that. On a 4 GB runner the kernel OOM killer takes the build part way # through, which surfaces as `exitcode '137'` at `ng build`, or as Karma # disconnecting mid-suite ("no message in 30000 ms"). Both read as a flaky # test rather than as an undersized box, and both cost a whole CI slot. # # 8 GB is the minimum supported runner size: it leaves roughly 2x headroom # over that peak. This check runs before the installs, so an undersized runner # fails in seconds with the numbers in the log. # # The floor is 7 GB rather than 8192 MB because an 8 GB box does not report 8 # GB: the fleet's report 7751 MB total, the rest going to firmware and the # kernel. A 4 GB box reports under 4000 MB, so 7 GB separates the two sizes # with room for a box that reserves a little more than ours do. # # What this does not catch is oversubscription: two client jobs landing on one # box. A box that passes the floor can still OOM a build when another job is # resident, and MemAvailable at start is the only sign of it - that is reported # as a warning, not a failure, because a busy box is transient where a small # box is not. Keep one client job per runner. # # Usage: check-runner-memory.sh [min_total_mb] # or: MIN_TOTAL_MB=4096 check-runner-memory.sh set -euo pipefail MIN_TOTAL_MB="${1:-${MIN_TOTAL_MB:-7000}}" # A typo here (8GB, a stray flag, an empty variable) would otherwise reach the # integer test under `set -e` and abort with "integer expression expected". case "$MIN_TOTAL_MB" in ''|*[!0-9]*) echo "usage: check-runner-memory.sh [min_total_mb] - got '$MIN_TOTAL_MB'" >&2 exit 2 ;; esac meminfo_field() { awk -v key="$1:" '$1 == key { print $2 }' /proc/meminfo } total_kb=$(meminfo_field MemTotal || true) avail_kb=$(meminfo_field MemAvailable || true) cpus=$(nproc 2>/dev/null || echo '?') if [ -z "$total_kb" ]; then echo "runner memory: /proc/meminfo has no MemTotal - cannot measure, continuing" exit 0 fi total_mb=$(( total_kb / 1024 )) avail_mb=$(( ${avail_kb:-0} / 1024 )) # Older kernels have no MemAvailable; report that rather than 0, which would # read as a permanently exhausted runner. if [ -n "$avail_kb" ]; then avail_display="${avail_mb} MB available" else avail_display="available unknown" fi # A container usually sees the host's RAM, but may carry its own lower ceiling # (cgroup v2 memory.max, v1 memory.limit_in_bytes). That ceiling is what the # build is actually killed against, so it counts as the runner's size. limit_mb="" for f in /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory/memory.limit_in_bytes; do [ -r "$f" ] || continue v=$(cat "$f") case "$v" in ''|max|*[!0-9]*) continue ;; esac # An unlimited v1 cgroup reports a sentinel near 2^63 bytes. [ "$v" -lt 100000000000 ] || continue candidate=$(( v / 1024 / 1024 )) if [ -z "$limit_mb" ] || [ "$candidate" -lt "$limit_mb" ]; then limit_mb="$candidate" fi done echo "runner memory: ${total_mb} MB total, ${avail_display}, ${cpus} vCPU${limit_mb:+, ${limit_mb} MB cgroup limit}" effective_mb=$total_mb if [ -n "$limit_mb" ] && [ "$limit_mb" -lt "$effective_mb" ]; then effective_mb=$limit_mb fi if [ "$effective_mb" -lt "$MIN_TOTAL_MB" ]; then echo "::error::runner has ${effective_mb} MB of memory, a client job needs at least ${MIN_TOTAL_MB} MB (8 GB is the supported minimum; a cold production build alone peaks at about 3.8 GB, and the Cypress and Lighthouse jobs add a dev server and a browser to that)." exit 1 fi # Not a gate - a second job landing on the same box shows up here first. if [ -n "$avail_kb" ] && [ "$avail_mb" -lt 5000 ]; then echo "::warning::only ${avail_mb} MB is available on this runner; another job may be running on the same box" fi