dc
f3d85d49bf
A MIN_TOTAL_MB typo reached the integer test under set -e and aborted with "integer expression expected"; it now prints usage and exits 2. A kernel without MemAvailable no longer reports 0 MB available in the summary or the warning, so the oversubscription warning stays meaningful.
100 lines
4.0 KiB
Bash
Executable File
100 lines
4.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Fail a client job before it starts when the runner cannot hold it.
|
|
#
|
|
# Every client job here compiles the frontend from cold: `ng build
|
|
# --configuration production` peaks at 3.8 GB resident on a 4 vCPU box
|
|
# (measured with no .angular/cache present, which is what a fresh runner
|
|
# has), and the Cypress and Lighthouse jobs add a dev server and a browser on
|
|
# top of that. On a 4 GB runner the kernel OOM killer takes the build part way
|
|
# through, which surfaces as `exitcode '137'` at `ng build`, or as Karma
|
|
# disconnecting mid-suite ("no message in 30000 ms"). Both read as a flaky
|
|
# test rather than as an undersized box, and both cost a whole CI slot.
|
|
#
|
|
# 8 GB is the minimum supported runner size: it leaves roughly 2x headroom
|
|
# over that peak. This check runs before the installs, so an undersized runner
|
|
# fails in seconds with the numbers in the log.
|
|
#
|
|
# The floor is 7 GB rather than 8192 MB because an 8 GB box does not report 8
|
|
# GB: the fleet's report 7751 MB total, the rest going to firmware and the
|
|
# kernel. A 4 GB box reports under 4000 MB, so 7 GB separates the two sizes
|
|
# with room for a box that reserves a little more than ours do.
|
|
#
|
|
# What this does not catch is oversubscription: two client jobs landing on one
|
|
# box. A box that passes the floor can still OOM a build when another job is
|
|
# resident, and MemAvailable at start is the only sign of it - that is reported
|
|
# as a warning, not a failure, because a busy box is transient where a small
|
|
# box is not. Keep one client job per runner.
|
|
#
|
|
# Usage: check-runner-memory.sh [min_total_mb]
|
|
# or: MIN_TOTAL_MB=4096 check-runner-memory.sh
|
|
set -euo pipefail
|
|
|
|
MIN_TOTAL_MB="${1:-${MIN_TOTAL_MB:-7000}}"
|
|
|
|
# A typo here (8GB, a stray flag, an empty variable) would otherwise reach the
|
|
# integer test under `set -e` and abort with "integer expression expected".
|
|
case "$MIN_TOTAL_MB" in
|
|
''|*[!0-9]*)
|
|
echo "usage: check-runner-memory.sh [min_total_mb] - got '$MIN_TOTAL_MB'" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
meminfo_field() {
|
|
awk -v key="$1:" '$1 == key { print $2 }' /proc/meminfo
|
|
}
|
|
|
|
total_kb=$(meminfo_field MemTotal || true)
|
|
avail_kb=$(meminfo_field MemAvailable || true)
|
|
cpus=$(nproc 2>/dev/null || echo '?')
|
|
|
|
if [ -z "$total_kb" ]; then
|
|
echo "runner memory: /proc/meminfo has no MemTotal - cannot measure, continuing"
|
|
exit 0
|
|
fi
|
|
|
|
total_mb=$(( total_kb / 1024 ))
|
|
avail_mb=$(( ${avail_kb:-0} / 1024 ))
|
|
# Older kernels have no MemAvailable; report that rather than 0, which would
|
|
# read as a permanently exhausted runner.
|
|
if [ -n "$avail_kb" ]; then
|
|
avail_display="${avail_mb} MB available"
|
|
else
|
|
avail_display="available unknown"
|
|
fi
|
|
|
|
# A container usually sees the host's RAM, but may carry its own lower ceiling
|
|
# (cgroup v2 memory.max, v1 memory.limit_in_bytes). That ceiling is what the
|
|
# build is actually killed against, so it counts as the runner's size.
|
|
limit_mb=""
|
|
for f in /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory/memory.limit_in_bytes; do
|
|
[ -r "$f" ] || continue
|
|
v=$(cat "$f")
|
|
case "$v" in
|
|
''|max|*[!0-9]*) continue ;;
|
|
esac
|
|
# An unlimited v1 cgroup reports a sentinel near 2^63 bytes.
|
|
[ "$v" -lt 100000000000 ] || continue
|
|
candidate=$(( v / 1024 / 1024 ))
|
|
if [ -z "$limit_mb" ] || [ "$candidate" -lt "$limit_mb" ]; then
|
|
limit_mb="$candidate"
|
|
fi
|
|
done
|
|
|
|
echo "runner memory: ${total_mb} MB total, ${avail_display}, ${cpus} vCPU${limit_mb:+, ${limit_mb} MB cgroup limit}"
|
|
|
|
effective_mb=$total_mb
|
|
if [ -n "$limit_mb" ] && [ "$limit_mb" -lt "$effective_mb" ]; then
|
|
effective_mb=$limit_mb
|
|
fi
|
|
|
|
if [ "$effective_mb" -lt "$MIN_TOTAL_MB" ]; then
|
|
echo "::error::runner has ${effective_mb} MB of memory, a client job needs at least ${MIN_TOTAL_MB} MB (8 GB is the supported minimum; a cold production build alone peaks at about 3.8 GB, and the Cypress and Lighthouse jobs add a dev server and a browser to that)."
|
|
exit 1
|
|
fi
|
|
|
|
# Not a gate - a second job landing on the same box shows up here first.
|
|
if [ -n "$avail_kb" ] && [ "$avail_mb" -lt 5000 ]; then
|
|
echo "::warning::only ${avail_mb} MB is available on this runner; another job may be running on the same box"
|
|
fi
|