Merge pull request 'ci(build): split the Cypress suite into four shards, and keep videos for failures' (#392) from ci/parallel-cypress into fix/va-parmap-review-572

Reviewed-on: #392

.gitea/scripts/cypress-shards.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Which Cypress specs run in which CI shard.
+#
+# One list, because the workflow used to spell the specs out in a single
+# --spec string and five files quietly fell out of it - including the VA embed
+# suite.  The guard step in Build-and-ng-test compares this against the specs on
+# disk and fails when a file belongs to no shard.
+#
+# Balance is on test count AND spec count: each spec pays a few seconds of
+# startup, so seventeen one-test files cost more than their test count suggests.
+set -euo pipefail
+
+shard_specs() {
+  case "$1" in
+    1) echo "editor csv metanav" ;;
+    2) echo "excel filtering download browser-info excel-multi-load hook-programs" ;;
+    3) echo "embed-va viewbox licensing filter-panel liveness" ;;
+    4) echo "viewer-labels row-cell-limits derived-variables ddl-export coltype-delete-record submitted-details startupservice-launcher stage full-table-search csv-limited" ;;
+    *) echo "unknown shard: $1" >&2; return 1 ;;
+  esac
+}
+
+all_specs() {
+  local shard
+  for shard in 1 2 3 4; do shard_specs "$shard"; done
+}
+
+spec_paths() {
+  local name
+  for name in $1; do printf 'cypress/e2e/%s.cy.ts,' "$name"; done
+}
+
+case "${1:-}" in
+  list)
+    # comma separated, the form `cypress run --spec` takes
+    spec_paths "$(shard_specs "$2")" | sed 's/,$//'
+    ;;
+  all)
+    spec_paths "$(all_specs)" | sed 's/,$//'
+    ;;
+  coverage)
+    # every spec on disk must appear exactly once across the shards
+    on_disk=$(cd "$2/client" && ls cypress/e2e/*.cy.ts | xargs -n1 basename | sed 's/\.cy\.ts$//' | sort)
+    assigned=$(printf '%s\n' $(all_specs) | sort)
+    missing=$(comm -23 <(echo "$on_disk") <(echo "$assigned"))
+    duplicated=$(echo "$assigned" | uniq -d)
+    if [ -n "$missing" ]; then
+      echo "Specs that belong to no shard:"
+      echo "$missing"
+    fi
+    if [ -n "$duplicated" ]; then
+      echo "Specs assigned to more than one shard:"
+      echo "$duplicated"
+    fi
+    if [ -n "$missing$duplicated" ]; then
+      echo "Add them to a shard in .gitea/scripts/cypress-shards.sh"
+      exit 1
+    fi
+    echo "All $(echo "$on_disk" | wc -l) specs are assigned to exactly one shard."
+    ;;
+  *)
+    echo "usage: $0 {list <shard>|all|coverage <repo-root>}" >&2
+    exit 2
+    ;;
+esac

.gitea/workflows/build.yaml

@@ -85,12 +85,23 @@ jobs:
           npm run test:tz-west
           npm run test:tz-east
 
+      - name: Every Cypress spec belongs to a shard
+        # The spec lists used to be spelled out in one --spec string, and five
+        # files quietly fell out of it - among them the VA embed suite.  Fails
+        # when a spec on disk appears in no shard, or in more than one.
+        run: bash .gitea/scripts/cypress-shards.sh coverage .
+
       - name: Production Build
         run: |
           cd client
           npm run build
 
-  Build-and-test-development:
+
+  # Cypress shard 1 of 4.  The setup repeats in every shard because each
+  # one gets its own runner and its own SASjs server; that fixed cost is what
+  # limits how far sharding helps.  The specs it runs come from
+  # .gitea/scripts/cypress-shards.sh.
+  cypress-shard-1:
     runs-on: ubuntu-latest
     needs: Build-and-ng-test
     env:
@@ -98,8 +109,9 @@ jobs:
       # Pin OS locale
       LANG: en_GB.UTF-8
       LC_ALL: en_GB.UTF-8
+      SHARD: '1'
+    steps: &dev-steps
 
-    steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
@@ -195,6 +207,14 @@ jobs:
         timeout-minutes: 50
         run: |
           cd ./client
+          # The spec list lives in one place, and a shard that resolves to nothing
+          # is a broken workflow rather than a green run with no tests.
+          SPECS="$(bash ../.gitea/scripts/cypress-shards.sh list "$SHARD")"
+          if [ -z "$SPECS" ]; then
+            echo "shard $SHARD resolved to no specs"
+            exit 1
+          fi
+          echo "shard $SHARD: $SPECS"
           mv ./cypress.env.example.json ./cypress.env.json
           replace-in-files --regex='"username".*' --replacement='"username":"'${{ secrets.CYPRESS_USERNAME_SASJS }}'",' ./cypress.env.json
           replace-in-files --regex='"password".*' --replacement='"password":"'${{ secrets.CYPRESS_PWD_SASJS }}'" ' ./cypress.env.json
@@ -206,20 +226,98 @@ jobs:
           replace-in-files --regex='serverType=".*?"' --replacement='serverType="SASJS"' ./src/index.html
           replace-in-files --regex='"hosturl".*' --replacement='hosturl:"http://localhost:4200",' ./cypress.config.ts
           cat ./cypress.config.ts
-          # Start frontend and run cypress
+          # Start frontend and run this shard's specs
           # timeout 2700: SIGTERM after 45 min so Cypress can flush video/screenshots
           # before the outer timeout-minutes hard-kills the step (avoids silent multi-hour hangs)
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 2700 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts,cypress/e2e/derived-variables.cy.ts,cypress/e2e/startupservice-launcher.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 2700 npx cypress run --browser chrome --spec "$SPECS"
 
       - name: Zip Cypress videos
-        if: always()
+        # Only on a failure: a green run has nothing to watch, and zipping and
+        # uploading every run's videos is the part of the video cost that lands
+        # on the clock.
+        if: failure()
         run: |
           mkdir -p ./client/cypress/videos
           zip -r cypress-videos ./client/cypress/videos
 
       - name: Add cypress videos artifacts
-        if: always()
-        uses: actions/upload-artifact@v3
+        if: failure()
+        uses: actions/upload-artifact@v4
         with:
-          name: cypress-videos.zip
+          name: cypress-videos-${{ env.SHARD }}
           path: cypress-videos.zip
+
+
+  # Cypress shard 2 of 4.  The setup repeats in every shard because each
+  # one gets its own runner and its own SASjs server; that fixed cost is what
+  # limits how far sharding helps.  The specs it runs come from
+  # .gitea/scripts/cypress-shards.sh.
+  cypress-shard-2:
+    runs-on: ubuntu-latest
+    needs: Build-and-ng-test
+    env:
+      CHROME_BIN: /usr/bin/google-chrome
+      # Pin OS locale
+      LANG: en_GB.UTF-8
+      LC_ALL: en_GB.UTF-8
+      SHARD: '2'
+    steps: *dev-steps
+
+
+  # Cypress shard 3 of 4.  The setup repeats in every shard because each
+  # one gets its own runner and its own SASjs server; that fixed cost is what
+  # limits how far sharding helps.  The specs it runs come from
+  # .gitea/scripts/cypress-shards.sh.
+  cypress-shard-3:
+    runs-on: ubuntu-latest
+    needs: Build-and-ng-test
+    env:
+      CHROME_BIN: /usr/bin/google-chrome
+      # Pin OS locale
+      LANG: en_GB.UTF-8
+      LC_ALL: en_GB.UTF-8
+      SHARD: '3'
+    steps: *dev-steps
+
+
+  # Cypress shard 4 of 4.  The setup repeats in every shard because each
+  # one gets its own runner and its own SASjs server; that fixed cost is what
+  # limits how far sharding helps.  The specs it runs come from
+  # .gitea/scripts/cypress-shards.sh.
+  cypress-shard-4:
+    runs-on: ubuntu-latest
+    needs: Build-and-ng-test
+    env:
+      CHROME_BIN: /usr/bin/google-chrome
+      # Pin OS locale
+      LANG: en_GB.UTF-8
+      LC_ALL: en_GB.UTF-8
+      SHARD: '4'
+    steps: *dev-steps
+  # The status check that branch protection requires keeps its old name, but it
+  # is produced here now, so a red shard cannot hide behind a green one.
+  #
+  # if: always() is what makes that true: without it a failed dependency skips
+  # this job, and a skipped job is not a failure on the one check that is
+  # watched - the gate would be silent in exactly the case it exists for.
+  Build-and-test-development:
+    runs-on: ubuntu-latest
+    needs: [cypress-shard-1, cypress-shard-2, cypress-shard-3, cypress-shard-4]
+    if: always()
+    steps:
+      - name: Every Cypress shard passed
+        run: |
+          results="${{ needs.cypress-shard-1.result }} ${{ needs.cypress-shard-2.result }} ${{ needs.cypress-shard-3.result }} ${{ needs.cypress-shard-4.result }}"
+          echo "shard results: $results"
+          set -- $results
+          if [ "$#" -ne 4 ]; then
+            echo "expected 4 shard results, got $# - the needs context did not resolve" >&2
+            exit 1
+          fi
+          for result in "$@"; do
+            if [ "$result" != "success" ]; then
+              echo "a Cypress shard did not succeed: $results" >&2
+              exit 1
+            fi
+          done
+          echo "all 4 shards passed"