dc/.gitea/workflows/release.yaml

name: Release
run-name: Testing and Releasing DC
on:
  push:
    branches:
      - main

env:
  # the server the mocked services and app tests run against - bump deliberately
  SASJS_SERVER_VERSION: '3.12.2'
  NODE_VERSION: '24.5.0'

jobs:
  Build-production-and-ng-test:
    runs-on: ubuntu-latest
    env:
      CHROME_BIN: /usr/bin/google-chrome

    steps:
      - uses: actions/checkout@v4
      - name: Check runner size
        # A release job that OOMs costs the tag as well as the slot; fail in
        # seconds on an undersized runner instead of twenty minutes in - see
        # the script.
        run: bash .gitea/scripts/check-runner-memory.sh
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Write .npmrc file
        run: |
          # the secret arrives through the environment, not interpolated into
          # this script: a quote or $( in the value would otherwise break the
          # step or execute
          printf '%s\n' "$NPMRC" > "$RUNNER_TEMP/npmrc"
          echo "NPM_CONFIG_USERCONFIG=$RUNNER_TEMP/npmrc" >> "$GITHUB_ENV"
        env:
          NPMRC: ${{ secrets.NPMRC}}

      - name: Install Chrome for Angular tests
        run: |
          apt-get update
          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
          apt install -y ./google-chrome*.deb

      - name: Write cypress credentials
        run: echo "$CYPRESS_CREDS" > ./client/cypress.env.json
        shell: bash
        env:
          CYPRESS_CREDS: ${{ secrets.CYPRESS_CREDS }}

      - name: Install dependencies
        run: |
          cd client
          # Decrypt and Install sheet
          echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
          npm ci
          # client install scripts are off (client/.npmrc); the generator that
          # creates src/environments/version.ts runs explicitly
          npm run postinstall

      - name: Check audit
        # Audit should fail and stop the CI on any vulnerability in root and sas, and on low+ in client
        run: |
          npm audit --omit=dev
          cd ./sas
          npm audit --omit=dev
          cd ../client
          npm audit --omit=dev

      - name: Angular Tests
        run: |
          cd client
          npm run test:headless

      - name: Angular Production Build
        run: |
          cd client
          npm run postinstall
          npm run check:archiver
          npm run build

      - name: Remove registry config
        # written earlier in this job; nothing deletes it on the way out
        if: always()
        run: rm -f "$RUNNER_TEMP/npmrc"

  Build-and-test-development:
    runs-on: ubuntu-latest
    needs: Build-production-and-ng-test
    env:
      CHROME_BIN: /usr/bin/google-chrome
      # Pin OS locale
      LANG: en_GB.UTF-8
      LC_ALL: en_GB.UTF-8

    steps:
      - uses: actions/checkout@v4
      - name: Check runner size
        # The development build plus a browser: same undersized-runner failure
        # as the review jobs, on the path that makes the tag - see the script.
        run: bash .gitea/scripts/check-runner-memory.sh
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Write .npmrc file
        run: |
          # the secret arrives through the environment, not interpolated into
          # this script: a quote or $( in the value would otherwise break the
          # step or execute
          printf '%s\n' "$NPMRC" > "$RUNNER_TEMP/npmrc"
          echo "NPM_CONFIG_USERCONFIG=$RUNNER_TEMP/npmrc" >> "$GITHUB_ENV"
        env:
          NPMRC: ${{ secrets.NPMRC}}

      - name: Install system dependencies
        run: |
          apt-get update
          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
          apt install -y ./google-chrome*.deb
          apt-get -y install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libnss3 libxss1 libasound2t64 libxtst6 xauth xvfb jq zip locales
          # Generate the en_GB.UTF-8 locale referenced by LANG/LC_ALL
          locale-gen en_GB.UTF-8
          update-locale LANG=en_GB.UTF-8 LC_ALL=en_GB.UTF-8

      - name: Write cypress credentials
        run: echo "$CYPRESS_CREDS" > ./client/cypress.env.json
        shell: bash
        env:
          CYPRESS_CREDS: ${{ secrets.CYPRESS_CREDS }}

      - name: Install dependencies
        run: |
          cd client
          # Decrypt and Install sheet
          echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
          npm ci
          # client install scripts are off (client/.npmrc); the generator that
          # creates src/environments/version.ts runs explicitly
          npm run postinstall

      - name: Build frontend for web streaming
        # The server-ci target streams client/dist as the 'clickme' web app, so
        # the production build has to exist before the mocked services are
        # deployed. Without it `sasjs cbd` reports
        # "webSourcePath: .../client/dist ... doesn't exist", streams no
        # frontend and the deploy step fails.
        run: |
          cd client
          npm run build

      - name: Cache the npm downloads
        # The server is installed from npm at a pinned version, so the tarball is
        # immutable and the key never needs invalidating by hand: a hit skips
        # the 10.7MB download, a miss costs exactly what the zip cost before.
        # continue-on-error, because a runner without cache support should still
        # run the pipeline - this step only ever saves time.
        continue-on-error: true
        uses: actions/cache@v4
        with:
          path: ~/.npm
          key: npm-sasjs-server-${{ env.SASJS_SERVER_VERSION }}

      - name: Setup and start SASjs server
        run: |
          npm i -g pm2
          touch .env
          echo RUN_TIMES=js >> .env
          echo NODE_PATH=node >> .env
          echo CORS=enable >> .env
          echo WHITELIST=http://localhost:4200 >> .env
          echo HTTP_PORT=5000 >> .env
          # The server refuses to run as root unless RUN_AS names an account
          # (a security change in the recent releases). This job runs as root
          # in a throwaway container, so opt in explicitly.
          echo RUN_AS=root >> .env
          cat .env
          pm2 start npx --name sasjs-server -- --yes @sasjs/server@${{ env.SASJS_SERVER_VERSION }}
          # `pm2 start` returns before the app is listening, and the next step
          # hits :5000 immediately. Wait for the server to answer, and dump its
          # log if it never does.
          for i in $(seq 1 60); do
            if curl -sf -o /dev/null http://localhost:5000/; then
              echo "SASjs server answered on attempt ${i}"
              exit 0
            fi
            sleep 2
          done
          echo "SASjs server did not come up within 120s"
          pm2 list
          pm2 logs sasjs-server --lines 100 --nostream
          exit 1

      - name: Deploy mocked services
        run: |
          cd ./sas/mocks/sasjs
          npm install -g @sasjs/cli
          npm install -g replace-in-files-cli
          # Remove any previous deployment (drive folder delete API) so the
          # deploy and makedata start from a clean appLoc
          curl -sS -X DELETE "http://localhost:5000/SASjsApi/drive/folder/?_folderPath=/Public/app/dc"
          sasjs cbd -t server-ci
          # Seed the DC database (mock data files on the drive).
          # makedata replies with HTML (it is normally called as a URL redirect),
          # which the CLI reports as "invalid Json string" - ignore that.
          # NOTE: -d paths resolve from the sasjs project root (sas/mocks),
          # not the cwd (sas/mocks/sasjs).
          sasjs request services/admin/makedata -t server-ci -d deploy/makedata.json -o ./makedata_out.json || true

      - name: Prepare and run frontend and cypress
        run: |
          cd ./client
          mv ./cypress.env.example.json ./cypress.env.json
          replace-in-files --regex='"username".*' --replacement='"username":"'${{ secrets.CYPRESS_USERNAME_SASJS }}'",' ./cypress.env.json
          replace-in-files --regex='"password".*' --replacement='"password":"'${{ secrets.CYPRESS_PWD_SASJS }}'" ' ./cypress.env.json
          cat ./cypress.env.json
          npm run postinstall
          # cypress downloads its binary in a dependency install hook, which the
          # client policy skips
          npx cypress install
          # Prepare index.html to SASJS local
          replace-in-files --regex='serverUrl=".*?"' --replacement='serverUrl="http://localhost:5000"' ./src/index.html
          replace-in-files --regex='appLoc=".*?"' --replacement='appLoc="/Public/app/dc"' ./src/index.html
          replace-in-files --regex='serverType=".*?"' --replacement='serverType="SASJS"' ./src/index.html
          replace-in-files --regex='"hosturl".*' --replacement='hosturl:"http://localhost:4200",' ./cypress.config.ts
          cat ./cypress.config.ts
          # Start frontend and run cypress
          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts"

      - name: Zip Cypress videos
        if: always()
        run: |
          mkdir -p ./client/cypress/videos
          zip -r cypress-videos ./client/cypress/videos

      - name: Add cypress videos artifacts
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: cypress-videos.zip
          path: cypress-videos.zip

      - name: Remove registry config
        # written earlier in this job; nothing deletes it on the way out
        if: always()
        run: rm -f "$RUNNER_TEMP/npmrc"

  release:
    runs-on: ubuntu-latest
    needs: [Build-production-and-ng-test, Build-and-test-development]
    # The semantic-release step below uses the per-job token injected by Gitea
    # Actions (secrets.GITEA_TOKEN). That token is scoped to THIS repository
    # and its permissions are derived from this block.
    #   contents: write -> push CHANGELOG.md / package.json commit + tag, and
    #                      create the Release via the Gitea API.
    #   actions: write  -> dispatch the Sites workflow for the new tag (that
    #                      route is scoped to the actions unit, which contents
    #                      does not grant).
    # The TSDOC_TOKEN and CODE_DATACONTROLLER_IO secrets continue to be used
    # because they target *different* servers (webdoc.datacontroller.io and
    # code.datacontroller.io), not the Gitea API.
    permissions:
      contents: write
      actions: write

    steps:
      - uses: actions/checkout@v4
      - name: Check runner size
        # This job builds the frontend twice (webdoc and the release check);
        # both die the same way on an undersized runner - see the script.
        run: bash .gitea/scripts/check-runner-memory.sh
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Write .npmrc file
        run: |
          echo "$NPMRC" > "$RUNNER_TEMP/npmrc"
          echo "legacy-peer-deps=true" >> "$RUNNER_TEMP/npmrc"
          echo "NPM_CONFIG_USERCONFIG=$RUNNER_TEMP/npmrc" >> "$GITHUB_ENV"
        shell: bash
        env:
          NPMRC: ${{ secrets.NPMRC}}

      - name: Install system packages
        run: |
          apt-get update
          apt-get install -y zip jq doxygen
          npm i -g @sasjs/cli

      - name: Frontend Preliminary Build
        description: We want to prevent creating empty release if frontend fails
        run: |
          cd client
          # Decrypt and Install sheet
          echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
          npm ci
          # client install scripts are off (client/.npmrc); the generator that
          # creates src/environments/version.ts runs explicitly
          npm run postinstall
          npm i webpack
          npm run build

      - name: Create Empty Release (assets are posted later)
        env:
          # Use the ephemeral per-job token injected by Gitea Actions instead
          # of a manually-managed PAT. This token always reports push=true on
          # its own repo via the Gitea API, which is what the
          # @saithodev/semantic-release-gitea verifyConditions step checks.
          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
          GITEA_URL: ${{ gitea.server_url }}
        run: |
          set -euo pipefail
          npm i
          npm i -g semantic-release

          # We do a semantic-release DRY RUN to make the job fail if there are
          # no changes to release. The previous implementation piped --dry-run
          # into `grep`, which swallowed semantic-release's non-zero exit codes
          # (e.g. auth/verifyConditions failures) — the real error then only
          # surfaced from the second invocation. Capture output and inspect it
          # so authentication failures fail the step cleanly with context.
          echo "::group::semantic-release --dry-run"
          DRY_EXIT=0
          DRY_OUTPUT=$(semantic-release --dry-run 2>&1) || DRY_EXIT=$?
          echo "$DRY_OUTPUT"
          echo "::endgroup::"

          if [ "$DRY_EXIT" -ne 0 ]; then
            echo "::error::semantic-release dry-run failed (exit $DRY_EXIT). See log above; common cause is the Gitea token lacking push permission on this repo (verifyConditions of @saithodev/semantic-release-gitea)."
            exit "$DRY_EXIT"
          fi

          if echo "$DRY_OUTPUT" | grep -q "There are no relevant changes, so no new version is released."; then
            echo "::error::No releasable changes since last tag - aborting release job."
            exit 1
          fi

          semantic-release

          # The tag this job cuts, for the steps that publish against it: a
          # `releases/latest` lookup would pick whichever release landed last
          # if two overlap.
          echo "RELEASE_TAG=v$(node -p "require('./package.json').version")" >> "$GITHUB_ENV"

      - name: Frontend Build
        description: Must be created AFTER the release as the version (git tag) is used in the interface
        run: |
          cd client
          npm run build

      - name: Build SAS9 EBI Release
        description: Compile SAS 9 services, remove tests & create deployment program
        run: |
          cd sas
          npm i
          sasjs c -t sas9
          rm -rf sasjsbuild/tests
          sasjs b -t sas9
          cp sasjsbuild/mysas9deploy.sas ./demostream_sas9.sas
          #
          # remove streamed component and rebuild SAS 9 services
          #
          rm -rf sasjsbuild/services/web9
          rm sasjsbuild/services/clickme.sas
          sasjs b -t sas9
          cp sasjsbuild/mysas9deploy.sas ./sas9.sas

      - name: Build SASjs Server Release
        description: Compile Base (SASjs) services, remove tests & create deployment JSON
        run: |
          cd sas
          cp sasjs/utils/favicon.ico ../client/dist/favicon.ico
          sasjs c -t server
          rm -rf sasjsbuild/tests
          server_apploc="/Public/app/dc"
          sed -i "s|apploc=\"[^\"]*\"|apploc=\"${server_apploc}\"|g" sasjsbuild/services/web/index.html
          sasjs b -t server
          cp sasjsbuild/server.json.zip ./sasjs_server.json.zip

      - name: Build Viya Release
        description: compile Viya Streaming Deploy (without tests)
        run: |
          cd sas
          sasjs c -t viya
          rm -rf sasjsbuild/tests
          sed -i -e 's/servertype="SASJS"/servertype="SASVIYA"/g' sasjsbuild/services/DC.html
          sasjs b -t viya
          cp sasjsbuild/viya.sas ./viya.sas
          # compile Viya Full deploy (without web)
          rm -rf sasjsbuild/services/web
          rm sasjsbuild/services/DC.html
          sasjs b -t viya
          cp sasjsbuild/viya.sas ./viya_noweb.sas
          cp sasjsbuild/viya.json ./viya_noweb.json

      - name: Zip Frontend (including viya.json for full viya deploy)
        run: |
          cd sas
          cp sasjsbuild/viya.json ../client/dist/viya.json
          # Zip from *inside* dist so the archive holds the frontend files at its
          # root rather than under a client/dist/ prefix (see #147).
          cd ../client/dist
          zip -r ../../frontend.zip .

      - name: Upload assets to release
        env:
          # Use the same ephemeral per-job Gitea Actions token that the
          # "Create Empty Release" step uses for semantic-release. It is scoped
          # to this repo and is granted write access via the workflow's
          # `permissions:` block at the top of the file (see contents: write).
          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
        run: |
          set -euo pipefail
          # Send the token via Authorization header rather than ?access_token=
          # (the query-string form is deprecated and leaks into access logs).
          AUTH_HEADER="Authorization: token ${GITEA_TOKEN}"
          BASE="${{ gitea.server_url }}/api/v1/repos/${{ gitea.repository }}"

          RELEASE_JSON=$(curl --fail-with-body -sS -H "$AUTH_HEADER" "$BASE/releases/tags/${RELEASE_TAG}")
          RELEASE_ID=$(echo "$RELEASE_JSON" | jq -r '.id')
          RELEASE_BODY=$(echo "$RELEASE_JSON" | jq -r '.body')

          # Generate SHA-256 checksums over the release assets, so downloads can
          # be verified. The file uses basename-only paths so that a plain
          # "sha256sum -c SHA256SUMS" works in the folder the assets were
          # downloaded to (the upload below stores them under their basename).
          ASSETS=(frontend.zip
                  sas/demostream_sas9.sas
                  sas/viya.sas
                  sas/sasjs_server.json.zip
                  sas/sas9.sas
                  sas/viya_noweb.sas
                  sas/viya_noweb.json)

          : > SHA256SUMS
          for f in "${ASSETS[@]}"; do
            sum=$(sha256sum "$f" | cut -d' ' -f1)
            printf '%s  %s\n' "$sum" "${f##*/}" >> SHA256SUMS
          done
          cat SHA256SUMS

          # Update body: keep the existing notes, add a verification section
          # ahead of the installation footer. The section text lives in a
          # dedicated script below (.gitea/scripts/verify-section.sh) so this
          # YAML scalar stays clean.
          # `jq -r` is required: without it jq emits the result as a JSON string
          # literal (quoted, with \n escapes), and the PATCH below then encodes
          # that literal a second time, so the release body renders as one
          # escaped line instead of markdown.
          VERIFY_SECTION=$(./.gitea/scripts/verify-section.sh)
          NEW_BODY=$(jq -nr --arg body "$RELEASE_BODY" --arg verify "$VERIFY_SECTION" \
            '$body + "\n" + $verify + "\n\nFor installation instructions, please visit https://docs.datacontroller.io/"')
          curl --fail-with-body -sS -X PATCH \
            -H "$AUTH_HEADER" \
            -H 'Content-Type: application/json' \
            --data "$(jq -n --arg body "$NEW_BODY" '{draft:false, body:$body}')" \
            "$BASE/releases/$RELEASE_ID"

          # Upload assets
          URL="$BASE/releases/$RELEASE_ID/assets"
          for f in "${ASSETS[@]}"; do
            echo "Uploading $f ..."
            curl --fail-with-body -sS -H "$AUTH_HEADER" "$URL" -F "attachment=@$f"
          done
          echo "Uploading SHA256SUMS ..."
          curl --fail-with-body -sS -H "$AUTH_HEADER" "$URL" -F "attachment=@SHA256SUMS"

      - name: Publish the code documentation
        # Typedoc (webdoc) and the SAS docs (code) are published by docs.yaml,
        # not inline here. Two reasons: the v6 `surfer put --token` flow is gone
        # in Surfer 7, which takes a Cloudron username plus an app password; and
        # a dispatch means a stale or failed docs deploy can be republished
        # without re-running this job, which semantic-release refuses once the
        # tag exists.
        #
        # This runs AFTER the release assets are uploaded: the documentation is
        # a side channel, and a docs problem must not cost a cut release its
        # artefacts.
        env:
          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
        run: |
          set -euo pipefail
          AUTH_HEADER="Authorization: token $GITEA_TOKEN"
          BASE="${{ gitea.server_url }}/api/v1/repos/${{ gitea.repository }}"

          TAG="${RELEASE_TAG}"
          echo "Dispatching code documentation for ${TAG}"

          curl --fail-with-body -sS -X POST \
            -H "$AUTH_HEADER" \
            -H 'Content-Type: application/json' \
            --data "$(jq -n --arg ref "refs/tags/$TAG" '{ref: $ref}')" \
            "$BASE/actions/workflows/docs.yaml/dispatches"

      - name: Publish the demo site for this release
        # The tag is created by this job's semantic-release step, so it carries
        # the Actions user as its author - and the forge suppresses ref events
        # from that user, which means `on: push: tags` in sites.yaml never fires.
        # Dispatch it instead: the API call is authenticated, and the run it
        # starts checks out the tag the same way.
        env:
          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
        run: |
          set -euo pipefail
          AUTH_HEADER="Authorization: token ${GITEA_TOKEN}"
          BASE="${{ gitea.server_url }}/api/v1/repos/${{ gitea.repository }}"

          TAG="${RELEASE_TAG}"
          echo "Dispatching Sites for ${TAG}"

          # the dispatch endpoint resolves a bare name as a branch; a tag needs
          # the refs/tags/ prefix
          curl --fail-with-body -sS -X POST \
            -H "$AUTH_HEADER" \
            -H 'Content-Type: application/json' \
            --data "$(jq -n --arg ref "refs/tags/$TAG" '{ref: $ref}')" \
            "$BASE/actions/workflows/sites.yaml/dispatches"

      - name: Remove registry config
        # written earlier in this job; nothing deletes it on the way out
        if: always()
        run: rm -f "$RUNNER_TEMP/npmrc"