Merge pull request 'Create Release' (#22) from development into main

Reviewed-on: #22

.gitea/workflows/build.yaml

@@ -1,5 +1,5 @@
 name: Build
-run-name: Building and testing DC
+run-name: Running Lint Check
 on: [pull_request]
 
 jobs:
@@ -18,21 +18,4 @@ jobs:
         env:
           NPMRC: ${{ secrets.NPMRC}}
 
-      - run: apt-get update
-      - run: wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-      - run: apt install -y ./google-chrome*.deb;
-      - run: export CHROME_BIN=/usr/bin/google-chrome
-      - run: npm run lint:check
-      # Install dependencies~
-      - run: npm ci
-      # Audit should fail and stop the CI if critical vulnerability found
-      - run: npm audit --audit-level=critical
-      - run: |
-          cd ./sas
-          npm audit --audit-level=critical
-      - run: |
-          cd ./client
-          npm audit --audit-level=critical
-          npm test -- --no-watch --no-progress --browsers=ChromeHeadlessCI
-          npm run postinstall
-          npm run build
+      - run: npm run lint:check

.gitea/workflows/development-test.yaml

@@ -6,6 +6,53 @@ on:
       - development
 
 jobs:
+  Build-production-and-ng-test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Write .npmrc file
+        run: |
+         touch client/.npmrc
+         echo '${{ secrets.NPMRC}}' > client/.npmrc
+
+      - name: Install Chrome for Angular tests
+        run: apt-get update
+        run: wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+        run: apt install -y ./google-chrome*.deb;
+        run: export CHROME_BIN=/usr/bin/google-chrome
+
+      - name: Write cypress credentials
+        run: echo "$CYPRESS_CREDS" > ./client/cypress.env.json
+        shell: bash
+        env:
+          CYPRESS_CREDS: ${{ secrets.CYPRESS_CREDS }}
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Check audit
+      # Audit should fail and stop the CI if critical vulnerability found
+        run: |
+          npm audit --audit-level=critical
+          cd ./sas
+          npm audit --audit-level=critical
+          cd ./client
+          npm audit --audit-level=critical
+
+      - name: Angular Tests
+        run: |
+          npm test -- --no-watch --no-progress --browsers=ChromeHeadlessCI
+
+      - name: Angular Production Build
+        run: |
+          npm run postinstall
+          npm run build
+
   Build-and-test-development:
     runs-on: ubuntu-latest
 
@@ -34,7 +81,9 @@ jobs:
         env:
           CYPRESS_CREDS: ${{ secrets.CYPRESS_CREDS }}
 
-      - run: npm ci
+      - name: Install dependencies
+        run: npm ci
+
       # Install pm2 and prepare SASJS server
       - run: npm i -g pm2
       - run: curl -L https://github.com/sasjs/server/releases/latest/download/linux.zip > linux.zip
@@ -77,6 +126,17 @@ jobs:
           # Start frontend and run cypress
           npm start & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
 
+      - name: Zip Cypress videos
+        if: always()
+        run: |
+          zip -r cypress-videos ./client/cypress/videos
+
+      - name: Cypress videos artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: cypress-videos.zip
+          path: cypress-videos.zip
+
   Build-and-test-development-latest-adapter:
     runs-on: ubuntu-latest
 
@@ -106,7 +166,9 @@ jobs:
         env:
           CYPRESS_CREDS: ${{ secrets.CYPRESS_CREDS }}
 
-      - run: npm ci
+      - name: Install dependencies
+        run: npm ci
+
       # Install pm2 and prepare SASJS server
       - run: npm i -g pm2
       - run: curl -L https://github.com/sasjs/server/releases/latest/download/linux.zip > linux.zip
@@ -125,7 +187,6 @@ jobs:
           npm install -g @sasjs/cli
           npm install -g replace-in-files-cli
           sasjs cbd -t server-ci
-          # sasjs request services/admin/makedata -t server-ci -d ./deploy/makeData4GL.json -c ./deploy/requestConfig.json -o ./output.json
 
       - name: Install ZIP
         run: |
@@ -148,4 +209,15 @@ jobs:
           replace-in-files --regex='"hosturl".*' --replacement='hosturl:"http://localhost:4200",' ./cypress.config.ts
           cat ./cypress.config.ts
           # Start frontend and run cypress
-          npm start & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
+          npm start & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
+
+      - name: Zip Cypress videos
+        if: always()
+        run: |
+          zip -r cypress-videos ./client/cypress/videos
+
+      - name: Cypress videos artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: cypress-videos-latest-adapter.zip
+          path: cypress-videos.zip

.gitea/workflows/release.yaml

@@ -22,32 +22,30 @@ jobs:
         env:
           NPMRC: ${{ secrets.NPMRC}}
 
-      - name: Install ZIP and SASjs CLI
+      - name: Install packages
         run: |
           apt-get update
           apt-get install zip -y
+          # sasjs cli is used to compile & build the SAS services
           npm i -g @sasjs/cli
-          # test
-
-      - name: Install JQ for parsing JSON
-        run: |
-          apt-get update
+          # jq is used to parse the release JSON
           apt-get install jq -y
 
-      - name: Install Semantic Release and plugins and create Release
+      - name: Create Empty Release (assets are posted later)
         run: |
           npm i
           npm i -g semantic-release
           GITEA_TOKEN=${{ secrets.RELEASE_TOKEN }} GITEA_URL=https://git.datacontroller.io semantic-release
 
       - name: Frontend Build
+        description: Must be created AFTER the release as the version (git tag) is used in the interface
         run: |
           cd client
           npm ci
           npm run build
 
       - name: Build SAS9 EBI Release
-        description: compile SAS 9 services, remove tests & create deployment program
+        description: Compile SAS 9 services, remove tests & create deployment program
         run: |
           cd sas
           npm ci
@@ -64,7 +62,7 @@ jobs:
           cp sasjsbuild/mysas9deploy.sas ./sas9.sas
 
       - name: Build SASjs Server Release
-        description: compile Base (SASjs) services, remove tests & create deployment JSON
+        description: Compile Base (SASjs) services, remove tests & create deployment JSON
         run: |
           cd sas
           cp sasjs/utils/favicon.ico ../client/dist/favicon.ico
@@ -99,11 +97,15 @@ jobs:
         run: |
           cd client
           npm run typedoc
-          # deploy docs
+          surfer put --token ${{ secrets.TSDOC_TOKEN }} --server tsdoc.datacontroller.io ../tsdoc /
 
       - name: Upload assets to release
         run: |
           RELEASE_ID=`curl -k 'https://git.datacontroller.io/api/v1/repos/dc/dc/releases/latest?access_token=${{ secrets.RELEASE_TOKEN }}' | jq -r '.id'`
+          RELEASE_BODY=`curl -k 'https://git.datacontroller.io/api/v1/repos/dc/dc/releases/latest?access_token=${{ secrets.RELEASE_TOKEN }}' | jq -r '.body'`
+          # Update body
+          curl --data '{"draft": true,"body":"'"$RELEASE_BODY\n\nFor installation instructions, please visit https://docs.datacontroller.io/"'"}' -X PATCH --header 'Content-Type: application/json' -k https://git.datacontroller.io/api/v1/repos/dc/dc/releases/$RELEASE_ID?access_token=${{ secrets.RELEASE_TOKEN }}
+          # Upload assets
           URL="https://git.datacontroller.io/api/v1/repos/dc/dc/releases/$RELEASE_ID/assets?access_token=${{ secrets.RELEASE_TOKEN }}"
           curl -k $URL -F attachment=@frontend.zip
           curl -k $URL -F attachment=@sas/demostream_sas9.sas

client/package.json

@@ -29,7 +29,7 @@
     "cy:run": "cypress run",
     "audit:prod": "npm audit --omit=dev",
     "sasdocs": "sasjs doc && ./sasjs/utils/deploydocs.sh",
-    "typedoc": "typedoc --options typedoc.json && cd ../dc-devdocs"
+    "typedoc": "typedoc --options typedoc.json && cd ../tsdoc"
   },
   "private": true,
   "dependencies": {
@@ -69,7 +69,6 @@
     "ngx-clipboard": "^16.0.0",
     "ngx-json-viewer": "file:libraries/ngx-json-viewer-3.2.1.tgz",
     "nodejs": "0.0.0",
-    "numbro": "^2.1.1",
     "os-browserify": "0.3.0",
     "rxjs": "^7.8.0",
     "save-svg-as-png": "^1.4.17",

client/src/app/app.d.ts

@@ -1,5 +1,5 @@
 declare module 'save-svg-as-png'
-
+declare module 'numbro/dist/languages.min'
 declare interface Navigator {
   msSaveBlob: (blob: any, defaultName?: string) => boolean
 }

client/src/app/editor/editor.component.html

@@ -385,7 +385,6 @@
               [class.hidden]="hotTable.hidden"
               [licenseKey]="hotTable.licenseKey"
             >
-              <!--[licenseKey]=""-->
             </hot-table>
           </div>
 

client/src/app/editor/editor.component.ts

@@ -63,6 +63,8 @@ import {
 } from './utils/renderers.utils'
 import { isStringDecimal, isStringNumber } from './utils/types.utils'
 import { LicenceService } from '../services/licence.service'
+import * as numbro from 'numbro'
+import * as languages from 'numbro/dist/languages.min'
 
 @Component({
   selector: 'app-editor',
@@ -362,6 +364,10 @@ export class EditorComponent implements OnInit, AfterViewInit {
     private cdf: ChangeDetectorRef,
     private hotRegisterer: HotTableRegisterer
   ) {
+    const lang = languages[window.navigator.language]
+    if (lang)
+      numbro.default.registerLanguage(languages[window.navigator.language])
+
     this.hotRegisterer = new HotTableRegisterer()
 
     this.parseRestrictions()

client/src/app/shared/dc-validator/dc-validator.ts

@@ -19,6 +19,7 @@ import { mergeColsRules } from './utils/mergeColsRules'
 import { parseColType } from './utils/parseColType'
 import { dqValidate } from './validations/dq-validation'
 import { specialMissingNumericValidator } from './validations/hot-custom-validators'
+import { applyNumericFormats } from './utils/applyNumericFormats'
 
 export class DcValidator {
   private rules: DcValidation[] = []
@@ -41,6 +42,7 @@ export class DcValidator {
     this.hotInstance = hotInstance
     this.rules = parseColType(sasparams.COLTYPE)
     this.rules = mergeColsRules(cols, this.rules, $dataFormats)
+    this.rules = applyNumericFormats(this.rules)
     this.dqrules = dqRules
     this.dqdata = dqData
     this.primaryKeys = sasparams.PK.split(' ')

client/src/app/shared/dc-validator/utils/applyNumericFormats.ts

@@ -0,0 +1,26 @@
+import { DcValidation } from '../models/dc-validation.model'
+import * as languages from 'numbro/dist/languages.min'
+/**
+ * Applying the numeric formats based on the browser locale/language
+ * So that correct decimal separators are applied.
+ * For example european format (thousand dot, decimal comma): 1.000,00
+ *
+ * @param rules Cell Validation rules to be updated
+ * Those rules are passed in the `columns` property Of handsontable settings.
+ */
+export const applyNumericFormats = (rules: DcValidation[]): DcValidation[] => {
+  const lang = languages[window.navigator.language]
+
+  if (!lang) return rules
+
+  for (let rule of rules) {
+    if (rule.type === 'numeric')
+      rule.numericFormat = {
+        pattern: '0,0',
+        culture: window.navigator.language // use this for EUR (German),
+        // more cultures available on http://numbrojs.com/languages.html
+      }
+  }
+
+  return rules
+}

client/src/app/shared/dc-validator/utils/mergeColsRules.ts

@@ -6,7 +6,8 @@ import { DcValidation } from '../models/dc-validation.model'
  *  Merging old validation params from sasparams with cols params
  * @param sasparams sasparams coming from SAS
  * @param cols cols coming from SAS
- * @param rules rules to be updated
+ * @param rules Cell Validation rules to be updated
+ * Those rules are passed in the `columns` property Of handsontable settings.
  * @returns
  */
 export const mergeColsRules = (

client/src/environments/_eula.ts

@@ -5,27 +5,17 @@ Licence Agreement for Data Controller for SAS®
 
 Copyright (c) Bowe IO Ltd
 
-Data Controller is a software distributed by 4GL Apps, a brand owned by Bowe IO Ltd, a UK Limited Company headquarted in 29 Oldfield Rd, Cumbria, registered by companies house under number 08777171, VAT number: 203914240
+Data Controller software is distributed by 4GL Apps, a brand owned by Bowe IO Ltd, a UK Limited Company headquarted in 29 Oldfield Rd, Cumbria, registered at Companies House with company number 08777171, VAT number: 203914240
 
-This software is protected by applicable copyright laws, including international treaties, and dual-
-licensed – depending on whether your use for commercial purposes, meaning intended for or
-resulting in commercial advantage or monetary compensation, or not.
+This software is protected by applicable copyright laws, including international treaties, and dual-licensed – depending on whether your use for commercial purposes, meaning intended for or resulting in commercial advantage or monetary compensation, or not.
 
-If your use is strictly personal or solely for evaluation purposes, meaning for the purposes of testing
-the suitability, performance, and usefulness of this software outside the production environment,
-you agree to be bound by the terms included in the "licence-non-commercial-datacontroller.md" file.
+If your use is strictly personal or solely for evaluation purposes, meaning for the purposes of testing the suitability, performance, and usefulness of this software outside the production environment, you agree to be bound by the terms included in the "licence-non-commercial-datacontroller.md" file available here:  https://git.datacontroller.io/dc/dc/src/branch/main/licence-non-commercial-datacontroller.md
 
-Your use of this software for commercial purposes is subject to the terms included in an applicable
-license agreement.
+Your use of this software for commercial purposes is subject to the terms included in an applicable license agreement.
 
-In any case, you must not make any such use of this software as to develop software which may be
-considered competitive with this software.
+In any case, you must not make any such use of this software as to develop software which may be considered competitive with this software.
 
-UNLESS EXPRESSLY AGREED OTHERWISE, 4GL APPS PROVIDES THIS SOFTWARE ON AN "AS IS"
-BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, AND IN NO EVENT AND UNDER NO
-LEGAL THEORY, SHALL 4GL APPS BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT,
-INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER ARISING FROM
-USE OR INABILITY TO USE THIS SOFTWARE.
+UNLESS EXPRESSLY AGREED OTHERWISE, 4GL APPS PROVIDES THIS SOFTWARE ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, AND IN NO EVENT AND UNDER NO LEGAL THEORY, SHALL 4GL APPS BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER ARISING FROM USE OR INABILITY TO USE THIS SOFTWARE.
 
 
 `

client/typedoc.json

@@ -1,12 +1,12 @@
 {
-    "out": "../dc-devdocs",
-    "tsconfig": "./tsconfig.app.json",
-    "entryPointStrategy": "expand",
-    "entryPoints": [
-      "./src"
-    ],
-    "exclude": "**/*+(index|.spec|.e2e).ts",
-    "externalPattern": "**/node_modules/**",
-    "excludeExternals": true,
-    "excludePrivate": true
-  }
+  "out": "../tsdoc",
+  "tsconfig": "./tsconfig.app.json",
+  "entryPointStrategy": "expand",
+  "entryPoints": [
+    "./src"
+  ],
+  "exclude": "**/*+(index|.spec|.e2e).ts",
+  "externalPattern": "**/node_modules/**",
+  "excludeExternals": true,
+  "excludePrivate": true
+}

sas/sasjs/macros/mpe_targetloader.sas

@@ -141,6 +141,13 @@ run;
 
     %mp_lockanytable(UNLOCK,lib=&lib,ds=&ds,ctl_ds=&dclib..mpe_lockanytable)
   %end;
+  %else %do;
+    /* is full replace so treat everything as a mod in diff screen */
+    data work.outds_mod work.outds_add work.outds_del;
+      set work.&staging_ds;
+      output work.outds_mod; /* _add and _del will be empty */
+    run;
+  %end;
 %end;
 %else %if &loadtype=UPDATE %then %do;
   %bitemporal_dataloader(bus_from=,bus_to=

sas/sasjs/services/auditors/postdata.sas

@@ -597,6 +597,7 @@ run;
   %if &LOADTYPE=REPLACE %then %do;
     data work.outds_add; run;
     data work.outds_mod; run;
+    data work.outds_del; run;
   %end;
   libname approve "&mpelocapprovals/&TABLE";
   data; set &libds;stop;run;