Merge pull request 'Restore Previous State / Data Rollback' (#94) from restore into main

Reviewed-on: #94

.gitea/workflows/build.yaml

@@ -10,7 +10,14 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
+
+      - name: Install Google Chrome
+        run: |
+          wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
+          echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list
+          apt-get update
+          apt-get install -y google-chrome-stable xvfb
 
       - name: Write .npmrc file
         run: echo "$NPMRC" > client/.npmrc
@@ -25,13 +32,17 @@ jobs:
         run: |
           cd client
           npm ci
-          # Install sheet
-          wget ${{ secrets.SHEETLINK }}
-          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
-          npm i ${{ secrets.SHEETNAME }}.tgz
+          # Decrypt and Install sheet
+          echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
+          npm i ./libraries/sheet-crypto.tgz
           # End
 
       - name: Licence checker
         run: |
           cd client
-          npm run license-checker
+          npm run license-checker
+
+      - name: Angular Tests
+        run: |
+          cd client
+          npm run test:headless

.gitea/workflows/release.yaml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
 
       - name: Write .npmrc file
         run: |
@@ -35,12 +35,11 @@ jobs:
 
       - name: Install dependencies
         run: |
-          npm ci
           cd client
-          # Install sheet
-          wget ${{ secrets.SHEETLINK }}
-          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
-          npm i ${{ secrets.SHEETNAME }}.tgz
+          npm ci
+          # Decrypt and Install sheet
+          echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
+          npm i ./libraries/sheet-crypto.tgz
           # End
 
       - name: Check audit
@@ -55,7 +54,7 @@ jobs:
       - name: Angular Tests
         run: |
           cd client
-          npm test -- --no-watch --no-progress --browsers=ChromeHeadlessCI
+          npm run test:headless
 
       - name: Angular Production Build
         run: |
@@ -71,7 +70,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
 
       - name: Write .npmrc file
         run: |
@@ -94,12 +93,11 @@ jobs:
 
       - name: Install dependencies
         run: |
-          npm ci
           cd client
-          # Install sheet
-          wget ${{ secrets.SHEETLINK }}
-          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
-          npm i ${{ secrets.SHEETNAME }}.tgz
+          npm ci
+          # Decrypt and Install sheet
+          echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
+          npm i ./libraries/sheet-crypto.tgz
           # End
 
       # Install pm2 and prepare SASJS server
@@ -199,10 +197,9 @@ jobs:
         run: |
           cd client
           npm ci
-          # Install sheet
-          wget ${{ secrets.SHEETLINK }}
-          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
-          npm i ${{ secrets.SHEETNAME }}.tgz
+          # Decrypt and Install sheet
+          echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
+          npm i ./libraries/sheet-crypto.tgz
           # End
           npm run build
 

client/angular.json

@@ -133,24 +133,21 @@
         "test": {
           "builder": "@angular-devkit/build-angular:karma",
           "options": {
-            "tsConfig": "tsconfig.spec.json",
-            "inlineStyleLanguage": "scss",
-            "codeCoverage": true,
             "polyfills": [
               "src/polyfills.ts",
               "zone.js",
               "zone.js/testing"
             ],
-            "styles": [
-              "src/styles.scss"
-            ],
-            "scripts": [
-
-            ],
+            "tsConfig": "tsconfig.spec.json",
+            "inlineStyleLanguage": "scss",
             "assets": [
               "src/favicon.ico",
               "src/assets"
             ],
+            "styles": [
+              "src/styles.scss"
+            ],
+            "scripts": [],
             "karmaConfig": "karma.conf.js"
           }
         },

client/cypress/e2e/editor.cy.ts

@@ -221,13 +221,13 @@ const submitExcel = (callback?: any) => {
 
 const rejectExcel = (callback?: any) => {
   cy.get('button', { timeout: longerCommandTimeout })
-    .should('contain', 'Go to approvals screen')
+    .should('contain', 'Approve')
     .then((allButtons: any) => {
       for (let approvalButton of allButtons) {
         if (
           approvalButton.innerText
             .toLowerCase()
-            .includes('go to approvals screen')
+            .includes('approve')
         ) {
           approvalButton.click()
           break

client/cypress/e2e/excel.cy.ts

@@ -405,13 +405,13 @@ const submitExcel = (callback?: any) => {
 
 const rejectExcel = (callback?: any) => {
   cy.get('button', { timeout: longerCommandTimeout })
-    .should('contain', 'Go to approvals screen')
+    .should('contain', 'Approve')
     .then((allButtons: any) => {
       for (let approvalButton of allButtons) {
         if (
           approvalButton.innerText
             .toLowerCase()
-            .includes('go to approvals screen')
+            .includes('approve')
         ) {
           approvalButton.click()
           break
@@ -438,13 +438,13 @@ const rejectExcel = (callback?: any) => {
 
 const acceptExcel = (callback?: any) => {
   cy.get('button', { timeout: longerCommandTimeout })
-    .should('contain', 'Go to approvals screen')
+    .should('contain', 'Approve')
     .then((allButtons: any) => {
       for (let approvalButton of allButtons) {
         if (
           approvalButton.innerText
             .toLowerCase()
-            .includes('go to approvals screen')
+            .includes('approve')
         ) {
           approvalButton.click()
           break

client/cypress/e2e/licensing.cy.ts

@@ -699,13 +699,13 @@ const submitTable = (callback?: any) => {
 
 const approveTable = (callback?: any) => {
   cy.get('button', { timeout: longerCommandTimeout })
-    .should('contain', 'Go to approvals screen')
+    .should('contain', 'Approve')
     .then((allButtons: any) => {
       for (let approvalButton of allButtons) {
         if (
           approvalButton.innerText
             .toLowerCase()
-            .includes('go to approvals screen')
+            .includes('approve')
         ) {
           approvalButton.click()
           break

client/cypress/e2e/liveness.cy.ts

@@ -125,13 +125,13 @@ const submitExcel = (callback?: any) => {
 
 const rejectExcel = (callback?: any) => {
   cy.get('button', { timeout: longerCommandTimeout })
-    .should('contain', 'Go to approvals screen')
+    .should('contain', 'Approve')
     .then((allButtons: any) => {
       for (let approvalButton of allButtons) {
         if (
           approvalButton.innerText
             .toLowerCase()
-            .includes('go to approvals screen')
+            .includes('approve')
         ) {
           approvalButton.click()
           break

client/cypress/integration/editor.tests.ts

@@ -221,14 +221,10 @@ const submitExcel = (callback?: any) => {
 
 const rejectExcel = (callback?: any) => {
   cy.get('button', { timeout: longerCommandTimeout })
-    .should('contain', 'Go to approvals screen')
+    .should('contain', 'Approve')
     .then((allButtons: any) => {
       for (let approvalButton of allButtons) {
-        if (
-          approvalButton.innerText
-            .toLowerCase()
-            .includes('go to approvals screen')
-        ) {
+        if (approvalButton.innerText.toLowerCase().includes('approve')) {
           approvalButton.click()
           break
         }

client/cypress/integration/excel.tests.ts

@@ -407,14 +407,10 @@ const submitExcel = (callback?: any) => {
 
 const rejectExcel = (callback?: any) => {
   cy.get('button', { timeout: longerCommandTimeout })
-    .should('contain', 'Go to approvals screen')
+    .should('contain', 'Approve')
     .then((allButtons: any) => {
       for (let approvalButton of allButtons) {
-        if (
-          approvalButton.innerText
-            .toLowerCase()
-            .includes('go to approvals screen')
-        ) {
+        if (approvalButton.innerText.toLowerCase().includes('approve')) {
           approvalButton.click()
           break
         }
@@ -440,14 +436,10 @@ const rejectExcel = (callback?: any) => {
 
 const acceptExcel = (callback?: any) => {
   cy.get('button', { timeout: longerCommandTimeout })
-    .should('contain', 'Go to approvals screen')
+    .should('contain', 'Approve')
     .then((allButtons: any) => {
       for (let approvalButton of allButtons) {
-        if (
-          approvalButton.innerText
-            .toLowerCase()
-            .includes('go to approvals screen')
-        ) {
+        if (approvalButton.innerText.toLowerCase().includes('approve')) {
           approvalButton.click()
           break
         }

client/cypress/integration/licensing.tests.ts

@@ -699,14 +699,10 @@ const submitTable = (callback?: any) => {
 
 const approveTable = (callback?: any) => {
   cy.get('button', { timeout: longerCommandTimeout })
-    .should('contain', 'Go to approvals screen')
+    .should('contain', 'Approve')
     .then((allButtons: any) => {
       for (let approvalButton of allButtons) {
-        if (
-          approvalButton.innerText
-            .toLowerCase()
-            .includes('go to approvals screen')
-        ) {
+        if (approvalButton.innerText.toLowerCase().includes('approve')) {
           approvalButton.click()
           break
         }

client/cypress/integration/liveness.tests.ts

@@ -125,14 +125,10 @@ const submitExcel = (callback?: any) => {
 
 const rejectExcel = (callback?: any) => {
   cy.get('button', { timeout: longerCommandTimeout })
-    .should('contain', 'Go to approvals screen')
+    .should('contain', 'Approve')
     .then((allButtons: any) => {
       for (let approvalButton of allButtons) {
-        if (
-          approvalButton.innerText
-            .toLowerCase()
-            .includes('go to approvals screen')
-        ) {
+        if (approvalButton.innerText.toLowerCase().includes('approve')) {
           approvalButton.click()
           break
         }

client/karma.conf.js

@@ -42,4 +42,4 @@ module.exports = function (config) {
       }
     },
   });
-};
+};

client/licenseChecker.js

@@ -10,7 +10,7 @@ const check = (cwd) => {
         onlyAllow:
           'AFLv2.1;Apache 2.0;Apache-2.0;Apache*;Artistic-2.0;0BSD;BSD*;BSD-2-Clause;BSD-3-Clause;CC0-1.0;CC-BY-3.0;CC-BY-4.0;ISC;MIT;MPL-2.0;ODC-By-1.0;Python-2.0;Unlicense;',
         excludePackages:
-          '@cds/city@1.1.0;@handsontable/angular@13.1.0;handsontable@13.1.0;hyperformula@2.6.2;jackspeak@2.2.0;path-scurry@1.7.0'
+          '@cds/city@1.1.0;@handsontable/angular@13.1.0;handsontable@13.1.0;hyperformula@2.7.0;jackspeak@2.2.0;path-scurry@1.7.0'
       },
       (error, json) => {
         if (error) {

client/package.json

@@ -18,8 +18,8 @@
     "deploy_sasjs": "rsync -avhe ssh ./dist/* --delete root@${npm_config_account}.4gl.io:/var/www/html/dc/dev",
     "viyabuild": "cd build; ./viyabuild.sh",
     "lint": "cd .. && npm run lint",
-    "test": "ng test",
-    "test:headless": "ng test --browsers ChromeHeadless",
+    "test": "npx ng test",
+    "test:headless": "npx ng test --no-watch --no-progress --browsers ChromeHeadlessCI",
     "watch": "ng test watch=true",
     "pree2e": "webdriver-manager update",
     "e2e": "protractor protractor.config.js",

client/src/app/editor/editor.component.ts

@@ -2263,8 +2263,8 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
       setTimeout(() => {
         let txt: any = document.getElementById('formFields_8')
-        txt.focus()
-      })
+        if (txt) txt.focus()
+      }, 200)
     })
 
     // let cnt = 0;

client/src/app/models/sas/editors-restore.model.ts

@@ -0,0 +1,9 @@
+import { BaseSASResponse } from './common/BaseSASResponse'
+
+export interface EditorsRestoreServiceResponse extends BaseSASResponse {
+  restore_out: RestoreOut[]
+}
+
+export interface RestoreOut {
+  LOADREF: string
+}

client/src/app/stage/stage.component.html

@@ -60,6 +60,7 @@
               <button
                 class="btn btn-sm btn-outline text-center mr-5i"
                 (click)="viewerTableScreen()"
+                [disabled]="revertingChanges"
               >
                 View base table
               </button>
@@ -71,12 +72,14 @@
                   tableDetails?.REVIEW_STATUS_ID === 'REJECTED'
                 "
                 (click)="approveTableScreen()"
+                [disabled]="revertingChanges"
               >
                 Approve
               </button>
               <button
                 class="btn btn-sm btn-info-outline text-center mr-5i"
                 (click)="goBack()"
+                [disabled]="revertingChanges"
               >
                 Edit base table
               </button>
@@ -90,13 +93,15 @@
               <clr-tooltip>
                 <button
                   *ngIf="tableDetails?.['ALLOW_RESTORE'] === 'YES'"
+                  (click)="revertChanges()"
                   clrTooltipTrigger
+                  [clrLoading]="revertingChanges"
                   class="btn btn-sm btn-danger text-center mt-20"
                 >
-                  REVERT (Coming Soon)
+                  REVERT
 
                   <clr-tooltip-content
-                    clrPosition="bottom-right"
+                    clrPosition="bottom-left"
                     clrSize="lg"
                     *clrIfOpen
                   >

client/src/app/stage/stage.component.ts

@@ -4,10 +4,10 @@ import { Router } from '@angular/router'
 import { ActivatedRoute } from '@angular/router'
 import { SasService } from '../services/sas.service'
 import { EventService } from '../services/event.service'
-import { AppService } from '../services/app.service'
 import { HotTableInterface } from '../models/HotTable.interface'
 import { LicenceService } from '../services/licence.service'
 import { globals } from '../_globals'
+import { EditorsRestoreServiceResponse } from '../models/sas/editors-restore.model'
 
 @Component({
   selector: 'app-stage',
@@ -23,6 +23,7 @@ export class StageComponent implements OnInit {
   public keysArray: any
   public tableDetails: any
   public loaded: boolean = false
+  public revertingChanges: boolean = false
   public licenceState = this.licenceService.licenceState
   public hotTable: HotTableInterface = {
     data: [],
@@ -162,6 +163,31 @@ export class StageComponent implements OnInit {
     }
   }
 
+  revertChanges() {
+    this.revertingChanges = true
+
+    const data = {
+      restore_in: [
+        {
+          load_ref: this.table_id
+        }
+      ]
+    }
+
+    this.sasService
+      .request('editors/restore', data)
+      .then((res: EditorsRestoreServiceResponse) => {
+        if (res.restore_out) {
+          this.route.navigate([`/stage`]).then(() => {
+            this.route.navigate([`/stage/${res.restore_out[0].LOADREF}`])
+          })
+        }
+      })
+      .finally(() => {
+        this.revertingChanges = false
+      })
+  }
+
   private setFocus() {
     setTimeout(() => {
       let approvalBtn: any = window.document.getElementById('approval-btn')

sas/mocks/sasjs/services/approvers/gethistory.js

@@ -14,6 +14,10 @@ _webout = `{"SYSDATE" : "26SEP22"
 "APPROVER": "sasdemo"
 }
 ]
+, "histparams":
+[
+{"HIST":100 ,"STARTROW":1 ,"NOBS":3 }
+]
 ,"_DEBUG" : ""
 ,"_METAUSER": "sasdemo@SAS"
 ,"_METAPERSON": "sasdemo"

sas/package-lock.json

@@ -7,7 +7,7 @@
       "name": "dc-sas",
       "dependencies": {
         "@sasjs/cli": "^4.11.1",
-        "@sasjs/core": "^4.49.0"
+        "@sasjs/core": "^4.52.0"
       }
     },
     "node_modules/@coolaj86/urequest": {
@@ -116,9 +116,9 @@
       "integrity": "sha512-Grwydm5GxBsYk238PZw41XPjXVVQ9vWcvfZ06L2P0bQbvK0sGn7l69JA7H5MGr3QcaLpiD4Kg70cAh7PgE+JOw=="
     },
     "node_modules/@sasjs/core": {
-      "version": "4.49.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.49.0.tgz",
-      "integrity": "sha512-hp3Hb4DkT6FmowyNHTOvSlgmSObW9WeuTJj+TQlwPgnBo59mAB4XFUnUaYSA+7ghvsHqUZf1OP2eSYqmnN5swQ=="
+      "version": "4.52.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.52.0.tgz",
+      "integrity": "sha512-r+yQYSTYdvNPJ82n2xW/rbJd/cJpk3HuFhCa49RTDlDWjXMXJlymvGB0ltv3/L8rNk4+TwoyMzb7cJYnILi6ag=="
     },
     "node_modules/@sasjs/lint": {
       "version": "2.3.1",
@@ -1828,9 +1828,9 @@
       }
     },
     "@sasjs/core": {
-      "version": "4.49.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.49.0.tgz",
-      "integrity": "sha512-hp3Hb4DkT6FmowyNHTOvSlgmSObW9WeuTJj+TQlwPgnBo59mAB4XFUnUaYSA+7ghvsHqUZf1OP2eSYqmnN5swQ=="
+      "version": "4.52.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.52.0.tgz",
+      "integrity": "sha512-r+yQYSTYdvNPJ82n2xW/rbJd/cJpk3HuFhCa49RTDlDWjXMXJlymvGB0ltv3/L8rNk4+TwoyMzb7cJYnILi6ag=="
     },
     "@sasjs/lint": {
       "version": "2.3.1",

sas/package.json

@@ -29,6 +29,6 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "^4.11.1",
-    "@sasjs/core": "^4.49.0"
+    "@sasjs/core": "^4.52.0"
   }
 }

sas/sasjs/macros/mpe_checkrestore.sas

@@ -0,0 +1,135 @@
+/**
+  @file
+  @brief Checks if a user is able to restore a LOAD_REF
+  @details Not all LOAD_REFs can be restored - maybe the user does not have
+    permission, maybe the load was never loaded, or maybe the load was not
+    tracked.
+
+    The macro creates two output (global) macro variables.
+
+  @param [in] LOAD_REF The Load Reference to check
+  @param [out] outresult= (ALLOW_RESTORE) Output macro variable NAME.  Will be
+    given the value of YES or NO depending on whether the user is allowed to
+    restore the load ref.
+  @param [out] outreason= (REASON) Output macro variable NAME.
+    Will be populated with the reason for which the restore decision was made.
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mf_getuser.sas
+  @li mpe_accesscheck.sas
+  @li mpe_getgroups.sas
+
+  <h4> Related Macros </h4>
+  @li mpe_checkrestore.test.sas
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+**/
+
+%macro mpe_checkrestore(load_ref,
+  outresult=ALLOW_RESTORE,
+  outreason=REASON
+);
+
+  %global &outresult &outreason;
+  %let &outresult=NO;
+  %let &outreason=NOTFOUND;
+
+  /* check if there is actually a version to restore */
+  %local chk;
+  %let chk=0;
+  proc sql noprint;
+  select count(*) into: chk from &dc_libref..mpe_audit
+    where load_ref="&load_ref";
+  %if &chk=0 %then %do;
+    %let allow_restore=NO;
+    %let reason=No entry for &load_ref in MPE_AUDIT;
+    %return;
+  %end;
+
+  /* grab user groups */
+  %local user;
+  %let user=%mf_getuser();
+  %mpe_getgroups(user=&user,outds=work.groups)
+
+  /* check if user is admin */
+  %local is_admin;
+  %let is_admin=0;
+  proc sql;
+  select count(*) into: is_admin from work.groups
+    where groupname="&dc_admin_group";
+
+  %if &is_admin>0 %then %do;
+    %let allow_restore=YES;
+    %let reason=IS ADMIN;
+    %return;
+  %end;
+
+  /* check if user has basic access */
+  %local libds;
+  proc sql noprint;
+  select cats(base_lib,'.',base_ds) into: libds
+    from &mpelib..mpe_submit
+    where TABLE_ID="&load_ref";
+  %mpe_accesscheck(&libds,outds=work.access_check
+    ,user=&user
+    ,access_level=EDIT
+  )
+  %if %mf_nobs(access_check)=0 %then %do;
+    %let allow_restore=NO;
+    %let reason=No access in MPE_TABLES;
+    %return;
+  %end;
+
+  /* check if user has column level security rules */
+  proc sql;
+  create table work.cls_rules as
+    select *
+    from &mpelib..mpe_column_level_security
+    where &dc_dttmtfmt. lt tx_to
+      and CLS_SCOPE in ("EDIT",'ALL')
+      and CLS_ACTIVE=1
+      and upcase(CLS_GROUP) in (select upcase(groupname) from work.groups)
+      and CLS_LIBREF="%upcase(&base_lib)"
+      and CLS_TABLE="%upcase(&base_ds)";
+  %if %mf_nobs(work.cls_rules)>0 %then %do;
+    %let allow_restore=NO;
+    %let reason=User has restrictions in MPE_COLUMN_LEVEL_SECURITY;
+    data _null_;
+      set work.cls_rules;
+      putlog (_all_)(=);
+      if _n_>5 then stop;
+    run;
+    %return;
+  %end;
+
+  /* check if user has row level security rules */
+  proc sql;
+  create table work.rls_rules as
+    select *
+    from &mpelib..mpe_row_level_security
+    where &dc_dttmtfmt. lt tx_to
+      and rls_scope in ("EDIT",'ALL')
+      and upcase(rls_group) in (select upcase(groupname) from work.groups)
+      and rls_libref="&base_lib"
+      and rls_table="&base_ds"
+      and rls_active=1;
+  %if %mf_nobs(work.rls_rules)>0 %then %do;
+    %let allow_restore=NO;
+    %let reason=User has restrictions in MPE_ROW_LEVEL_SECURITY;
+    data _null_;
+      set work.rls_rules;
+      putlog (_all_)(=);
+      if _n_>5 then stop;
+    run;
+    %return;
+  %end;
+  %else %do;
+    %let allow_restore=YES;
+    %let reason=CHECKS PASSED;
+  %end;
+%mend mpe_checkrestore;

sas/sasjs/macros/mpe_checkrestore.test.sas

@@ -0,0 +1,59 @@
+/**
+  @file
+  @brief Testing mpe_checkrestore macro
+  @details Checking functionality of mpe_checkrestore.sas macro
+
+  <h4> SAS Macros </h4>
+  @li dc_getsettings.sas
+  @li mpe_checkrestore.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_testservice.sas
+
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+/* first, run a data update */
+%mp_testservice(&appLoc/tests/services/auditors/postdata.test.1,
+  viyacontext=&defaultcontext
+)
+
+/* now grab the latest update */
+%let loadref=0;
+data APPROVE1;
+  set &mpelib..mpe_submit end=last;
+  if last then call symputx('loadref',table_id);
+run;
+
+%global dc_repo_users dc_macros dc_libref dc_staging_area dc_admin_group
+  mpelib dc_dttmtfmt;
+%dc_getsettings()
+
+%put checking it is restorable;
+%global allow_restore reason;
+%mp_assertscope(SNAPSHOT)
+%mpe_checkrestore(&loadref,outresult=ALLOW_RESTORE,outreason=REASON)
+%mp_assertscope(COMPARE,
+  desc=Checking macro variables against previous snapshot,
+  ignorelist=ALLOW_RESTORE REASON
+    MCLIB0_JADP1LEN MCLIB0_JADP2LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+    MCLIB2_JADP1LEN MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking successful submission
+)
+%mp_assert(
+  iftrue=(&ALLOW_RESTORE=YES),
+  desc=Checking restoring is possible
+)
+%mp_assert(
+  iftrue=(&REASON=IS ADMIN),
+  desc=Checking reason code returned
+)

sas/sasjs/macros/mpe_loader.sas

@@ -387,7 +387,7 @@ run;
   /* get log back */
   proc printto log=&logloc;run;
   data _null_; infile tmp; input; putlog _infile_;run;
-  /* scan log for invalid data warning */
+  /* scan log for invalid data warnings */
   data _null_;
     infile tmp;
     input;

sas/sasjs/macros/mpe_makedata.sas

@@ -26,14 +26,6 @@
 %end;
 
 proc sql;
-insert into &lib..mpe_alerts set
-    tx_from=0
-    ,tx_to='31DEC9999:23:59:59'dt
-    ,alert_event='*ALL*'
-    ,alert_lib='*ALL*'
-    ,alert_ds='*ALL*'
-    ,alert_user="&sysuserid";
-
 insert into &lib..mpe_column_level_security set
     tx_from=0
     ,tx_to='31DEC9999:23:59:59'dt

sas/sasjs/macros/mpeinit.sas

@@ -33,7 +33,8 @@
         dc_activation_key /* extracted in dc_getsettings */
         dc_locale /* extracted in dc_getsettings */
         dc_dttmtfmt /* can be overridden in dc_getsettings */
-        _debug
+        _debug /* automatic variable when provided in URL */
+        sasjs_mdebug /* used to show extra info when _debug is enabled */
         ;
 
 %if &mpeinit=1 %then %return;
@@ -110,4 +111,9 @@ run;
   ,msg=%str(Problem during compilation or with STP precode (&syswarningtext))
 )
 
+%if "&_debug"="2477" or "&_debug"="fields,log,trace" or "&_debug"="131"
+%then %do;
+  %let sasjs_mdebug=1;
+%end;
+
 %mend mpeinit;

sas/sasjs/sasjsconfig.json

@@ -233,7 +233,7 @@
         "streamWeb": true,
         "streamWebFolder": "web",
         "webSourcePath": "../client/dist",
-        "streamServiceName": "DataController",
+        "streamServiceName": "DataController2",
         "streamLogo": "favicon.ico",
         "assetPaths": []
       }

sas/sasjs/services/auditors/postdata.test.1.sas

@@ -34,12 +34,17 @@ data work.jsdata;
   SOME_DATETIME=put(dttm2,datetime19.);
   some_time=put(tm2,time.);
   drop dt2 dttm2 tm2;
-  _____DELETE__THIS__RECORD_____='No';
   if _n_=1 then do;
-    primary_key_field=sum(&maxpk,1);
-    some_char='   leadingblanks';
-    some_num=._;
+    _____DELETE__THIS__RECORD_____='Yes';
     output;
+    _____DELETE__THIS__RECORD_____='No';
+    some_char='   leadingblanks';
+    some_bestnum=._;
+    /* modify 1 record and add 4 more */
+    do primary_key_field=&maxpk to (&maxpk+5);
+      some_num=ranuni(0);
+      output;
+    end;
   end;
   else stop;
 run;
@@ -52,7 +57,7 @@ run;
 )
 
 %let status=0;
-data work.sasparams;
+data _null_;
   set web1.sasparams;
   putlog (_all_)(=);
   if status='SUCCESS' then call symputx('status',1);
@@ -91,7 +96,7 @@ data _null_;
   /* the JSON libname engine removes leading blanks!!!! */
   if index(_infile_,'   leadingblanks') then call symputx('leadcheck',1);
   /* there is no clean way to send a special missing - so is sent as string */
-  if index(_infile_,',"SOME_NUM":"_"') then call symputx('speshcheck',1);
+  if index(_infile_,',"SOME_BESTNUM":"_"') then call symputx('speshcheck',1);
 run;
 
 
@@ -103,3 +108,32 @@ run;
   iftrue=(&leadcheck=1),
   desc=Checking special characters were applied
 )
+
+/* Now actually approve the table */
+data work.sascontroltable;
+  ACTION='APPROVE_TABLE';
+  TABLE="&dsid";
+  /* difftime is numeric for approve action */
+  DIFFTIME="%sysfunc(datetime())";
+  output;
+  stop;
+run;
+%mx_testservice(&appLoc/services/auditors/postdata,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable,
+  outlib=web3,
+  outref=wb3,
+  mdebug=&sasjs_mdebug
+)
+
+%let status=0;
+data _null_;
+  set web3.apparams;
+  putlog (_all_)(=);
+  if response='SUCCESS!' then call symputx('status',1);
+run;
+
+%mp_assert(
+  iftrue=(&status=1 and &syscc=0),
+  desc=Checking successful submission
+)

sas/sasjs/services/editors/restore.sas

@@ -0,0 +1,157 @@
+/**
+  @file restore.sas
+  @brief Restores a data version
+  @details Only applies if the history is stored in the audit table
+
+  <h4> SAS Macros </h4>
+  @li dc_assignlib.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_ds2csv.sas
+  @li mp_stripdiffs.sas
+  @li mpeinit.sas
+  @li mpe_checkrestore.sas
+  @li mpe_loader.sas
+
+  <h4> Service Inputs </h4>
+  <h5> restore_in </h5>
+
+  |LOAD_REF:$32|
+  |---|
+  |DCXXXXXX|
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%mpeinit()
+
+%let loadref=;
+data _null_;
+  set work.restore_in;
+  call symputx('loadref',load_ref);
+run;
+
+/**
+  * Check if user has basic access permission to RESTORE the table
+  */
+%put checking access;
+%global allow_restore reason;
+%mpe_checkrestore(&loadref,outresult=ALLOW_RESTORE,outreason=REASON)
+
+%mp_abort(iftrue= (&ALLOW_RESTORE ne YES)
+  ,mac=&_program..sas
+  ,msg=%str(Cannot restore because: &reason)
+)
+
+/* grab the base DS */
+proc sql noprint;
+select cats(base_lib,'.',base_ds) into: tgtds
+  from &mpelib..mpe_submit
+  where TABLE_ID="&loadref";
+
+/* find the audit table */
+select coalescec(audit_libds,"&mpelib..MPE_AUDIT"), loadtype, var_txto
+  into: difftable, :loadtype, :txto
+  from &mpelib..MPE_TABLES
+  where libref="%scan(&tgtds,1,.)"
+        & dsn="%scan(&tgtds,2,.)"
+        & &dc_dttmtfmt<tx_to;
+%mp_abort(iftrue= ("&difftable"="0")
+  ,mac=&_program..sas
+  ,msg=%str(No audit table configured for &tgtds)
+)
+
+/* assign the library */
+%dc_assignlib(READ,%scan(&tgtds,1,.))
+
+/* if the target is an SCD2 table, create a view to get current snapshot */
+%let fltr=%sysfunc(ifc(&loadtype=TXTEMPORAL,&dc_dttmtfmt<&txto,1=1));
+
+/* extract the differences to be applied */
+%mp_stripdiffs(&tgtds,&loadref,&difftable
+  ,outds=work.mp_stripdiffs
+  ,filtervar=fltr
+  ,mdebug=&sasjs_mdebug
+)
+
+/* abort if any issues */
+%mp_abort(iftrue= (&syscc>0)
+  ,mac=&_program..sas
+  ,msg=%str(syscc=&syscc after stripdiffs)
+)
+%mp_abort(iftrue= (%mf_nobs(work.mp_stripdiffs)=0)
+  ,mac=&_program..sas
+  ,msg=%str(THERE ARE NO DIFFERENCES TO APPLY)
+)
+
+/* create a new load ref */
+%let mperef=DC%left(%sysfunc(datetime(),B8601DT19.3))_%substr(
+  %sysfunc(ranuni(0)),3,6)_%substr(%str(&sysjobid    ),1,4);
+
+/* Create package folder */
+%let dir=&mpelocapprovals/&mperef;
+%mf_mkdir(&dir)
+options notes mprint;
+libname approve "&dir";
+
+/* take copy of macvars */
+data _null_;
+  file "&dir/macvars.sas";
+  set sashelp.vmacro;
+  where scope='GLOBAL';
+  put '%let ' name '=' value ';';
+run;
+
+/* copy the diffs dataset */
+data approve.jsdset;
+  length _____DELETE__THIS__RECORD_____ $3;
+  if 0 then call missing(_____DELETE__THIS__RECORD_____);
+  set work.mp_stripdiffs;
+run;
+
+/* export to csv */
+%mp_ds2csv(approve.jsdset
+  ,dlm=COMMA
+  ,outfile="&dir/%trim(&tgtds).csv"
+  ,outencoding="UTF-8"
+  ,headerformat=NAME
+  ,termstr=CRLF
+)
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&_program
+  ,msg=%str(syscc=&syscc when writing the CSV)
+)
+
+%mpe_loader(mperef=&mperef
+  ,submitted_reason_txt=Restoring &loadref
+  ,dc_dttmtfmt=&dc_dttmtfmt
+)
+%mp_abort(mode=INCLUDE)
+
+%mp_abort(
+  iftrue=(%sysfunc(fileexist(%sysfunc(pathname(work))/mf_abort.error))=1)
+  ,mac=&_program..sas
+  ,msg=%str(mf_abort.error=1)
+)
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&_program..sas
+  ,msg=%str(syscc=&syscc)
+)
+
+/* send relevant SUCCESS values */
+data work.restore_out;
+  loadref="&mperef";
+run;
+
+
+%webout(OPEN)
+%webout(OBJ,restore_out)
+%webout(CLOSE)
+

sas/sasjs/services/editors/restore.test.sas

@@ -0,0 +1,109 @@
+/**
+  @file
+  @brief testing restore process
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mx_testservice.sas
+  @li mp_assert.sas
+
+
+**/
+
+%let _program=&appLoc/services/editors/restore;
+
+/* take a snapshot of the table for later comparison */
+proc sort data=&mpelib..mpe_x_test out=work.origds;
+by primary_key_field;
+run;
+
+/* run an update */
+%mx_testservice(&appLoc/tests/services/auditors/postdata.test.1,
+  viyacontext=&defaultcontext
+)
+
+/* grab the loadref for later reversion */
+%let loadref=0;
+data APPROVE1;
+  set &mpelib..mpe_submit end=last;
+  if last then call symputx('loadref',table_id);
+run;
+
+/* run another update for good measure */
+%mx_testservice(&appLoc/tests/services/auditors/postdata.test.1,
+  viyacontext=&defaultcontext
+)
+
+/* now we are ready to revert */
+data work.restore_in;
+  load_ref="&loadref";
+  output;
+  stop;
+run;
+%mx_testservice(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.restore_in,
+  outlib=web1,
+  mdebug=&sasjs_mdebug
+)
+
+/* check for success */
+%let loadref=0;
+data work.restore_out;
+  set web1.restore_out;
+  putlog (_all_)(=);
+  call symputx('newref',loadref);
+run;
+%mp_assert(
+  iftrue=(&newref ne 0),
+  desc=Checking successful submission of a reversion,
+  outds=work.test_results
+)
+
+/* approve the reversion */
+data work.sascontroltable;
+  ACTION='APPROVE_TABLE';
+  TABLE="&newref";
+  /* difftime is numeric for approve action */
+  DIFFTIME="%sysfunc(datetime())";
+  output;
+  stop;
+run;
+%mx_testservice(&appLoc/services/auditors/postdata,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable,
+  outlib=web3,
+  outref=wb3,
+  mdebug=&sasjs_mdebug
+)
+
+%let status=0;
+data _null_;
+  set web3.apparams;
+  putlog (_all_)(=);
+  if response='SUCCESS!' then call symputx('status',1);
+run;
+
+%mp_assert(
+  iftrue=(&status=1 and &syscc=0),
+  desc=Checking successful submission of reversion
+)
+
+/* compare snapshot with latest data */
+proc sort data=&mpelib..mpe_x_test out=work.compareds;
+by primary_key_field;
+run;
+
+proc compare base=work.origds compare=work.compareds
+  out=work.resultds outnoequal;
+run;
+data _null_;
+  set work.resultds;
+  if &sasjs_mdebug=1 then putlog (_all_)(=);
+  if _n_>10 then stop;
+run;
+%mp_assert(
+  iftrue=(&sysinfo le 41),
+  desc=Checking compare of MPE_X_TEST,
+  outds=work.test_results
+)

sas/sasjs/services/public/getchangeinfo.sas

@@ -4,13 +4,9 @@
   @details
 
   <h4> SAS Macros </h4>
-  @li dc_assignlib.sas
   @li mf_getengine.sas
-  @li mf_getuser.sas
-  @li mf_nobs.sas
   @li mp_abort.sas
-  @li mpe_accesscheck.sas
-  @li mpe_getgroups.sas
+  @li mpe_checkrestore.sas
 
   <h4> Service Inputs </h4>
   <h5> sascontroltable </h5>
@@ -25,6 +21,12 @@
   @li allow_restore YES if a user can restore, else NO
   @li REASON reason why a restore is / is no possible
 
+  <h4> Data Inputs </h4>
+  @li MPE_AUDIT
+  @li MPE_COLUMN_LEVEL_SECURITY
+  @li MPE_ROW_LEVEL_SECURITY
+  @li MPE_SUBMIT
+
   @version 9.2
   @author 4GL Apps Ltd
   @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
@@ -41,10 +43,6 @@ data _null_;
   call symputx('table',table);
 run;
 
-%dc_assignlib(WRITE,%scan(&table,1,.))
-
-%let max_ver_dttm=0;
-
 data APPROVE1;
   set &mpelib..mpe_submit
     (rename=(SUBMITTED_ON_DTTM=submitted_on REVIEWED_ON_DTTM=REVIEWED_ON));
@@ -52,7 +50,6 @@ data APPROVE1;
   TABLE_NM=cats(base_lib,'.',base_ds);
   BASE_TABLE=table_nm;
   call symputx('base_lib',base_lib);
-  call symputx('base_ds',base_ds);
   REVIEWED_ON_DTTM=put(reviewed_on,datetime19.);
   SUBMITTED_ON_DTTM=put(submitted_on,datetime19.);
 run;
@@ -61,91 +58,8 @@ run;
   * Check if user has basic access permission to RESTORE the table
   */
 %put checking access;
-
 %global allow_restore reason;
-%let allow_restore=NO;
-%let reason=NOTFOUND;
-
-%macro access_check();
-
-  /* grab user groups */
-  %let user=%mf_getuser();
-  %mpe_getgroups(user=&user,outds=work.groups)
-
-  /* check if user is admin */
-  %let is_admin=0;
-  proc sql;
-  select count(*) into: is_admin from work.groups where groupname="&MPEADMINS";
-
-  %if &is_admin>0 %then %do;
-    %let allow_restore=YES;
-    %let reason=IS ADMIN;
-    %return;
-  %end;
-
-  /* check if user has basic access */
-  %mpe_accesscheck(&base_lib..&base_ds,outds=work.access_check
-    ,user=&user
-    ,access_level=EDIT
-  )
-  %if %mf_nobs(access_check)=0 %then %do;
-    %let allow_restore=NO;
-    %let reason=No access in MPE_TABLES;
-    %return;
-  %end;
-
-  /* check if user has column level security rules */
-  proc sql;
-  create table work.cls_rules as
-    select *
-    from &mpelib..mpe_column_level_security
-    where &dc_dttmtfmt. lt tx_to
-      and CLS_SCOPE in ("EDIT",'ALL')
-      and CLS_ACTIVE=1
-      and upcase(CLS_GROUP) in (select upcase(groupname) from work.groups)
-      and CLS_LIBREF="%upcase(&base_lib)"
-      and CLS_TABLE="%upcase(&base_ds)";
-  %if %mf_nobs(work.cls_rules)>0 %then %do;
-    %let allow_restore=NO;
-    %let reason=User has restrictions in MPE_COLUMN_LEVEL_SECURITY;
-    data _null_;
-      set work.cls_rules;
-      putlog (_all_)(=);
-      if _n_>5 then stop;
-    run;
-    %return;
-  %end;
-
-  /* check if user has row level security rules */
-  proc sql;
-  create table work.rls_rules as
-    select *
-    from &mpelib..mpe_row_level_security
-    where &dc_dttmtfmt. lt tx_to
-      and rls_scope in ("EDIT",'ALL')
-      and upcase(rls_group) in (select upcase(groupname) from work.groups)
-      and rls_libref="&base_lib"
-      and rls_table="&base_ds"
-      and rls_active=1;
-  %if %mf_nobs(work.rls_rules)>0 %then %do;
-    %let allow_restore=NO;
-    %let reason=User has restrictions in MPE_ROW_LEVEL_SECURITY;
-    data _null_;
-      set work.rls_rules;
-      putlog (_all_)(=);
-      if _n_>5 then stop;
-    run;
-    %return;
-  %end;
-  %else %do;
-    %let allow_restore=YES;
-    %let reason=CHECKS PASSED;
-    %return;
-  %end;
-%mend access_check;
-
-%access_check();
-
+%mpe_checkrestore(&table,outresult=ALLOW_RESTORE,outreason=REASON)
 
 data work.jsParams;
   set approve1;

sas/sasjs/services/public/getchangeinfo.test.sas

@@ -92,6 +92,6 @@ run;
   desc=Checking data was returned
 )
 %mp_assert(
-  iftrue=(&allow_edit=YES),
-  desc=Checking admin user can restore
+  iftrue=(&ALLOW_RESTORE=NO),
+  desc=Checking admin user cannot restore - as table was not approved
 )