Refactor web access (#93)

* working to get a more encompasing/ yet minimally required github token in place - not yet working

* added missing metadata and set to null

* set gitleaks pass results to null

* Added trivy to the dev container dockerfile. Modified gitleaks to have null metadata is check passes

* hadolint metadata is set to null if no dockerfiles found

* modified hadolint to pick up any file with 'dockerfile' in filename

* set metadata to null in cases where it's n/a

* starting to clean up accessibility checks as its not easy to query from db

* starting to similfy web accessibility code

* accessibility scan works as before, but moved out page close out of the page evaluation loop.

* minor clean up before mutation refactor.

* changed api to be false and not null in github cloned repo check.

* returned programming languages to empty array if none, in order to have it displayed in db

* Added to do note for all languagues to change color to size or percentage

api/README.md

@@ -4,4 +4,62 @@ This folder contains documentation for the GraphQL API component of the ruok-ser
 
 ## Getting Started with Local Development
 
-Run `make` to create a local `venv` folder (`.gitignore`d) and `pip install` all of the `requirements.txt` dependencies into the `venv` folder. The default target in the `Makefile` also activates the virtual environment.
+Run `make` to create a local `venv` folder (`.gitignore`d) and `pip install` all of the `requirements.txt` dependencies into the `venv` folder. The default target in the `Makefile` also activates the virtual environment.
+
+### Sample Queries
+```
+FOR doc in endpointNodes
+    FILTER doc.kind == "Github"
+    RETURN {"url": doc.url, "visibility": doc.visibility, "Kind": doc.kind}
+```
+
+```
+FOR doc in endpointNodes
+    RETURN doc
+```
+
+Failed accessibility:
+```
+FOR doc in endpointNodes
+    FILTER doc.kind == 'Web' 
+LET failedChecks = (
+    FOR check IN doc.accessibility
+      FILTER check.check_passes == false
+      RETURN check
+  )
+  FILTER LENGTH(failedChecks) > 0
+  RETURN {
+    url: doc.url,
+    failedChecks: failedChecks
+  }
+  ```
+
+
+FOR doc IN endpointNodes
+  FILTER doc.kind == 'Web'
+  RETURN {
+    url: doc.url,
+    check_passes: doc.accessibility[0].check_passes 
+  }
+
+
+  FOR doc IN endpointNodes
+    FILTER doc.kind == 'Web'
+//    RETURN {
+//        url: doc.url,
+//        pages: (
+          FOR page IN doc.accessibility
+//            FOR pageKey, page IN OBJECT_VALUES(doc.accessibility)
+//                RETURN {
+//                    page: page.url,
+
+//                    checks: (
+ //                       FOR check IN page 
+                            RETURN {
+                              //check_passes: check.check_passes,
+                              url
+                              }
+//                    )
+//                }
+//        )
+//  }

scanners/github-cloned-repo/index.js

@@ -68,7 +68,8 @@ process.on('SIGINT', () => process.exit(0))
                         kind: "Github"
                         owner: "${orgName}"
                         repo: "${repoName}"
-                        api: ${results.hasApiDirectory.checkPasses? results.hasApiDirectory.checkPasses : null}
+                        api: ${results.hasApiDirectory.checkPasses}
+                        
                         hasSecurityMd: {
                             checkPasses: ${results.hasSecurityMd.checkPasses}
                             metadata: ${results.hasSecurityMd.metadata}
@@ -87,6 +88,7 @@ process.on('SIGINT', () => process.exit(0))
                         }
                         trivyRepoVulnerability: {
                             checkPasses: ${results.trivy_repo_vulnerability ? results.trivy_repo_vulnerability.checkPasses : null}
+
                             metadata: ${results.trivy_repo_vulnerability && results.trivy_repo_vulnerability.metadata !== undefined ?
                                 JSON.stringify(results.trivy_repo_vulnerability.metadata, null, 4).replace(/"([^"]+)":/g, '$1:') :
                                 null
@@ -128,4 +130,5 @@ await nc.closed();
 // nats pub "EventsScanner.githubEndpoints" "{\"endpoint\":\"https://github.com/PHACDataHub/csi-projects\"}"
 // metadata: ${JSON.stringify(results.trivy_repo_vulnerability.metadata, null, 4).replace(/"([^"]+)":/g, '$1:')}
 
-// checkPasses: ${JSON.stringify(results.gitleaks.checkPasses, null, 4).replace(/"([^"]+)":/g, '$1:')}
+// checkPasses: ${JSON.stringify(results.gitleaks.checkPasses, null, 4).replace(/"([^"]+)":/g, '$1:')}
+//                        api: ${results.hasApiDirectory.checkPasses? results.hasApiDirectory.checkPasses : null}

scanners/github-octokit/index.js

@@ -16,10 +16,10 @@ const {
 } = process.env;
 
 // Authenicate with GitHub 
+
 const octokit = new Octokit({ auth: GITHUB_TOKEN_CLASSIC, });
 // const octokit = new Octokit({ auth: GITHUB_TOKEN_FINE_GRAINED, });
 
-
 // NATs connection 
 const nc = await connect({ servers: NATS_URL, })
 const jc = JSONCodec();
@@ -59,11 +59,8 @@ process.on('SIGINT', () => process.exit(0))
               description: ${result.GetRepoDetailsStrategy.metadata.description ? JSON.stringify(result.GetRepoDetailsStrategy.metadata.description) :  null}
               visibility: ${JSON.stringify(result.GetRepoDetailsStrategy.metadata.visibility)}
               license: ${result.GetRepoDetailsStrategy.metadata.license ? JSON.stringify(result.GetRepoDetailsStrategy.metadata.license) :  null}
-              programmingLanguage: ${
-                Array.from(Object.keys(result.ProgrammingLanguagesStrategy.metadata)).length > 0
-                  ? `["${Array.from(Object.keys(result.ProgrammingLanguagesStrategy.metadata)).join('", "')}"]`
-                  : null
-              }              automatedSecurityFixes: {
+              programmingLanguage: ["${Array.from(Object.keys(result.ProgrammingLanguagesStrategy.metadata)).join('", "')}"]
+              automatedSecurityFixes: {
                 checkPasses: ${result.AutomatedSecurityFixesStrategy.checkPasses}
                 metadata: {
                   enabled: ${result.AutomatedSecurityFixesStrategy.metadata.enabled}
@@ -107,5 +104,9 @@ await nc.closed();
 
 // securityAndAnalysis: "${JSON.stringify(result.GetRepoDetailsStrategy.metadata.security_and_analysis, null, 4).replace(/"([^"]+)":/g, '$1:')}"
 
-// rules: ${payload.BranchProtectionStrategy.metadata.rules.length > 0 ? `["${Array.from(payload.BranchProtectionStrategy.metadata.rules).join('", "')}"]` : "null"},
-// programmingLanguage: ["${Array.from(Object.keys(result.ProgrammingLanguagesStrategy.metadata)).join('", "')}"]       
+
+// programmingLanguage: ${
+//   Object.keys(result.ProgrammingLanguagesStrategy.metadata).length > 0
+//     ? `["${Object.keys(result.ProgrammingLanguagesStrategy.metadata).join('", "')}"]`
+//     : '[]'   
+// }   

scanners/github-octokit/src/all-langauges.js

@@ -50,8 +50,10 @@ export class ProgrammingLanguagesStrategy extends OctokitCheckStrategy {
       return {
         ...obj,
         [item.node.name]: item.node.color,
+        // TODO - change colour to size/ percentage (https://docs.github.com/en/graphql/reference/objects#languageconnection)
       };
     }, {});
+
 
     return {
       checkPasses: null,

scanners/web-endpoint/index.js

@@ -3,6 +3,7 @@ import { GraphQLClient, gql } from 'graphql-request'
 import { getPages } from './src/get-url-slugs.js'
 import { isWebEndpointType } from './src/check-endpoint-type.js'
 import { evaluateAccessibility } from './src/accessibility-checks.js'
+import { processAxeReport } from './src/process-axe-report.js';
 import puppeteer from 'puppeteer';
 import 'dotenv-safe/config.js'
 
@@ -43,59 +44,44 @@ process.on('SIGINT', () => process.exit(0))
         // Get pages for webendpoints (with url slugs)
         const pages = await getPages(endpoint, pageInstance, browser);
 
-        let webEndpointAxeResults = {}  //form response
+        // let webEndpointAxeResults = {}  //form response
 
         for (const pageToEvaluate of pages) {
           console.log('Evaluating page: ', pageToEvaluate)
           const axeReport = await evaluateAccessibility(pageToEvaluate, pageInstance, browser)
 
-          // Process report (create camelCase key for each evaluated criterion )
-          for (let i = 0; i < axeReport.length; i++) {
-            const camelize = s => s.replace(/-./g, x => x[1].toUpperCase())
-            const criterion = axeReport[i];
-            const criterionKey = Object.keys(criterion )[0];
-            const criterionValue = criterion [criterionKey];
-            const criterionKeyCamelCase = camelize(criterionKey);
-            if (!webEndpointAxeResults[pageToEvaluate]) {
-              webEndpointAxeResults[pageToEvaluate] = {}
-            }
-            if (typeof criterionValue.checkPasses === 'boolean') {
-              criterionValue.checkPasses = criterionValue.checkPasses.toString()
-            }
-            webEndpointAxeResults[pageToEvaluate][criterionKeyCamelCase] = {
-              checkPasses: criterionValue.checkPasses,
-              metadata: criterionValue.metadata,
-            }
-          }
+          const accessibilityPages = processAxeReport(axeReport, pageToEvaluate);
+
+
+          const mutation = gql`
+              mutation {
+                webEndpoint(
+                  endpoint: {
+                    url: "${endpoint}"
+                    kind: "Web"
+                    accessibility: ${JSON.stringify(accessibilityPages, null, 4).replace(/"([^"]+)":/g, '$1:')}
+                  }
+                )
+              }
+              `;
+
+          console.log('*****************************************************\n', mutation)
+          // API connection 
+          const graphQLClient = new GraphQLClient(GRAPHQL_URL);
+          // Write mutation to GraphQL API
+
+          const mutationResponse = await graphQLClient.request(mutation);
+          console.info("wrote mutation to GraphQL API with response", mutationResponse);
+
         }
-        const accessibilityPages = Object.keys(webEndpointAxeResults).map(page => {
-          return {
-            url: page,
-            ...webEndpointAxeResults[page],
-          }
-        })
-        const mutation = gql`
-            mutation {
-              webEndpoint(
-                endpoint: {
-                  url: "${endpoint}"
-                  kind: "Web"
-                  accessibility: ${JSON.stringify(accessibilityPages, null, 4).replace(/"([^"]+)":/g, '$1:')}
-                }
-              )
-            }
-            `;
-        // API connection 
-        const graphQLClient = new GraphQLClient(GRAPHQL_URL);
-        // Write mutation to GraphQL API
-        // console.log('*****************************************************\n', mutation)
-        const mutationResponse = await graphQLClient.request(mutation);
-        console.info("wrote mutation to GraphQL API with response", mutationResponse);
         await pageInstance.close()
       }
+      console.log(`Accessibility scan of ${endpoint} complete.`)
+      console.log('-----')
     }
     await browser.close()
 
+
   })();
 
 

scanners/web-endpoint/src/accessibility-checks.js

@@ -9,7 +9,7 @@
 
 import AxePuppeteer  from "@axe-core/puppeteer";
 import puppeteer from 'puppeteer'
-// import fs from 'fs'
+
 
 export async function evaluateAccessibility(url, page, browser) {
     await page.goto(url, { cache: 'no-store' })

scanners/web-endpoint/src/process-axe-report.js

@@ -0,0 +1,29 @@
+// processAxeReport.js
+
+export function processAxeReport(axeReport, pageToEvaluate) {
+    const webEndpointAxeResults = {};
+
+    for (let i = 0; i < axeReport.length; i++) {
+        const camelize = s => s.replace(/-./g, x => x[1].toUpperCase())
+        const criterion = axeReport[i];
+        const criterionKey = Object.keys(criterion )[0];
+        const criterionValue = criterion [criterionKey];
+        const criterionKeyCamelCase = camelize(criterionKey);
+        if (!webEndpointAxeResults[pageToEvaluate]) {
+          webEndpointAxeResults[pageToEvaluate] = {}
+        }
+        if (typeof criterionValue.checkPasses === 'boolean') {
+          criterionValue.checkPasses = criterionValue.checkPasses.toString()
+        }
+        webEndpointAxeResults[pageToEvaluate][criterionKeyCamelCase] = {
+          checkPasses: criterionValue.checkPasses,
+          metadata: criterionValue.metadata,
+        }
+      }
+
+    return Object.keys(webEndpointAxeResults).map(page => ({
+      url: page,
+      ...webEndpointAxeResults[page],
+    }));
+  }
+