Adding queries (#94)

* changed licences to set to false so key shows even when no license, added another commented out service

* cleaning up to refactor out processsing

* ectracted report generating and report processing out of index.js

* cleaned formatting of queries in api readme

api/README.md

@@ -6,60 +6,97 @@ This folder contains documentation for the GraphQL API component of the ruok-ser
 
 Run `make` to create a local `venv` folder (`.gitignore`d) and `pip install` all of the `requirements.txt` dependencies into the `venv` folder. The default target in the `Makefile` also activates the virtual environment.
 
-### Sample Queries
+### Sample AQL Queries to start from 
+TODO - add to src/query.py
+
+#### All endpoints
 ```
 FOR doc in endpointNodes
     FILTER doc.kind == "Github"
     RETURN {"url": doc.url, "visibility": doc.visibility, "Kind": doc.kind}
 ```
-
 ```
 FOR doc in endpointNodes
     RETURN doc
 ```
 
-Failed accessibility:
+#### GitHub Endpoints with API
 ```
-FOR doc in endpointNodes
-    FILTER doc.kind == 'Web' 
-LET failedChecks = (
-    FOR check IN doc.accessibility
-      FILTER check.check_passes == false
-      RETURN check
-  )
-  FILTER LENGTH(failedChecks) > 0
+FOR doc IN endpointNodes
+  FILTER doc.kind == 'Github' && doc.api == true
+  RETURN {"endpoint":doc.url}
+```
+
+#### All failing checks for all GitHub Repos
+```
+FOR doc IN endpointNodes
+  FILTER doc.kind == 'Github'
   RETURN {
-    url: doc.url,
-    failedChecks: failedChecks
+    web_url: doc.url, 
+    failing_checks: (
+      FOR key IN ATTRIBUTES(doc)
+        FILTER doc[key].check_passes == false
+        RETURN key
+    )
   }
-  ```
+```
 
+#### All Trivy vulerabilities (with high/ critical status)
+```
+FOR doc IN endpointNodes
+  FILTER doc.kind == 'Github' && doc.trivy_repo_vulnerability.metadata
+  LET vulnerabilities = doc.trivy_repo_vulnerability.metadata
+  LET filtered_vulnerabilities = (
+    FOR vuln IN vulnerabilities
+      FILTER vuln.severity == 'HIGH' || vuln.severity == 'CRITICAL'
+      RETURN vuln
+  )
+  RETURN { "endpoint": doc.url, "high_critical_vulnerabilities": filtered_vulnerabilities }
+```
 
+#### Filter for failing web checks
+```
 FOR doc IN endpointNodes
   FILTER doc.kind == 'Web'
   RETURN {
-    url: doc.url,
-    check_passes: doc.accessibility[0].check_passes 
+    web_url: doc.url, 
+    page: (
+      FOR endpoint IN doc.accessibility
+        RETURN {
+          page: endpoint.url,
+          failing_checks: (
+            FOR key IN ATTRIBUTES(endpoint)
+              FILTER endpoint[key].check_passes == "false"
+              RETURN { 
+                check: key, 
+                check_passes: endpoint[key].check_passes 
+              }
+          )
+        }
+    )
   }
+```
 
-
-  FOR doc IN endpointNodes
-    FILTER doc.kind == 'Web'
-//    RETURN {
-//        url: doc.url,
-//        pages: (
-          FOR page IN doc.accessibility
-//            FOR pageKey, page IN OBJECT_VALUES(doc.accessibility)
-//                RETURN {
-//                    page: page.url,
-
-//                    checks: (
- //                       FOR check IN page 
-                            RETURN {
-                              //check_passes: check.check_passes,
-                              url
-                              }
-//                    )
-//                }
-//        )
-//  }
+Passing web checks
+```
+FOR doc IN endpointNodes
+  FILTER doc.kind == 'Web'
+  RETURN {
+    web_url: doc.url, 
+    page: (
+      FOR endpoint IN doc.accessibility
+        RETURN {
+          page: endpoint.url,
+          passing_checks: (
+            FOR key IN ATTRIBUTES(endpoint)
+              FILTER endpoint[key].check_passes == "true"
+              RETURN { 
+                check: key, 
+                check_passes: endpoint[key].check_passes 
+              }
+          )
+        }
+    )
+  }
+```
+

scanners/github-cloned-repo/index.js

@@ -68,8 +68,7 @@ process.on('SIGINT', () => process.exit(0))
                         kind: "Github"
                         owner: "${orgName}"
                         repo: "${repoName}"
-                        api: ${results.hasApiDirectory.checkPasses}
-                        
+                        api: ${results.hasApiDirectory.checkPasses}                       
                         hasSecurityMd: {
                             checkPasses: ${results.hasSecurityMd.checkPasses}
                             metadata: ${results.hasSecurityMd.metadata}
@@ -125,6 +124,7 @@ await nc.closed();
 // nats pub "EventsScanner.githubEndpoints" "{\"endpoint\":\"https://github.com/PHACDataHub/phac-bots\"}"
 // nats pub "EventsScanner.githubEndpoints" "{\"endpoint\":\"https://github.com/PHACDataHub/pelias-canada\"}"
 // nats pub "EventsScanner.githubEndpoints" "{\"endpoint\":\"https://github.com/PHACDataHub/safe-inputs\"}"
+// nats pub "EventsScanner.githubEndpoints" "{\"endpoint\":\"https://github.com/PHACDataHub/data-catalog\"}"
 
 
 // nats pub "EventsScanner.githubEndpoints" "{\"endpoint\":\"https://github.com/PHACDataHub/csi-projects\"}"

scanners/github-octokit/index.js

@@ -58,7 +58,7 @@ process.on('SIGINT', () => process.exit(0))
               repo: "${repoName}"
               description: ${result.GetRepoDetailsStrategy.metadata.description ? JSON.stringify(result.GetRepoDetailsStrategy.metadata.description) :  null}
               visibility: ${JSON.stringify(result.GetRepoDetailsStrategy.metadata.visibility)}
-              license: ${result.GetRepoDetailsStrategy.metadata.license ? JSON.stringify(result.GetRepoDetailsStrategy.metadata.license) :  null}
+              license: ${result.GetRepoDetailsStrategy.metadata.license ? JSON.stringify(result.GetRepoDetailsStrategy.metadata.license) :  false}
               programmingLanguage: ["${Array.from(Object.keys(result.ProgrammingLanguagesStrategy.metadata)).join('", "')}"]
               automatedSecurityFixes: {
                 checkPasses: ${result.AutomatedSecurityFixesStrategy.checkPasses}

scanners/web-endpoint/index.js

@@ -13,69 +13,55 @@ const {
   NATS_SUB_STREAM,
 } = process.env;
 
-
 // NATs connection 
 const nc = await connect({ servers: NATS_URL, })
 const jc = JSONCodec()
-
 const sub = nc.subscribe(NATS_SUB_STREAM)
-console.log('🚀 Connected to NATS server - listening on ...', sub.subject, "channel...");
+console.log('Connected to NATS server - listening on ...', sub.subject, "channel...");
 
 process.on('SIGTERM', () => process.exit(0))
 process.on('SIGINT', () => process.exit(0))
-
   ; (async () => {
-
     const browser = await puppeteer.launch({
       executablePath: '/usr/bin/google-chrome',
       args: ['--no-sandbox', '--disable-setuid-sandbox'],
       headless: "new",
     });
-
     for await (const message of sub) {
       const webEventPayload = await jc.decode(message.data)
       console.log(webEventPayload)
       const { endpoint } = webEventPayload
-
       const pageInstance = await browser.newPage();
       await pageInstance.setBypassCSP(true);
-
       if (await isWebEndpointType(endpoint, pageInstance)) { //filtering for only web endpoints
         // Get pages for webendpoints (with url slugs)
         const pages = await getPages(endpoint, pageInstance, browser);
-
-        // let webEndpointAxeResults = {}  //form response
-
-        for (const pageToEvaluate of pages) {
-          console.log('Evaluating page: ', pageToEvaluate)
-          const axeReport = await evaluateAccessibility(pageToEvaluate, pageInstance, browser)
-
-          const accessibilityPages = processAxeReport(axeReport, pageToEvaluate);
-
-
-          const mutation = gql`
-              mutation {
-                webEndpoint(
-                  endpoint: {
-                    url: "${endpoint}"
-                    kind: "Web"
-                    accessibility: ${JSON.stringify(accessibilityPages, null, 4).replace(/"([^"]+)":/g, '$1:')}
-                  }
-                )
-              }
-              `;
-
-          console.log('*****************************************************\n', mutation)
-          // API connection 
-          const graphQLClient = new GraphQLClient(GRAPHQL_URL);
-          // Write mutation to GraphQL API
-
-          const mutationResponse = await graphQLClient.request(mutation);
-          console.info("wrote mutation to GraphQL API with response", mutationResponse);
-
-        }
+        const accessibilityPages = await processAxeReport(pageInstance, pages, browser)
+
+        const mutation = gql`
+            mutation {
+              webEndpoint(
+                endpoint: {
+                  url: "${endpoint}"
+                  kind: "Web"
+                  accessibility: ${JSON.stringify(accessibilityPages, null, 4).replace(/"([^"]+)":/g, '$1:')}
+                }
+              )
+            }
+            `;
+        // console.log('*****************************************************\n', mutation)
+
+        // API connection 
+        const graphQLClient = new GraphQLClient(GRAPHQL_URL);
+
+        // Write mutation to GraphQL API
+        const mutationResponse = await graphQLClient.request(mutation);
+
+        console.info("wrote mutation to GraphQL API with response", mutationResponse);
+
         await pageInstance.close()
-      }
+      } // end of if web endpoint
+
       console.log(`Accessibility scan of ${endpoint} complete.`)
       console.log('-----')
     }
@@ -84,8 +70,5 @@ process.on('SIGINT', () => process.exit(0))
 
   })();
 
-
-await nc.closed();
-
 // nats pub "EventsScanner.webEndpoints" "{\"endpoint\":\"https://safeinputs.phac.alpha.canada.ca\"}"
-// nats pub "EventsScanner.webEndpoints" "{\"endpoint\":\"https://hopic-sdpac.phac-aspc.alpha.canada.ca\"}"
+// nats pub "EventsScanner.webEndpoints" "{\"endpoint\":\"https://hopic-sdpac.phac-aspc.alpha.canada.ca\"}"

scanners/web-endpoint/src/process-axe-report.js

@@ -1,29 +1,38 @@
 // processAxeReport.js
+import { evaluateAccessibility } from './accessibility-checks.js'
 
-export function processAxeReport(axeReport, pageToEvaluate) {
-    const webEndpointAxeResults = {};
-
+export async function processAxeReport(pageInstance, pages, browser) {
+  let webEndpointAxeResults = {}  //form response
+
+  for (const pageToEvaluate of pages) {
+    console.log('Evaluating page: ', pageToEvaluate)
+    const axeReport = await evaluateAccessibility(pageToEvaluate, pageInstance, browser)
+
+    // Process report (create camelCase key for each evaluated criterion )
     for (let i = 0; i < axeReport.length; i++) {
-        const camelize = s => s.replace(/-./g, x => x[1].toUpperCase())
-        const criterion = axeReport[i];
-        const criterionKey = Object.keys(criterion )[0];
-        const criterionValue = criterion [criterionKey];
-        const criterionKeyCamelCase = camelize(criterionKey);
-        if (!webEndpointAxeResults[pageToEvaluate]) {
-          webEndpointAxeResults[pageToEvaluate] = {}
-        }
-        if (typeof criterionValue.checkPasses === 'boolean') {
-          criterionValue.checkPasses = criterionValue.checkPasses.toString()
-        }
-        webEndpointAxeResults[pageToEvaluate][criterionKeyCamelCase] = {
-          checkPasses: criterionValue.checkPasses,
-          metadata: criterionValue.metadata,
-        }
+      const camelize = s => s.replace(/-./g, x => x[1].toUpperCase())
+      const criterion = axeReport[i];
+      const criterionKey = Object.keys(criterion )[0];
+      const criterionValue = criterion [criterionKey];
+      const criterionKeyCamelCase = camelize(criterionKey);
+      if (!webEndpointAxeResults[pageToEvaluate]) {
+        webEndpointAxeResults[pageToEvaluate] = {}
+      }
+      if (typeof criterionValue.checkPasses === 'boolean') {
+        criterionValue.checkPasses = criterionValue.checkPasses.toString()
       }
+      webEndpointAxeResults[pageToEvaluate][criterionKeyCamelCase] = {
+        checkPasses: criterionValue.checkPasses,
+        metadata: criterionValue.metadata,
+      }
+    }
 
-    return Object.keys(webEndpointAxeResults).map(page => ({
+  }
+  const accessibilityPages = Object.keys(webEndpointAxeResults).map(page => {
+    return {
       url: page,
       ...webEndpointAxeResults[page],
-    }));
-  }
-
+    }
+  })
+  return accessibilityPages
+}