Merge pull request #65 from PHACDataHub/scanner-graphql

Scanners integration with GraphQL API

.devcontainer/Dockerfile

@@ -4,6 +4,22 @@ ARG VARIANT=18-bullseye
 FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:${VARIANT}
 
 
+#   ____        _   _                 
+#  |  _ \ _   _| |_| |__   ___  _ __  
+#  | |_) | | | | __| '_ \ / _ \| '_ \ 
+#  |  __/| |_| | |_| | | | (_) | | | |
+#  |_|    \__, |\__|_| |_|\___/|_| |_|
+#         |___/                       
+
+RUN apt-get update
+RUN apt-get install -y python3.10
+RUN apt-get install -y curl
+RUN curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh && \
+    bash miniconda.sh -bfp /home/node/.local/bin/condabin && \
+    rm miniconda.sh && \
+    sudo chown node -R /home/node && \
+    sudo echo "export PATH=$PATH:/home/node/.local/bin/condabin/bin" >> /home/node/.bashrc
+
 #       _                 _              _     
 #    __| | _____   __    | |_ ___   ___ | |___ 
 #   / _` |/ _ \ \ / /____| __/ _ \ / _ \| / __|
@@ -25,25 +41,21 @@ RUN curl -fsSL https://github.com/nats-io/natscli/releases/download/v0.1.1/nats-
     chmod +x nats-0.1.1-linux-amd64/nats && \
     mv nats-0.1.1-linux-amd64/nats /usr/local/bin/nats && \
     rm -r nats-0.1.1-linux-amd64 && rm nats-0.1.1-linux-amd64.zip
-# Install dependencies for Chrome
-RUN sudo apt-get update && \
-    sudo apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libasound2
+# # Install dependencies for Chrome
+# RUN sudo apt-get update && \
+#     sudo apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libasound2
 
-#   ____        _   _                 
-#  |  _ \ _   _| |_| |__   ___  _ __  
-#  | |_) | | | | __| '_ \ / _ \| '_ \ 
-#  |  __/| |_| | |_| | | | (_) | | | |
-#  |_|    \__, |\__|_| |_|\___/|_| |_|
-#         |___/                       
+# We don't need the standalone Chromium
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
 
-RUN apt-get update
-RUN apt-get install -y python3.10
-RUN apt-get install -y curl
-RUN curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh && \
-    bash miniconda.sh -bfp /home/node/.local/bin/condabin && \
-    rm miniconda.sh && \
-    sudo chown -R /home/node && \
-    sudo echo "export PATH=$PATH:/home/node/.local/bin/condabin/bin" >> /home/node/.zshrc
+# Install Google Chrome Stable and fonts
+# Note: this installs the necessary libs to make the browser work with Puppeteer.
+RUN apt-get update && apt-get install gnupg wget -y && \
+  wget --quiet --output-document=- https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor > /etc/apt/trusted.gpg.d/google-archive.gpg && \
+  sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' && \
+  apt-get update && \
+  apt-get install google-chrome-stable -y --no-install-recommends && \
+  rm -rf /var/lib/apt/lists/*
 
 # Start container as non-root user
 USER node

.github/workflows/publish.yml

@@ -3,7 +3,7 @@ on:
   push:
     branches:
     - main
-    - graph-updater
+    - scanner-graphql
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -19,6 +19,8 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           if [ -f docs/requirements.txt ]; then pip install -r docs/requirements.txt; fi
+          pip install mkdocstrings
+          pip install mkdocstrings-python
 
       - name: Build
         run: |

Makefile

@@ -16,14 +16,21 @@ CONTAINER_RUNTIME := $(shell command -v podman 2> /dev/null || echo docker)
 #  |_.__/ \__,_|_|_|\__,_|
 
 # Build all images in the repo
-build: build-api-image build-webhook-server-image build-graph-updater
+build: build-api-image build-webhook-server-image build-graph-updater build-octokit-scanner build-cloned-repo-scanner build-web-endpoint-scanner
 
 # GraphQL API
 build-api-image:
 	$(CONTAINER_RUNTIME) build ./api/ -t localhost/$(APP_NAME)-api:$(APP_VERSION)
 
 # Scanners
-# TODO
+build-octokit-scanner:
+	$(CONTAINER_RUNTIME) build ./scanners/github-octokit-checks/ -t localhost/$(APP_NAME)-octokit-scanner:$(APP_VERSION)
+
+build-cloned-repo-scanner:
+	$(CONTAINER_RUNTIME) build ./scanners/github-cloned-repo-checks/ -t localhost/$(APP_NAME)-cloned-repo-scanner:$(APP_VERSION)
+
+build-web-endpoint-scanner:
+	$(CONTAINER_RUNTIME) build ./scanners/web-endpoint-checks/ -t localhost/$(APP_NAME)-web-endpoint-scanner:$(APP_VERSION)
 
 # Webhook Server
 build-webhook-server-image:
@@ -48,14 +55,18 @@ kind-push-all: kind-push-webhook-server kind-push-api
 #  | (_| |  __/ |_) | | (_) | |_| |
 #   \__,_|\___| .__/|_|\___/ \__, |
 #             |_|            |___/ 
-k8s:
+k8s-deploy:
     # Deploy CRDs separately from rest of manifests to avoid race condition
 	echo DEPLOY ARANGODB OPERATOR CRDS
 	kubectl apply -k ./k8s/arangodb-crds/
 	echo DEPLOY MAIN APPLICATION AFTER CRDS ARE INSTALLED
 	sleep 2
 	kubectl apply -k ./k8s/
 
+k8s-destroy:
+	kubectl delete -k ./k8s/
+	kubectl delete -k ./k8s/arangodb-crds/
+
 port-forward:
 	kubectl port-forward svc/example-simple-single-ea 8529:8529 &
 	kubectl port-forward svc/nats 4222:4222 &

api/requirements.txt

@@ -50,4 +50,4 @@ urllib3==1.26.16
 uvicorn==0.23.2
 wheel==0.41.2
 zipp==3.17.0
-zstandard==0.19.0
+zstandard==0.19.0

api/src/constants.py

@@ -2,8 +2,9 @@
 
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+
 class Settings(BaseSettings):
-    model_config = SettingsConfigDict(env_file='.env')
+    model_config = SettingsConfigDict(env_file=".env")
 
     GRAPHQL_HOST: str = Field("127.0.0.1")
     GRAPHQL_PORT: int = Field(4000)
@@ -15,4 +16,4 @@ class Settings(BaseSettings):
     PASSWORD: str = Field("changeme")
     GRAPH_NAME: str = Field("endpoints")
     VERTEX_COLLECTION: str = Field("endpointNodes")
-    EDGE_COLLECTION: str = Field("endpointEdges")
+    EDGE_COLLECTION: str = Field("endpointEdges")

api/src/graphql_types/input_types.py

@@ -0,0 +1,107 @@
+from typing import List, Optional
+
+import strawberry
+
+from graphql_types.typedef import JSON
+
+@strawberry.input
+class CheckPassesInput:
+    check_passes: Optional[bool] = None
+    metadata: Optional[JSON] = None
+
+# TODO: we should find a more elegant want to unify this. The issue is that
+# the AXE accessibility reports have true/false/incomplete/null, where
+# incomplete isn't a boolean, but indicates that the accessibility check can't
+# be automated because human judgement is required.
+@strawberry.input
+class AccessibilityCheckPassesInput:
+    check_passes: Optional[str] = None  # true, false, incomplete, or null
+    metadata: Optional[JSON] = None
+
+@strawberry.input
+class GithubEndpointInput:
+    url: str
+    kind: str
+    owner: str
+    repo: str
+    license: Optional[str] = None
+    visibility: Optional[str] = None
+    programming_language: Optional[List[str]] = None
+    automated_security_fixes: Optional[CheckPassesInput] = None
+    vulnerability_alerts: Optional[CheckPassesInput] = None
+    branch_protection: Optional[CheckPassesInput] = None
+    has_security_md: Optional[CheckPassesInput] = None
+    has_dependabot_yaml: Optional[CheckPassesInput] = None
+
+
+@strawberry.input
+class AccessibilityInput:
+    url: str
+    area_alt: Optional[AccessibilityCheckPassesInput] = None
+    aria_braille_equivalent: Optional[AccessibilityCheckPassesInput] = None
+    aria_command_name: Optional[AccessibilityCheckPassesInput] = None
+    aria_hidden_focus: Optional[AccessibilityCheckPassesInput] = None
+    aria_input_field_name: Optional[AccessibilityCheckPassesInput] = None
+    aria_meter_name: Optional[AccessibilityCheckPassesInput] = None
+    aria_progressbar_name: Optional[AccessibilityCheckPassesInput] = None
+    aria_required_children: Optional[AccessibilityCheckPassesInput] = None
+    aria_required_parent: Optional[AccessibilityCheckPassesInput] = None
+    aria_roledescription: Optional[AccessibilityCheckPassesInput] = None
+    aria_toggle_field_name: Optional[AccessibilityCheckPassesInput] = None
+    aria_tooltip_name: Optional[AccessibilityCheckPassesInput] = None
+    audio_caption: Optional[AccessibilityCheckPassesInput] = None
+    blink: Optional[AccessibilityCheckPassesInput] = None
+    definition_list: Optional[AccessibilityCheckPassesInput] = None
+    dlitem: Optional[AccessibilityCheckPassesInput] = None
+    duplicate_id_aria: Optional[AccessibilityCheckPassesInput] = None
+    frame_focusable_content: Optional[AccessibilityCheckPassesInput] = None
+    frame_title_unique: Optional[AccessibilityCheckPassesInput] = None
+    frame_title: Optional[AccessibilityCheckPassesInput] = None
+    html_xml_lang_mismatch: Optional[AccessibilityCheckPassesInput] = None
+    imageAlt: Optional[AccessibilityCheckPassesInput] = None
+    input_button_name: Optional[AccessibilityCheckPassesInput] = None
+    input_image_alt: Optional[AccessibilityCheckPassesInput] = None
+    link_in_text_block: Optional[AccessibilityCheckPassesInput] = None
+    list: Optional[AccessibilityCheckPassesInput] = None
+    listitem: Optional[AccessibilityCheckPassesInput] = None
+    marquee: Optional[AccessibilityCheckPassesInput] = None
+    meta_refresh: Optional[AccessibilityCheckPassesInput] = None
+    object_alt: Optional[AccessibilityCheckPassesInput] = None
+    role_img_alt: Optional[AccessibilityCheckPassesInput] = None
+    scrollable_region_focusable: Optional[AccessibilityCheckPassesInput] = None
+    select_name: Optional[AccessibilityCheckPassesInput] = None
+    server_side_image_map: Optional[AccessibilityCheckPassesInput] = None
+    svg_img_alt: Optional[AccessibilityCheckPassesInput] = None
+    td_headers_attr: Optional[AccessibilityCheckPassesInput] = None
+    th_has_data_cells: Optional[AccessibilityCheckPassesInput] = None
+    valid_lang: Optional[AccessibilityCheckPassesInput] = None
+    video_caption: Optional[AccessibilityCheckPassesInput] = None
+    no_autoplay_audio: Optional[AccessibilityCheckPassesInput] = None
+    aria_allowed_attr: Optional[AccessibilityCheckPassesInput] = None
+    aria_conditional_attr: Optional[AccessibilityCheckPassesInput] = None
+    aria_deprecated_role: Optional[AccessibilityCheckPassesInput] = None
+    aria_hidden_body: Optional[AccessibilityCheckPassesInput] = None
+    aria_prohibited_attr: Optional[AccessibilityCheckPassesInput] = None
+    aria_required_attr: Optional[AccessibilityCheckPassesInput] = None
+    aria_roles: Optional[AccessibilityCheckPassesInput] = None
+    aria_valid_attr_value: Optional[AccessibilityCheckPassesInput] = None
+    aria_valid_attr: Optional[AccessibilityCheckPassesInput] = None
+    button_name: Optional[AccessibilityCheckPassesInput] = None
+    color_contrast: Optional[AccessibilityCheckPassesInput] = None
+    document_title: Optional[AccessibilityCheckPassesInput] = None
+    form_field_multiple_labels: Optional[AccessibilityCheckPassesInput] = None
+    html_has_lang: Optional[AccessibilityCheckPassesInput] = None
+    html_lang_valid: Optional[AccessibilityCheckPassesInput] = None
+    label: Optional[AccessibilityCheckPassesInput] = None
+    linkName: Optional[AccessibilityCheckPassesInput] = None
+    meta_viewport: Optional[AccessibilityCheckPassesInput] = None
+    nested_interactive: Optional[AccessibilityCheckPassesInput] = None
+    bypass: Optional[AccessibilityCheckPassesInput] = None
+
+
+@strawberry.input
+class WebEndpointInput:
+    url: str
+    kind: str
+    accessibility: Optional[List[AccessibilityInput]] = None
+