Docs to onboard contributor

.isort.cfg

@@ -1,6 +1,6 @@
 [settings]
 known_first_party = code_coverage_backend,code_coverage_bot,code_coverage_events,code_coverage_tools,conftest,firefox_code_coverage
-known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,libmozevent,logbook,magic,pytest,pytz,raven,redis,requests,responses,setuptools,structlog,taskcluster,tenacity,werkzeug,zstandard
+known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,libmozevent,logbook,magic,pytest,pytz,raven,redis,requests,responses,setuptools,structlog,taskcluster,tenacity,werkzeug,yaml,zstandard
 force_single_line = True
 default_section=FIRSTPARTY
 line_length=159

README.md

@@ -6,3 +6,7 @@ This project has 4 parts:
 * `backend` is a Python API built with Flask, that serves the aggregated code coverage data, in an efficient way,
 * `frontend` is a vanilla Javascript SPA displaying code coverage data in your browser,
 * `addon` is a Web Extension for Firefox, extending several Mozilla websites with code coverage data.
+
+## Help
+
+You can reach us on our Matrix instance: [#codecoverage:mozilla.org](https://chat.mozilla.org/#/room/#codecoverage:mozilla.org)

backend/README.md

@@ -37,14 +37,27 @@ docker run -v /tmp/ccov-redis:/data -p 6379:6379 redis
 
 The development webserver will run on **http://localhost:8000**
 
-Using default secret `project-relman/code-coverage/dev`:
+You need to setup a local YAML configuration file, with the following content:
+
+```yaml
+---
+common:
+  APP_CHANNEL: dev
+  GOOGLE_CLOUD_STORAGE: null
+```
+
+Using your local configuration:
 
 ```shell
-./run.sh
+LOCAL_CONFIGURATION=/path/to/code-coverage-conf.yml ./run.sh
 ```
 
-You can specify any other secret as:
+You can specify a firefox-ci Taskcluster secret (e.g. project-relman/code-coverage/dev) instead using:
 
 ```shell
 TASKCLUSTER_SECRET=path/to/secret ./run.sh
 ```
+
+## Help
+
+You can reach us on our Matrix instance: [#codecoverage:mozilla.org](https://chat.mozilla.org/#/room/#codecoverage:mozilla.org)

backend/code_coverage_backend/backend/__init__.py

@@ -6,6 +6,7 @@
 import os.path
 
 import structlog
+import yaml
 
 import code_coverage_backend.datadog
 import code_coverage_backend.gcp
@@ -17,12 +18,20 @@
 
 def create_app():
     # Load secrets from Taskcluster
+    local_secrets_path = os.environ.get("LOCAL_CONFIGURATION")
+    if local_secrets_path is not None:
+        assert os.path.exists(
+            local_secrets_path
+        ), f"Invalid local secrets path {local_secrets_path}"
     taskcluster.auth()
     taskcluster.load_secrets(
         os.environ.get("TASKCLUSTER_SECRET"),
         prefixes=["common", "backend", "code-coverage-backend"],
         required=["GOOGLE_CLOUD_STORAGE", "APP_CHANNEL"],
         existing={"REDIS_URL": os.environ.get("REDIS_URL", "redis://localhost:6379")},
+        local_secrets=yaml.safe_load(open(local_secrets_path))
+        if local_secrets_path
+        else None,
     )
 
     # Configure logger

backend/code_coverage_backend/datadog.py

@@ -25,7 +25,7 @@ def get_stats():
 
     app_channel = taskcluster.secrets["APP_CHANNEL"]
 
-    if taskcluster.secrets["DATADOG_API_KEY"]:
+    if taskcluster.secrets.get("DATADOG_API_KEY"):
         datadog.initialize(
             api_key=taskcluster.secrets["DATADOG_API_KEY"],
             host_name=f"coverage.{app_channel}.moz.tools",

bot/README.md

@@ -0,0 +1,66 @@
+# Code Coverage Bot
+
+This project runs as [Taskcluster hooks](https://firefox-ci-tc.services.mozilla.com/hooks) on the firefox-ci instance, to extract and store the code coverage information from mozilla-central and try builds.
+
+It's built using Python 3.8 and few dependencies.
+
+## Developer setup
+
+Requirements:
+
+- Python 3.8
+- Mercurial 5.3
+- (optional) [virtualenv](https://virtualenv.pypa.io/en/stable/)
+- (optional) [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io/en/latest/)
+
+Setup on your computer:
+
+```console
+# If you have virtualenvwrapper:
+mkvirtualenv -p /usr/bin/python3.8 code-coverage-bot
+
+# Mandatory steps
+pip install -r requirements.txt -r requirements-dev.txt
+pip install -e .
+pre-commit install
+```
+
+Check linting (it should be automatically done before any commit):
+
+```console
+pre-commint run -a
+```
+
+Check unit tests:
+
+```console
+pytest -v
+```
+
+Write your local configuration as YAML:
+
+```yaml
+---
+common:
+  APP_CHANNEL: dev
+bot:
+  BACKEND_HOST: 'http://localhost:8000'
+  EMAIL_ADDRESSES: []
+  PHABRICATOR_TOKEN: api-xxx
+  PHABRICATOR_ENABLED: false
+  PHABRICATOR_URL: 'https://phabricator-dev.allizom.org/api/'
+  GOOGLE_CLOUD_STORAGE: null
+```
+
+Run the bot (in cron mode):
+
+```console
+mkdir -p build/{cache,work} # or elsewhere on your system
+code-coverage-cron --cache-root=build/cache --working-dir=build/work --local-configuration=path/to/code-coverage.yml
+```
+
+The repo mode (with `code-coverage-repo`) is harder to use, as it requires a Google Cloud Storage and a Phabricator account.
+
+## Help
+
+You can reach us on our Matrix instance: [#codecoverage:mozilla.org](https://chat.mozilla.org/#/room/#codecoverage:mozilla.org)

bot/code_coverage_bot/cli.py

@@ -6,6 +6,8 @@
 import argparse
 import os
 
+import yaml
+
 from code_coverage_bot.secrets import secrets
 from code_coverage_bot.taskcluster import taskcluster_config
 from code_coverage_tools.log import init_logger
@@ -38,6 +40,11 @@ def setup_cli(ask_repository=True, ask_revision=True):
         help="Taskcluster Secret path",
         default=os.environ.get("TASKCLUSTER_SECRET"),
     )
+    parser.add_argument(
+        "--local-configuration",
+        help="Path to a local YAML configuration file",
+        type=open,
+    )
     parser.add_argument("--taskcluster-client-id", help="Taskcluster Client ID")
     parser.add_argument("--taskcluster-access-token", help="Taskcluster Access token")
     args = parser.parse_args()
@@ -46,7 +53,12 @@ def setup_cli(ask_repository=True, ask_revision=True):
     taskcluster_config.auth(args.taskcluster_client_id, args.taskcluster_access_token)
 
     # Then load secrets
-    secrets.load(args.taskcluster_secret)
+    secrets.load(
+        args.taskcluster_secret,
+        local_secrets=yaml.safe_load(args.local_configuration)
+        if args.local_configuration
+        else None,
+    )
 
     init_logger(
         "bot",

bot/code_coverage_bot/secrets.py

@@ -15,10 +15,10 @@ class Secrets(dict):
     PHABRICATOR_TOKEN = "PHABRICATOR_TOKEN"
     GOOGLE_CLOUD_STORAGE = "GOOGLE_CLOUD_STORAGE"
 
-    def load(self, taskcluster_secret):
+    def load(self, taskcluster_secret=None, local_secrets=None):
         taskcluster_config.load_secrets(
             taskcluster_secret,
-            prefixes=["common", "code-coverage-bot"],
+            prefixes=["common", "bot"],
             required=[
                 Secrets.APP_CHANNEL,
                 Secrets.BACKEND_HOST,
@@ -27,6 +27,7 @@ def load(self, taskcluster_secret):
                 Secrets.PHABRICATOR_URL,
                 Secrets.PHABRICATOR_TOKEN,
             ],
+            local_secrets=local_secrets,
         )
         self.update(taskcluster_config.secrets)
 

frontend/README.md

@@ -1,7 +1,7 @@
 # Mozilla Code Coverage frontend
 This is a simple JavaScript application displaying information from the Mozilla code coverage backend (hosted as https://coverage.moz.tools)
 
-You can use it right now from  [mozilla.github.io/code-coverage-reports](https://mozilla.github.io/code-coverage-reports/)
+You can use it right now from  [coverage.moz.tools](https://coverage.moz.tools/)
 
 ## File viewer
 
@@ -11,4 +11,25 @@ You can browse the aggregated code coverage data for mozilla-central, per direct
 
 ## Zero coverage reports
 
-We've also [developed a page](https://mozilla.github.io/code-coverage-reports/zero_coverage_report.html) to easily browse the directories & files with no coverage: these files are interesting because they can sometimes be removed altogether from mozilla-central...
+We've also [developed a page](https://coverage.moz.tools/#view=zero) to easily browse the directories & files with no coverage: these files are interesting because they can sometimes be removed altogether from mozilla-central...
+
+## Developer setup
+
+It's a pretty simple Single Page application setup using webpack:
+
+```console
+npm install
+npm run start
+```
+
+The frontend should now be available on http://localhost:9000/ and will use a backend running on http://localhost:8000/
+
+You can specify another remote backend like so:
+
+```
+BACKEND_URL=https://api.coverage.moz.tools npm run start
+```
+
+## Help
+
+You can reach us on our Matrix instance: [#codecoverage:mozilla.org](https://chat.mozilla.org/#/room/#codecoverage:mozilla.org)