Add integration test and add documentation site (#27)

* docs: update conda command

* test: add integration test

* test: add requirements-dev.txt

* ci: remove default_language_version from .pre-commit-config.yaml

* ci: lint pytest-multiversion.yml

* ci: update pyopenms version (to install in python 3.10+)

* ci: allow pyopenms to be installed in python 3.8+

* ci: remove deepspeed from requirements.txt

* ci: use environment markers for pyopenms and scikit-learn

See: https://peps.python.org/pep-0508/#environment-markers

* ci: remove torchaudio and torchvision

* docs: Publish docs via GitHub Pages

* ci: install ninja-build

* ci: Setup Ninja for Python 3.11

* ci: move install of ninja-build before installing requirements

* ci: use newer version of Levenstein on Python 3.11

* ci: update pyarrow version due to security vurnability

See: https://github.com/instadeepai/InstaNovo/security/dependabot/2

* ci: upgeade jiwer version and remove specific levensthein version for python 3.11

* ci: installing ninja-build no longer needed

* ci: update actions/setup-python to v5

* docs: upgrade pip before building docs

* ci: upgrade pytorch to be able to run tests on python 3.11

* docs: switch to latest mkdocs-material docker image

* ci: upgrade actions/checkout to v4

* docs: upgrade pip before building docs (via forked GitHub Action)

* ci: pin version of libCST

* docs: split out requirements for docs

* docs: add mkdocs.yml

* docs: change gitlab to github url

* docs: add assets and index page

* ci: replace gitlab with github

* ci: cache pip dependencies

* ci: add cache-dependency-path

* feat: add environment.yml file

.coveragerc

@@ -0,0 +1,23 @@
+[html]
+directory = coverage
+
+[run]
+source = instanovo
+omit =
+    */__init__.py
+    *_test.py
+
+[report]
+omit =
+    __init__.py
+    *_test.py
+
+
+exclude_lines =
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:

.github/workflows/docs.yml

@@ -0,0 +1,21 @@
+name: Publish docs via GitHub Pages
+on:
+  push:
+    branches:
+      - main
+      - tests
+
+jobs:
+  build:
+    name: Deploy docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+      - name: Deploy docs
+        uses: BioGeek/mkdocs-deploy-gh-pages@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CONFIG_FILE: mkdocs.yml
+          EXTRA_PACKAGES: build-base
+          REQUIREMENTS: requirements-docs.txt

.github/workflows/pytest-multiversion.yml

@@ -1,38 +1,38 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
-name: Python package
+name: Test on multiple Python versions
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ["main"]
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install -r requirements.txt
-        python -m pip install -r requirements-dev.txt
-    - name: Lint with pre-commit
-      run: |
-        pre-commit run --all-files -c .pre-commit-config.yaml
-    - name: Test with pytest
-      run: |
-        pytest -v --alluredir=allure_results  --cov-report=html --cov  --cov-config=.coveragerc --random-order
-
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip" # caching pip dependencies
+          cache-dependency-path: "**/requirements*.txt"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+          python -m pip install -r requirements-dev.txt
+      - name: Lint with pre-commit
+        run: |
+          pre-commit run --all-files -c .pre-commit-config.yaml
+      - name: Test with pytest
+        run: |
+          pytest -v --alluredir=allure_results  --cov-report=html --cov  --cov-config=.coveragerc --random-order

.github/workflows/python-publish.yml

@@ -21,11 +21,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.x"
+          cache: "pip" # caching pip dependencies
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.gitignore

@@ -165,7 +165,6 @@ docs/reference
 # Other folders
 checkpoints/
 data/
-docs/
 docs_public/
 logs/
 mlruns/

.pre-commit-config.yaml

@@ -1,6 +1,3 @@
-default_language_version:
-  python: python3.8
-
 default_stages: [commit]
 
 repos:
@@ -58,6 +55,7 @@ repos:
     rev: v1.6.0
     hooks:
       - id: mypy
+        additional_dependencies: ["types-requests"]
 
   - repo: https://github.com/compilerla/conventional-pre-commit
     rev: v2.4.0

README.md

@@ -38,7 +38,7 @@ if you have
 installed:
 
 ```bash
-conda create -n instanovo python=3.8
+conda env create -f environment.yml
 conda activate instanovo
 ```
 

docs/LICENSE.md

@@ -0,0 +1 @@
+--8<-- "LICENSE.md"

docs/gen_ref_nav.py

@@ -0,0 +1,47 @@
+"""Generate the code reference pages and navigation."""
+# This script is used by the mkdocs-gen-files plugin (https://oprypin.github.io/mkdocs-gen-files/)
+# for MkDocs (https://www.mkdocs.org/). It creates for each module in the code a stub page
+# and it creates a "docs/reference/SUMMARY.md" page which contains a Table of Contents with links to
+# all the stub pages. When MkDocs runs, it will populate the stub pages with the documentation
+# pulled from the docstrings
+from __future__ import annotations
+
+import pathlib
+
+import mkdocs_gen_files
+
+# Folders for which we don't want to create code documentation but which can contain *.py files
+IGNORE = ("public", "docs", "tests", ".")
+
+nav = mkdocs_gen_files.Nav()
+
+src_folder = "."
+module = "instanovo"
+for path in sorted(pathlib.Path(src_folder).rglob("*.py")):
+    module_path = path.relative_to(src_folder).with_suffix("")
+    if (
+        not any(str(module_path.parent).startswith(ignore) for ignore in IGNORE)
+        and module_path.parts[-1].split("_")[-1] != "test"
+    ):
+        doc_path = path.relative_to(src_folder, module).with_suffix(".md")
+        full_doc_path = pathlib.Path("reference", doc_path)
+
+        parts = tuple(module_path.parts)
+
+        if parts[-1] == "__init__":
+            parts = parts[:-1]
+            doc_path = doc_path.with_name("index.md")
+            full_doc_path = full_doc_path.with_name("index.md")
+        elif parts[-1] == "__main__":
+            continue
+
+        nav[parts] = doc_path.as_posix()
+
+        with mkdocs_gen_files.open(full_doc_path, "w") as fd:
+            ident = ".".join(parts)
+            fd.write(f"::: {ident}")
+
+        mkdocs_gen_files.set_edit_path(full_doc_path, ".." / path)
+
+with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
+    nav_file.writelines(nav.build_literate_nav())

docs/index.md

@@ -0,0 +1 @@
+--8<-- "README.md"

environment.yml

@@ -0,0 +1,12 @@
+# This is an environment for running git commands and other tools locally.
+name: instanovo
+channels:
+  - default
+dependencies:
+  - python=3.8
+  - pip
+  - setuptools=68.2.2
+  - pip:
+      - -r requirements-dev.txt
+      - -r requirements.txt
+      - pre-commit==2.20.0

instanovo/transformer/predict.py

@@ -14,6 +14,7 @@
 from torch.utils.data import DataLoader
 from tqdm import tqdm
 
+from instanovo.constants import MASS_SCALE
 from instanovo.inference.knapsack import Knapsack
 from instanovo.inference.knapsack_beam_search import KnapsackBeamSearchDecoder
 from instanovo.transformer.dataset import collate_batch
@@ -121,8 +122,8 @@ def get_preds(
                 max_length=config["max_length"],
             )
 
-            preds += ["".join(x.sequence) if type(x) != list else "" for x in p]
-            probs += [x.log_probability if type(x) != list else -1 for x in p]
+            preds += ["".join(x.sequence) if not isinstance(x, list) else "" for x in p]
+            probs += [x.log_probability if not isinstance(x, list) else -1 for x in p]
             targs += list(peptides)
 
     delta = time.time() - start
@@ -189,7 +190,6 @@ def main() -> None:
 
 
 def _setup_knapsack(model: InstaNovo) -> Knapsack:
-    MASS_SCALE = 10000
     residue_masses = model.peptide_mass_calculator.masses
     residue_masses["$"] = 0
     residue_indices = model.decoder._aa2idx

instanovo/transformer/train.py

@@ -149,7 +149,7 @@ def validation_step(
             )
 
         # targets = self.model.batch_idx_to_aa(peptides)
-        y = ["".join(x.sequence) if type(x) != list else "" for x in p]
+        y = ["".join(x.sequence) if not isinstance(x, list) else "" for x in p]
         targets = peptides
 
         aa_prec, aa_recall, pep_recall, _ = self.metrics.compute_precision_recall(targets, y)

mkdocs.yml

@@ -0,0 +1,87 @@
+site_name: InstaNovo
+site_description: Documentation for InstaNovo
+site_author: InstaDeep Developers
+site_url: https://instadeepai.github.io/InstaNovo/
+site_dir: public
+repo_name: instadeep/dtu-denovo-sequencing
+repo_url: https://github.com/instadeepai/InstaNovo
+strict: false
+
+theme:
+  name: material
+  language: en
+  palette:
+    - scheme: default
+      primary: white
+      accent: purple
+      toggle:
+        icon: material/weather-sunny
+        name: Switch to dark mode
+    - scheme: slate
+      primary: black
+      accent: lime
+      toggle:
+        icon: material/weather-night
+        name: Switch to light mode
+  logo: assets/instadeep-logo.png
+  favicon: assets/instadeep-logo.png
+  icon:
+    repo: fontawesome/brands/github
+  font:
+    text: Avenir Next
+  features:
+    - navigation.tracking # the URL is automatically updated with the active anchor
+    - navigation.sections # top-level sections are rendered as groups in the sidebar
+    - navigation.tabs # horizontal tabs at the top of the page
+    - navigation.tabs.sticky # navigation tabs will lock below the header and always remain visible when scrolling
+    - navigation.indexes # documents can be directly attached to sections
+    - search.highlight # highlight search result
+    - search.share # share button
+    - search.suggest # smart suggestion
+    - toc.integrate
+    - toc.follow
+    - content.code.annotate
+    - navigation.tabs
+    - navigation.top
+
+markdown_extensions:
+  - pymdownx.highlight:
+      anchor_linenums: true
+  - pymdownx.inlinehilite
+  - pymdownx.snippets
+  - pymdownx.superfences
+
+plugins:
+  - search
+  - autorefs
+  # - git-revision-date
+  - include-markdown
+  - gen-files:
+      scripts:
+        - docs/gen_ref_nav.py
+  - mkdocstrings:
+      default_handler: python
+      handlers:
+        python:
+          paths: [instanovo]
+          options:
+            docstring_style: google
+            merge_init_into_class: yes
+            show_submodules: no
+          selection:
+            inherited_members: false
+          rendering:
+            show_source: false
+            members_order: source
+            show_if_no_docstring: true
+            show_signature: true
+            show_signature_annotations: true
+            show_root_full_path: false
+            show_root_heading: true
+            merge_init_into_class: true
+            docstring_section_style: spacy
+
+nav:
+  - Home: index.md
+  - License: LICENSE.md
+  - Code reference: reference/SUMMARY.md