Add precommit and run on all files (#1541)

* Add precommit

* Use pre-commit

* Run precommit

* Add EOL

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-ignore = 
+ignore =
         # E*** / W*** pycodestyle codes
         # whitespace before ','
         E203,

.github/ISSUE_TEMPLATE/jira_service_desk.md

@@ -1,5 +1,5 @@
 ---
-name: Please file issues in Jira Service Desk 
+name: Please file issues in Jira Service Desk
 about: https://sagebionetworks.jira.com/servicedesk/customer/portal/5/group/8/create/87
 title: ''
 labels: ''
@@ -10,4 +10,4 @@ assignees: ''
 **Describe the bug**
 Please file issues or requests for help with schematic in [Jira Service Desk](https://sagebionetworks.jira.com/servicedesk/customer/portal/5/group/8/create/87).
 
-These issues are not monitored. 
+These issues are not monitored.

.github/workflows/api_test.yml

@@ -43,7 +43,7 @@ jobs:
             | python3 - --version ${{ env.POETRY_VERSION }};
           poetry config virtualenvs.create true;
           poetry config virtualenvs.in-project true;
-          
+
       #----------------------------------------------
       # install dependencies and root project
       #----------------------------------------------

.github/workflows/docker_build.yml

@@ -24,7 +24,7 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-          
+
       - name: Set env variable for version tag
         run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
 
@@ -54,4 +54,3 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             TAG=${{ env.RELEASE_VERSION }}
-            

.github/workflows/pdoc.yml

@@ -12,12 +12,12 @@ permissions:
   contents: read
 
 concurrency:
-  # cancel the current running workflow from the same branch, PR when a new workflow is triggered 
+  # cancel the current running workflow from the same branch, PR when a new workflow is triggered
   # when the trigger is not a PR but a push, it will use the commit sha to generate the concurrency group
   # {{ github.workflow }}: the workflow name is used to generate the concurrency group. This allows you to have more than one workflows
-  # {{ github.ref_type }}: the type of Git ref object created in the repository. Can be either branch or tag 
-  # {{ github.event.pull_request.number}}: get PR number 
-  # {{ github.sha }}: full commit sha 
+  # {{ github.ref_type }}: the type of Git ref object created in the repository. Can be either branch or tag
+  # {{ github.event.pull_request.number}}: get PR number
+  # {{ github.sha }}: full commit sha
   # credit: https://github.com/Sage-Bionetworks-Workflows/sagetasks/blob/main/.github/workflows/ci.yml
   group: >-
     ${{ github.workflow }}-${{ github.ref_type }}-
@@ -32,7 +32,7 @@ jobs:
 
     steps:
       #----------------------------------------------
-      #       check-out repo and set-up python     
+      #       check-out repo and set-up python
       #----------------------------------------------
       - name: Check out repository
         uses: actions/checkout@v4
@@ -41,16 +41,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-      
+
       #----------------------------------------------
-      #          install & configure poetry         
+      #          install & configure poetry
       #----------------------------------------------
       - name: Install Poetry
         run: |
           curl -sSL https://install.python-poetry.org \
             | python3 - --version ${{ env.POETRY_VERSION }};
-          poetry config virtualenvs.create true; 
-          poetry config virtualenvs.in-project true; 
+          poetry config virtualenvs.create true;
+          poetry config virtualenvs.in-project true;
 
       #----------------------------------------------
       # install dependencies and root project

.github/workflows/publish.yml

@@ -17,7 +17,7 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - id: build-package
-        run: |        
+        run: |
           python3 -m pip install --upgrade pip
           python3 -m pip install setuptools
           python3 -m pip install wheel
@@ -35,7 +35,7 @@ jobs:
 
           echo "sdist-release-url=${RELEASE_URL_PREFIX}${SDIST_PACKAGE_NAME}" >> $GITHUB_OUTPUT
           echo "bdist-release-url=${RELEASE_URL_PREFIX}${BDIST_PACKAGE_NAME}" >> $GITHUB_OUTPUT
-        
+
       - name: upload-sdist-artifact
         uses: actions/upload-artifact@v4
         with:
@@ -90,4 +90,3 @@ jobs:
           # Post to the `fair-data-tools` slack channel
           channel-id: 'C01ANC02U59'
           slack-message: "A new version of Schematic has been released. Check out the new version: ${{ github.ref_name }}"
-

.github/workflows/scan_repo.yml

@@ -6,7 +6,7 @@ on:
     branches:
       - develop
   pull_request:
-  workflow_dispatch: 
+  workflow_dispatch:
 
 jobs:
   trivy:

.github/workflows/test.yml

@@ -87,11 +87,7 @@ jobs:
       #----------------------------------------------
       #             check formatting
       #----------------------------------------------
-      - name: Code formatting with black
-        run: |
-          # ran only on certain files for now
-          # add here when checked
-          poetry run black schematic tests schematic_api --check
+      - uses: pre-commit/[email protected]
 
       #----------------------------------------------
       #             type checking/enforcement
@@ -121,7 +117,7 @@ jobs:
           SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
           SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
         run: >
-          poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov 
+          poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov
           --cov-report=xml:coverage.xml --cov=schematic/ --reruns 4 -n 8 tests/unit;
 
       #----------------------------------------------
@@ -151,7 +147,7 @@ jobs:
           LOGGING_EXPORT_FORMAT: ${{ vars.LOGGING_EXPORT_FORMAT }}
           TRACING_SERVICE_NAME: ${{ vars.TRACING_SERVICE_NAME }}
           LOGGING_SERVICE_NAME: ${{ vars.LOGGING_SERVICE_NAME }}
-          SERVICE_INSTANCE_ID: ${{ github.head_ref || github.ref_name }} 
+          SERVICE_INSTANCE_ID: ${{ github.head_ref || github.ref_name }}
         run: >
           poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/
           -m "not (rule_benchmark or single_process_execution)" --reruns 4 -n 8 --ignore=tests/unit

.gitignore

@@ -180,4 +180,4 @@ manifests/*
 https:*
 
 # schematic config file
-config.yml
+config.yml

.pre-commit-config.yaml

@@ -1,21 +1,68 @@
-repos:
-  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
-  # This is recommended by psf/black: https://github.com/psf/black/blob/main/.pre-commit-hooks.yaml
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 23.7.0
-    hooks:
-      - id: black
-        # It is recommended to specify the latest version of Python
-        # supported by your project here, or alternatively use
-        # pre-commit's default_language_version, see
-        # https://pre-commit.com/#top_level-default_language_version
-        language_version: python3.10
-        files: ^(tests|schematic|schematic_api)/
-
-  - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        name: isort (python)
-        files: ^(tests|schematic|schematic_api)/
-        args: ["--profile", "black", "--filter-files"]
+exclude: "^docs/conf.py|^docs/build/"
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: check-added-large-files
+      - id: check-ast
+      # - id: check-json
+      - id: check-merge-conflict
+      - id: check-xml
+      - id: check-yaml
+        exclude: ^mkdocs\.yml$
+      - id: debug-statements
+      - id: end-of-file-fixer
+      - id: requirements-txt-fixer
+      - id: mixed-line-ending
+        args: ["--fix=auto"] # replace 'auto' with 'lf' to enforce Linux/Mac line endings or 'crlf' for Windows
+
+  # - repo: https://github.com/astral-sh/ruff-pre-commit
+  #   rev: "v0.0.270"
+  #   hooks:
+  #     - id: ruff
+
+  # - repo: https://github.com/pycqa/isort
+  #   rev: 5.13.2
+  #   hooks:
+  #     - id: isort
+  #       name: isort (python)
+
+  - repo: https://github.com/psf/black
+    rev: 23.12.1
+    hooks:
+      - id: black
+        language_version: python3
+
+  # - repo: https://github.com/PyCQA/bandit
+  #   rev: 1.7.5
+  #   hooks:
+  #     - id: bandit
+  #       args: ["-c", "pyproject.toml"]
+  #       additional_dependencies: ["bandit[toml]"]
+# - repo: https://github.com/asottile/blacken-docs
+#   rev: v1.12.0
+#   hooks:
+#   - id: blacken-docs
+#     additional_dependencies: [black]
+
+# - repo: https://github.com/pre-commit/mirrors-mypy
+#   rev: 'v1.0.1'
+#   hooks:
+#   - id: mypy
+#     additional_dependencies: [pydantic~=1.10]
+
+# Checks for missing docstrings
+# - repo: https://github.com/econchick/interrogate
+#   rev: 1.5.0
+#   hooks:
+#   - id: interrogate
+#     exclude: ^(docs/conf.py|setup.py|tests)
+#     args: [--config=pyproject.toml]
+
+# finds dead python code
+# - repo: https://github.com/jendrikseipp/vulture
+#   rev: 'v2.7'
+#   hooks:
+#     - id: vulture

.pylintrc

@@ -1,2 +1,2 @@
 [MAIN]
-load-plugins=pylint.extensions.docparams
+load-plugins=pylint.extensions.docparams

.readthedocs.yml

@@ -33,4 +33,4 @@ sphinx:
 # Optionally declare the Python requirements required to build your docs
 # python:
 #    install:
-#    - requirements: docs/requirements.txt
+#    - requirements: docs/requirements.txt

CONTRIBUTION.md

@@ -100,7 +100,9 @@ When a release (no `-rc` suffix and the `prerelease` flag set to `false`) is cre
 pip install schematicpy
 ```
 
-## Testing 
+> You'll need to [register](https://pypi.org/account/register/) for a PyPI account before uploading packages to the package index. Similarly for [Test PyPI](https://test.pypi.org/account/register/) as well.
+
+## Testing
 
 * All new code must include tests.
 
@@ -119,4 +121,3 @@ pytest -vs tests/
 4. Open a PR as per the usual process (see above).
 5. Once the PR is merged, leave the original copies on Synapse to maintain support for feature branches that were forked from `develop` before your update.
    - If the old copies are problematic and need to be removed immediately (_e.g._ contain sensitive data), proceed with the deletion and alert the other contributors that they need to merge the latest `develop` branch into their feature branches for their tests to work.
-

Dockerfile

@@ -14,7 +14,7 @@ WORKDIR /usr/src/app
 RUN apt-get update -qqy \
   && apt-get install -qqy \
   libopenblas-dev \
-  gfortran 
+  gfortran
 
 # remove libtiff5 for security reasons
 RUN apt remove -y libtiff5