From 1c68964f46e16514300c3c5d39b9df325f172e8d Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Tue, 17 Dec 2024 16:34:05 -0800 Subject: [PATCH] Add precommit and run on all files (#1541) * Add precommit * Use pre-commit * Run precommit * Add EOL --- .flake8 | 2 +- .github/ISSUE_TEMPLATE/jira_service_desk.md | 4 +- .github/workflows/api_test.yml | 2 +- .github/workflows/docker_build.yml | 3 +- .github/workflows/pdoc.yml | 18 +-- .github/workflows/publish.yml | 5 +- .github/workflows/scan_repo.yml | 2 +- .github/workflows/test.yml | 10 +- .gitignore | 2 +- .pre-commit-config.yaml | 89 +++++++++---- .pylintrc | 2 +- .readthedocs.yml | 2 +- CONTRIBUTION.md | 5 +- Dockerfile | 2 +- README.md | 40 +++--- certificate.conf | 2 +- docker-compose.yml | 8 +- docs/md/details.md | 2 +- docs/md/manifest_validation_submission.md | 1 - docs/source/conf.py | 6 +- docs/source/index.rst | 4 +- docs/source/utils.py | 4 +- env.example | 4 +- great_expectations/.gitignore | 2 +- main.py | 6 +- run_api.py | 7 +- schematic/__main__.py | 1 + schematic/configuration/configuration.py | 12 +- schematic/configuration/dataclasses.py | 3 +- schematic/etc/README.md | 4 +- .../etc/data_models/schema_org.model.jsonld | 2 +- .../etc/validation_schemas/class.schema.json | 2 +- .../etc/validation_schemas/model.schema.json | 2 +- schematic/exceptions.py | 2 +- schematic/loader.py | 5 +- schematic/schemas/__init__.py | 2 +- schematic/schemas/commands.py | 19 ++- schematic/schemas/data_model_edges.py | 3 +- schematic/schemas/data_model_json_schema.py | 2 +- schematic/schemas/data_model_jsonld.py | 9 +- schematic/schemas/data_model_nodes.py | 13 +- schematic/schemas/data_model_relationships.py | 4 +- schematic/schemas/data_model_validator.py | 8 +- schematic/schemas/json_schema_validator.py | 4 +- schematic/store/database/README.md | 2 +- .../database/synapse_database_wrapper.py | 2 +- schematic/store/synapse.py | 14 +-- schematic/utils/schema_utils.py | 3 +- schematic/utils/viz_utils.py | 3 +- schematic/visualization/commands.py | 11 +- schematic_api/Dockerfile | 18 +-- schematic_api/api/README.md | 62 ++++----- schematic_api/api/__main__.py | 1 + schematic_api/api/openapi/api.yaml | 118 +++++++++--------- self-signed.conf | 2 +- ssl-params.conf | 2 +- tests/data/example.model.jsonld | 2 +- tests/data/example.single_rule.model.jsonld | 2 +- tests/data/example_required_vr_test.model.csv | 2 +- .../example_required_vr_test.model.jsonld | 2 +- tests/data/example_test_nones.model.csv | 2 +- tests/data/example_test_nones.model.jsonld | 2 +- ...pecimen_required_vr_test_fail.manifest.csv | 4 +- ...pecimen_required_vr_test_pass.manifest.csv | 4 +- .../Example.Patient.manifest.json | 2 +- ...id_Biospecimen_Missing_Column_Manifest.csv | 4 +- .../mock_manifests/Invalid_Test_Manifest.csv | 2 +- .../MockComponent-cross-manifest-1.csv | 2 +- .../MockComponent-cross-manifest-2.csv | 2 +- .../Mock_Component_rule_combination.csv | 2 +- ...Patient_required_vr_test_pass.manifest.csv | 4 +- ...ntry_for_cond_required_column.manifest.csv | 2 +- ...nested_manifest_table_and_file_replace.csv | 4 +- ...t_manifest_with_blacklisted_characters.csv | 4 +- ..._test_submit_manifest_with_hide_blanks.csv | 4 +- ...n_test_manifest_validation_basic_valid.csv | 2 +- .../mock_manifests/Valid_Test_Manifest.csv | 2 +- .../Valid_Test_Manifest_with_nones.csv | 2 +- .../Valid_none_value_test_manifest.csv | 2 +- ...le.biospecimen_component_rule.manifest.csv | 2 +- ...xample.patient_component_rule.manifest.csv | 2 +- .../example_biospecimen_test.csv | 2 +- .../mock_example_biospecimen_manifest.csv | 2 +- tests/data/test_configs/invalid_config2.yml | 2 +- tests/data/test_configs/invalid_config3.yml | 1 - tests/data/validator_dag_test.model.csv | 2 +- tests/data/validator_test.model.csv | 4 +- tests/integration/test_manifest_generation.py | 2 +- tests/integration/test_manifest_validation.py | 2 +- tests/test_configuration.py | 13 +- tests/unit/test_data_model_validator.py | 16 +-- tests/unit/test_validate_attribute.py | 4 +- uwsgi-nginx-entrypoint.sh | 2 +- uwsgi.ini | 2 +- 94 files changed, 365 insertions(+), 322 deletions(-) diff --git a/.flake8 b/.flake8 index de4f13a42..e828e0d11 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -ignore = +ignore = # E*** / W*** pycodestyle codes # whitespace before ',' E203, diff --git a/.github/ISSUE_TEMPLATE/jira_service_desk.md b/.github/ISSUE_TEMPLATE/jira_service_desk.md index 68110bd82..6326d9630 100644 --- a/.github/ISSUE_TEMPLATE/jira_service_desk.md +++ b/.github/ISSUE_TEMPLATE/jira_service_desk.md @@ -1,5 +1,5 @@ --- -name: Please file issues in Jira Service Desk +name: Please file issues in Jira Service Desk about: https://sagebionetworks.jira.com/servicedesk/customer/portal/5/group/8/create/87 title: '' labels: '' @@ -10,4 +10,4 @@ assignees: '' **Describe the bug** Please file issues or requests for help with schematic in [Jira Service Desk](https://sagebionetworks.jira.com/servicedesk/customer/portal/5/group/8/create/87). -These issues are not monitored. +These issues are not monitored. diff --git a/.github/workflows/api_test.yml b/.github/workflows/api_test.yml index 3e48bd09a..420e1f518 100644 --- a/.github/workflows/api_test.yml +++ b/.github/workflows/api_test.yml @@ -43,7 +43,7 @@ jobs: | python3 - --version ${{ env.POETRY_VERSION }}; poetry config virtualenvs.create true; poetry config virtualenvs.in-project true; - + #---------------------------------------------- # install dependencies and root project #---------------------------------------------- diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml index f1beb3489..de8354c19 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/docker_build.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - + - name: Set env variable for version tag run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV @@ -54,4 +54,3 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | TAG=${{ env.RELEASE_VERSION }} - \ No newline at end of file diff --git a/.github/workflows/pdoc.yml b/.github/workflows/pdoc.yml index d24ef8652..509462fe9 100644 --- a/.github/workflows/pdoc.yml +++ b/.github/workflows/pdoc.yml @@ -12,12 +12,12 @@ permissions: contents: read concurrency: - # cancel the current running workflow from the same branch, PR when a new workflow is triggered + # cancel the current running workflow from the same branch, PR when a new workflow is triggered # when the trigger is not a PR but a push, it will use the commit sha to generate the concurrency group # {{ github.workflow }}: the workflow name is used to generate the concurrency group. This allows you to have more than one workflows - # {{ github.ref_type }}: the type of Git ref object created in the repository. Can be either branch or tag - # {{ github.event.pull_request.number}}: get PR number - # {{ github.sha }}: full commit sha + # {{ github.ref_type }}: the type of Git ref object created in the repository. Can be either branch or tag + # {{ github.event.pull_request.number}}: get PR number + # {{ github.sha }}: full commit sha # credit: https://github.com/Sage-Bionetworks-Workflows/sagetasks/blob/main/.github/workflows/ci.yml group: >- ${{ github.workflow }}-${{ github.ref_type }}- @@ -32,7 +32,7 @@ jobs: steps: #---------------------------------------------- - # check-out repo and set-up python + # check-out repo and set-up python #---------------------------------------------- - name: Check out repository uses: actions/checkout@v4 @@ -41,16 +41,16 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - + #---------------------------------------------- - # install & configure poetry + # install & configure poetry #---------------------------------------------- - name: Install Poetry run: | curl -sSL https://install.python-poetry.org \ | python3 - --version ${{ env.POETRY_VERSION }}; - poetry config virtualenvs.create true; - poetry config virtualenvs.in-project true; + poetry config virtualenvs.create true; + poetry config virtualenvs.in-project true; #---------------------------------------------- # install dependencies and root project diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a89410740..0cc464924 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,7 +17,7 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - id: build-package - run: | + run: | python3 -m pip install --upgrade pip python3 -m pip install setuptools python3 -m pip install wheel @@ -35,7 +35,7 @@ jobs: echo "sdist-release-url=${RELEASE_URL_PREFIX}${SDIST_PACKAGE_NAME}" >> $GITHUB_OUTPUT echo "bdist-release-url=${RELEASE_URL_PREFIX}${BDIST_PACKAGE_NAME}" >> $GITHUB_OUTPUT - + - name: upload-sdist-artifact uses: actions/upload-artifact@v4 with: @@ -90,4 +90,3 @@ jobs: # Post to the `fair-data-tools` slack channel channel-id: 'C01ANC02U59' slack-message: "A new version of Schematic has been released. Check out the new version: ${{ github.ref_name }}" - diff --git a/.github/workflows/scan_repo.yml b/.github/workflows/scan_repo.yml index 434b4b522..329e57bb8 100644 --- a/.github/workflows/scan_repo.yml +++ b/.github/workflows/scan_repo.yml @@ -6,7 +6,7 @@ on: branches: - develop pull_request: - workflow_dispatch: + workflow_dispatch: jobs: trivy: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0090d1558..06cb446e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,11 +87,7 @@ jobs: #---------------------------------------------- # check formatting #---------------------------------------------- - - name: Code formatting with black - run: | - # ran only on certain files for now - # add here when checked - poetry run black schematic tests schematic_api --check + - uses: pre-commit/action@v3.0.1 #---------------------------------------------- # type checking/enforcement @@ -121,7 +117,7 @@ jobs: SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }} SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }} run: > - poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov + poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/ --reruns 4 -n 8 tests/unit; #---------------------------------------------- @@ -151,7 +147,7 @@ jobs: LOGGING_EXPORT_FORMAT: ${{ vars.LOGGING_EXPORT_FORMAT }} TRACING_SERVICE_NAME: ${{ vars.TRACING_SERVICE_NAME }} LOGGING_SERVICE_NAME: ${{ vars.LOGGING_SERVICE_NAME }} - SERVICE_INSTANCE_ID: ${{ github.head_ref || github.ref_name }} + SERVICE_INSTANCE_ID: ${{ github.head_ref || github.ref_name }} run: > poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/ -m "not (rule_benchmark or single_process_execution)" --reruns 4 -n 8 --ignore=tests/unit diff --git a/.gitignore b/.gitignore index 91f6d7c64..b8aebe08e 100644 --- a/.gitignore +++ b/.gitignore @@ -180,4 +180,4 @@ manifests/* https:* # schematic config file -config.yml \ No newline at end of file +config.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 16c72afc9..7d3a4d258 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,21 +1,68 @@ -repos: - # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - # This is recommended by psf/black: https://github.com/psf/black/blob/main/.pre-commit-hooks.yaml - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.7.0 - hooks: - - id: black - # It is recommended to specify the latest version of Python - # supported by your project here, or alternatively use - # pre-commit's default_language_version, see - # https://pre-commit.com/#top_level-default_language_version - language_version: python3.10 - files: ^(tests|schematic|schematic_api)/ - - - repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort - name: isort (python) - files: ^(tests|schematic|schematic_api)/ - args: ["--profile", "black", "--filter-files"] +exclude: "^docs/conf.py|^docs/build/" + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: check-added-large-files + - id: check-ast + # - id: check-json + - id: check-merge-conflict + - id: check-xml + - id: check-yaml + exclude: ^mkdocs\.yml$ + - id: debug-statements + - id: end-of-file-fixer + - id: requirements-txt-fixer + - id: mixed-line-ending + args: ["--fix=auto"] # replace 'auto' with 'lf' to enforce Linux/Mac line endings or 'crlf' for Windows + + # - repo: https://github.com/astral-sh/ruff-pre-commit + # rev: "v0.0.270" + # hooks: + # - id: ruff + + # - repo: https://github.com/pycqa/isort + # rev: 5.13.2 + # hooks: + # - id: isort + # name: isort (python) + + - repo: https://github.com/psf/black + rev: 23.12.1 + hooks: + - id: black + language_version: python3 + + # - repo: https://github.com/PyCQA/bandit + # rev: 1.7.5 + # hooks: + # - id: bandit + # args: ["-c", "pyproject.toml"] + # additional_dependencies: ["bandit[toml]"] +# - repo: https://github.com/asottile/blacken-docs +# rev: v1.12.0 +# hooks: +# - id: blacken-docs +# additional_dependencies: [black] + +# - repo: https://github.com/pre-commit/mirrors-mypy +# rev: 'v1.0.1' +# hooks: +# - id: mypy +# additional_dependencies: [pydantic~=1.10] + +# Checks for missing docstrings +# - repo: https://github.com/econchick/interrogate +# rev: 1.5.0 +# hooks: +# - id: interrogate +# exclude: ^(docs/conf.py|setup.py|tests) +# args: [--config=pyproject.toml] + +# finds dead python code +# - repo: https://github.com/jendrikseipp/vulture +# rev: 'v2.7' +# hooks: +# - id: vulture diff --git a/.pylintrc b/.pylintrc index b5a5c8eb1..75f38fc09 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,2 +1,2 @@ [MAIN] -load-plugins=pylint.extensions.docparams \ No newline at end of file +load-plugins=pylint.extensions.docparams diff --git a/.readthedocs.yml b/.readthedocs.yml index 5e86f8ca8..896c5e427 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -33,4 +33,4 @@ sphinx: # Optionally declare the Python requirements required to build your docs # python: # install: -# - requirements: docs/requirements.txt \ No newline at end of file +# - requirements: docs/requirements.txt diff --git a/CONTRIBUTION.md b/CONTRIBUTION.md index 74ceea343..c9d8b26ae 100644 --- a/CONTRIBUTION.md +++ b/CONTRIBUTION.md @@ -100,7 +100,9 @@ When a release (no `-rc` suffix and the `prerelease` flag set to `false`) is cre pip install schematicpy ``` -## Testing +> You'll need to [register](https://pypi.org/account/register/) for a PyPI account before uploading packages to the package index. Similarly for [Test PyPI](https://test.pypi.org/account/register/) as well. + +## Testing * All new code must include tests. @@ -119,4 +121,3 @@ pytest -vs tests/ 4. Open a PR as per the usual process (see above). 5. Once the PR is merged, leave the original copies on Synapse to maintain support for feature branches that were forked from `develop` before your update. - If the old copies are problematic and need to be removed immediately (_e.g._ contain sensitive data), proceed with the deletion and alert the other contributors that they need to merge the latest `develop` branch into their feature branches for their tests to work. - diff --git a/Dockerfile b/Dockerfile index 7653ee335..98dc77382 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ WORKDIR /usr/src/app RUN apt-get update -qqy \ && apt-get install -qqy \ libopenblas-dev \ - gfortran + gfortran # remove libtiff5 for security reasons RUN apt remove -y libtiff5 diff --git a/README.md b/README.md index 32318c9a4..6423c36d9 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ SCHEMATIC is an acronym for _Schema Engine for Manifest Ingress and Curation_. T * Your installed python version must be 3.9.0 ≤ version < 3.11.0 * You need to be a registered and certified user on [`synapse.org`](https://www.synapse.org/) -> [!NOTE] +> [!NOTE] > To create Google Sheets files from Schematic, please follow our credential policy for Google credentials. You can find a detailed tutorial [here](https://scribehow.com/shared/Get_Credentials_for_Google_Drive_and_Google_Sheets_APIs_to_use_with_schematicpy__yqfcJz_rQVeyTcg0KQCINA). > If you're using config.yml, make sure to specify the path to `schematic_service_account_creds.json` (see the `google_sheets > service_account_creds` section for more information). @@ -103,7 +103,7 @@ source .venv/bin/activate ``` 3. **Verify your `conda` setup**: Follow the prompts to complete your setup. Then verify your setup by running the `conda` command. - + 4. **Create your `schematic` environment**: Begin by creating a fresh `conda` environment for `schematic` like so: ``` @@ -146,25 +146,25 @@ Any function that interacts with a google sheet (such as `schematic manifest get * Depending on your institution's policies, your institutional Google account may or may not have the required permissions to complete this. A possible workaround is to use a personal or temporary Google account. > [!WARNING] -> At the time of writing, Sage Bionetworks employees do not have the appropriate permissions to create projects with their Sage Bionetworks Google accounts. You would follow instructions using a personal Google account. +> At the time of writing, Sage Bionetworks employees do not have the appropriate permissions to create projects with their Sage Bionetworks Google accounts. You would follow instructions using a personal Google account. 2. **Option 2**: Ask your DCC/development team if they have credentials previously set up with a service account. Once you have obtained credentials, be sure that the json file generated is named in the same way as the `service_acct_creds` parameter in your `config.yml` file. You will find more context on the `config.yml` in section [6. Set up configuration files](#6-set-up-configuration-files). > [!NOTE] -> Running `schematic init` is no longer supported due to security concerns. To obtain `schematic_service_account_creds.json`, please follow the instructions [here](https://scribehow.com/shared/Enable_Google_Drive_and_Google_Sheets_APIs_for_project__yqfcJz_rQVeyTcg0KQCINA). +> Running `schematic init` is no longer supported due to security concerns. To obtain `schematic_service_account_creds.json`, please follow the instructions [here](https://scribehow.com/shared/Enable_Google_Drive_and_Google_Sheets_APIs_for_project__yqfcJz_rQVeyTcg0KQCINA). schematic uses Google’s API to generate google sheet templates that users fill in to provide (meta)data. Most Google sheet functionality could be authenticated with service account. However, more complex Google sheet functionality requires token-based authentication. As browser support that requires the token-based authentication diminishes, we are hoping to deprecate -token-based authentication and keep only service account authentication in the future. +token-based authentication and keep only service account authentication in the future. > [!NOTE] > Use the ``schematic_service_account_creds.json`` file for the service > account mode of authentication (*for Google services/APIs*). Service accounts > are special Google accounts that can be used by applications to access Google APIs > programmatically via OAuth2.0, with the advantage being that they do not require -> human authorization. +> human authorization. ### 6. Set up configuration files @@ -241,7 +241,7 @@ you would like to store your local fork before running the following command: git clone https://github.com/Sage-Bionetworks/schematic.git ``` -### 2. Install `poetry` +### 2. Install `poetry` Install `poetry` (version 1.3.0 or later) using either the [official installer](https://python-poetry.org/docs/#installing-with-the-official-installer) or `pip`. If you have an older installation of Poetry, we recommend uninstalling it first. @@ -348,7 +348,7 @@ Any function that interacts with a google sheet (such as `schematic manifest get * Depending on your institution's policies, your institutional Google account may or may not have the required permissions to complete this. A possible workaround is to use a personal or temporary Google account. > [!WARNING] -> At the time of writing, Sage Bionetworks employees do not have the appropriate permissions to create projects with their Sage Bionetworks Google accounts. You would follow instructions using a personal Google account. +> At the time of writing, Sage Bionetworks employees do not have the appropriate permissions to create projects with their Sage Bionetworks Google accounts. You would follow instructions using a personal Google account. 2. **Option 2**: Ask your DCC/development team if they have credentials previously set up with a service account. @@ -358,18 +358,18 @@ Once you have obtained credentials, be sure that the json file generated is name > For testing, make sure there is no environment variable `SCHEMATIC_SERVICE_ACCOUNT_CREDS`. Check the file `.env` to ensure this is not set. Also, check that config files used for testing, such as `config_example.yml` do not contain service_acct_creds_synapse_id. > [!NOTE] -> Running `schematic init` is no longer supported due to security concerns. To obtain `schematic_service_account_creds.json`, please follow the instructions [here](https://scribehow.com/shared/Enable_Google_Drive_and_Google_Sheets_APIs_for_project__yqfcJz_rQVeyTcg0KQCINA). +> Running `schematic init` is no longer supported due to security concerns. To obtain `schematic_service_account_creds.json`, please follow the instructions [here](https://scribehow.com/shared/Enable_Google_Drive_and_Google_Sheets_APIs_for_project__yqfcJz_rQVeyTcg0KQCINA). schematic uses Google’s API to generate google sheet templates that users fill in to provide (meta)data. Most Google sheet functionality could be authenticated with service account. However, more complex Google sheet functionality requires token-based authentication. As browser support that requires the token-based authentication diminishes, we are hoping to deprecate -token-based authentication and keep only service account authentication in the future. +token-based authentication and keep only service account authentication in the future. > [!NOTE] > Use the ``schematic_service_account_creds.json`` file for the service > account mode of authentication (*for Google services/APIs*). Service accounts > are special Google accounts that can be used by applications to access Google APIs > programmatically via OAuth2.0, with the advantage being that they do not require -> human authorization. +> human authorization. ### 7. Set up pre-commit hooks @@ -397,7 +397,7 @@ After running the steps above, your setup is complete, and you can test it on a schematic manifest -c /path/to/config.yml get -dt -s ``` -2. Grab an existing manifest from synapse +2. Grab an existing manifest from synapse ``` schematic manifest -c /path/to/config.yml get -dt -d -s @@ -415,7 +415,7 @@ schematic model -c /path/to/config.yml validate -dt -mp -d -vc -mrt file_only ``` -Please visit more documentation [here](https://sage-schematic.readthedocs.io/en/stable/cli_reference.html#) for more information. +Please visit more documentation [here](https://sage-schematic.readthedocs.io/en/stable/cli_reference.html#) for more information. # Docker Usage @@ -427,7 +427,7 @@ using how to use `schematic` based on your OS (macOS/Linux). Use the Docker image to run `schematic`s REST API. You can either use the file path for the `config.yml` created using the installation instructions, or set up authentication with environment variables. -#### Example 1: Using the `config.yml` path +#### Example 1: Using the `config.yml` path ``` docker run --rm -p 3001:3001 \ -v $(pwd):/schematic -w /schematic --name schematic \ @@ -435,7 +435,7 @@ docker run --rm -p 3001:3001 \ -e GE_HOME=/usr/src/app/great_expectations/ \ sagebionetworks/schematic \ python /usr/src/app/run_api.py -``` +``` #### Example 2: Use environment variables 1. save content of `config.yml` as to environment variable `SCHEMATIC_CONFIG_CONTENT` by doing: `export SCHEMATIC_CONFIG_CONTENT=$(cat /path/to/config.yml)` @@ -452,7 +452,7 @@ docker run --rm -p 3001:3001 \ -e SERVICE_ACCOUNT_CREDS=$SERVICE_ACCOUNT_CREDS \ sagebionetworks/schematic \ python /usr/src/app/run_api.py -``` +``` ### Running `schematic` to Validate Manifests You can also use Docker to run `schematic` commands like validating manifests. Below are examples for different platforms. @@ -476,7 +476,7 @@ docker run \ -mp /schematic/tests/data/mock_manifests/Valid_Test_Manifest.csv \ -dt MockComponent \ -js /schematic/tests/data/example.model.jsonld -``` +``` #### Example for Windows @@ -509,8 +509,8 @@ variables wherever the application is running. Those variables are: Authentication (Oauth2 client credential exchange): Used in cases where an intermediate opentelemetry collector is not, or can not be used. -This option is not preferred over using an intermediate opentelemetry collector, but is -left in the code to show how we may export telemetry data with an authorization header +This option is not preferred over using an intermediate opentelemetry collector, but is +left in the code to show how we may export telemetry data with an authorization header deried from an oauth2 client credential exchange flow. - `TELEMETRY_EXPORTER_CLIENT_ID`: The ID of the client to use when executing the OAuth2.0 "Client Credentials" flow. @@ -548,7 +548,7 @@ curl --request POST \ ``` 3) Set an environment variable in your `.env` file like: `OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer ey...` -If you fail to create a new access token after 24 hours you will see HTTP 403 JWT +If you fail to create a new access token after 24 hours you will see HTTP 403 JWT Expired messages when the application attempts to export telemetry data. # Contributors diff --git a/certificate.conf b/certificate.conf index a61e8af82..b207d3804 100644 --- a/certificate.conf +++ b/certificate.conf @@ -23,4 +23,4 @@ server { return 200 'alive'; add_header Content-Type text/plain; } -} \ No newline at end of file +} diff --git a/docker-compose.yml b/docker-compose.yml index 862f41247..f12a6d972 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,19 +19,19 @@ services: GE_HOME: /usr/src/app/great_expectations/ # run schematic APIs in a docker container with uWSGI and nginx schematic-aws: - build: + build: context: ../schematic dockerfile: schematic_api/Dockerfile container_name: schematic-api-aws image: sagebionetworks/schematic-aws-api:latest restart: always - env_file: + env_file: - .env networks: - schematic - ports: + ports: - "${USE_LISTEN_PORT}:80" - "443:443" networks: schematic: - name: schematic \ No newline at end of file + name: schematic diff --git a/docs/md/details.md b/docs/md/details.md index 3e90ad2b8..4eea1c1a4 100644 --- a/docs/md/details.md +++ b/docs/md/details.md @@ -15,4 +15,4 @@ Steps involved in the `OAuth2` [authorization flow](https://github.com/Sage-Bion - The Google OAuth 2.0 system supports server-to-server interactions such as those between a web application and a Google service. For this scenario you need a service account, which is an account that belongs to your application instead of to an individual end user. Your application calls Google APIs on behalf of the service account, so users aren't directly involved. - Typically, an application uses a service account when the application uses Google APIs to work with its own data rather than a user's data. - The `service_account_creds.json` file is a key file that schematic will need access to in order to use the service account mode of authentication. -- \ No newline at end of file +- diff --git a/docs/md/manifest_validation_submission.md b/docs/md/manifest_validation_submission.md index 04c4aa602..a255f0717 100644 --- a/docs/md/manifest_validation_submission.md +++ b/docs/md/manifest_validation_submission.md @@ -29,4 +29,3 @@ As of March 29, 2022, the hybrid validation rules are as follows: protectAges unique inRange - diff --git a/docs/source/conf.py b/docs/source/conf.py index 1cd10df3d..677de60a5 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,11 +12,13 @@ # import os import sys -file_dir = os.path.dirname(__file__) + +file_dir = os.path.dirname(__file__) sys.path.append(file_dir) -from utils import _parse_toml import pathlib +from utils import _parse_toml + sys.path.insert(0, os.path.abspath("..")) diff --git a/docs/source/index.rst b/docs/source/index.rst index 444807c0f..2d235a77a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,5 +8,5 @@ Welcome to Schematic's documentation! .. toctree:: :maxdepth: 1 - - cli_reference \ No newline at end of file + + cli_reference diff --git a/docs/source/utils.py b/docs/source/utils.py index d35ed36b1..f8dad1130 100644 --- a/docs/source/utils.py +++ b/docs/source/utils.py @@ -1,7 +1,6 @@ import re - -from typing import List, Dict from pathlib import Path +from typing import Dict, List import pkg_resources import toml @@ -22,7 +21,6 @@ def _extract_author_names(authors_list: List[str]) -> str: author_names = [] for author in authors_list: - # extract name of each author by removing # portion from each list item name = re.sub(r" \<[^)]*\>", "", author) diff --git a/env.example b/env.example index bf999908b..6b68e3ed1 100644 --- a/env.example +++ b/env.example @@ -28,7 +28,7 @@ SERVICE_ACCOUNT_CREDS='Provide service account creds' # Used during integration test run to determine if files will be output for manual # inspection. These tests cannot fully finish all validation via code. All of these # tests will be marked by pytest "manual_verification_required" -# More information: https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/3055779846/Schematic+API+test+plan +# More information: https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/3055779846/Schematic+API+test+plan MANUAL_TEST_VERIFICATION=false # Used to determine if a local flask instance is created during integration testing. If @@ -40,4 +40,4 @@ USE_DEPLOYED_SCHEMATIC_API_SERVER=false # dev: https://schematic-dev.api.sagebionetworks.org # staging: https://schematic-staging.api.sagebionetworks.org # prod: https://schematic.api.sagebionetworks.org -SCHEMATIC_API_SERVER_URL=http://localhost:3001 \ No newline at end of file +SCHEMATIC_API_SERVER_URL=http://localhost:3001 diff --git a/great_expectations/.gitignore b/great_expectations/.gitignore index f03094397..a331178e0 100644 --- a/great_expectations/.gitignore +++ b/great_expectations/.gitignore @@ -1 +1 @@ -uncommitted/ \ No newline at end of file +uncommitted/ diff --git a/main.py b/main.py index f5b51bcac..97ce25e2d 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,9 @@ import os + from schematic_api.api import app -def main(): +def main(): # Get app configuration host = os.environ.get("APP_HOST", "0.0.0.0") port = os.environ.get("APP_PORT", "3001") @@ -12,5 +13,6 @@ def main(): # CORS(app, resources={r"*": {"origins": "*"}}) app.run(host=host, port=port, debug=False) + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/run_api.py b/run_api.py index 4b86ad2a9..774400c6d 100755 --- a/run_api.py +++ b/run_api.py @@ -2,9 +2,10 @@ # import our application # Run our application -from schematic_api.api import create_app import os +from schematic_api.api import create_app + if __name__ == "__main__": # Get app configuration host = os.environ.get("APP_HOST", "0.0.0.0") @@ -13,5 +14,5 @@ # Launch app app = create_app() - #TO DO: add a flag --debug to control debug parameter - app.run(host=host, port=port, debug=False) \ No newline at end of file + # TO DO: add a flag --debug to control debug parameter + app.run(host=host, port=port, debug=False) diff --git a/schematic/__main__.py b/schematic/__main__.py index 094624d7e..ab7a3fc76 100644 --- a/schematic/__main__.py +++ b/schematic/__main__.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import logging + import click import click_log diff --git a/schematic/configuration/configuration.py b/schematic/configuration/configuration.py index 63bf55313..e5f8fa7fc 100644 --- a/schematic/configuration/configuration.py +++ b/schematic/configuration/configuration.py @@ -1,15 +1,13 @@ """Configuration singleton for the Schematic Package""" -from typing import Optional, Any import os +from typing import Any, Optional + import yaml + from schematic.utils.general import normalize_path -from .dataclasses import ( - SynapseConfig, - ManifestConfig, - ModelConfig, - GoogleSheetsConfig, -) + +from .dataclasses import GoogleSheetsConfig, ManifestConfig, ModelConfig, SynapseConfig class ConfigNonAllowedFieldError(Exception): diff --git a/schematic/configuration/dataclasses.py b/schematic/configuration/dataclasses.py index 4b3d1560f..d5c661340 100644 --- a/schematic/configuration/dataclasses.py +++ b/schematic/configuration/dataclasses.py @@ -2,8 +2,9 @@ import re from dataclasses import field + +from pydantic import ConfigDict, Extra, validator from pydantic.dataclasses import dataclass -from pydantic import validator, ConfigDict, Extra # This turns on validation for value assignments after creation pydantic_config = ConfigDict(validate_assignment=True, extra=Extra.forbid) diff --git a/schematic/etc/README.md b/schematic/etc/README.md index 6b845c9ac..739ca8763 100644 --- a/schematic/etc/README.md +++ b/schematic/etc/README.md @@ -1,11 +1,11 @@ ## Package-specific files: #### The files within `etc` folder are: -`data_models`: +`data_models`: - `biothings.model.jsonld`: Base knowledge graph/vocabulary as specified by the [biolink model](https://biolink.github.io/biolink-model/). - `schema_org.model.jsonld`: Schema vocabulary as specified by [schema.org](https://schema.org/docs/gs.html#schemaorg_types). `validation_schemas`: - `class.schema.json`: JSON Schema used for validation against schema.org class definition standard. - `property.schema.json`: JSON Schema used for validation against schema.org property definition standard. -- `model.schema.json`: JSON Schema used for validation against schema.org standard. \ No newline at end of file +- `model.schema.json`: JSON Schema used for validation against schema.org standard. diff --git a/schematic/etc/data_models/schema_org.model.jsonld b/schematic/etc/data_models/schema_org.model.jsonld index e4e31c7d5..1a6d97a86 100644 --- a/schematic/etc/data_models/schema_org.model.jsonld +++ b/schematic/etc/data_models/schema_org.model.jsonld @@ -42158,4 +42158,4 @@ } } ] -} \ No newline at end of file +} diff --git a/schematic/etc/validation_schemas/class.schema.json b/schematic/etc/validation_schemas/class.schema.json index 37405eebd..fae4ddc97 100644 --- a/schematic/etc/validation_schemas/class.schema.json +++ b/schematic/etc/validation_schemas/class.schema.json @@ -235,4 +235,4 @@ "rdfs:label", "rdfs:subClassOf" ] -} \ No newline at end of file +} diff --git a/schematic/etc/validation_schemas/model.schema.json b/schematic/etc/validation_schemas/model.schema.json index 8ffd4a065..5e3fd55e0 100644 --- a/schematic/etc/validation_schemas/model.schema.json +++ b/schematic/etc/validation_schemas/model.schema.json @@ -411,4 +411,4 @@ "@graph", "@id" ] -} \ No newline at end of file +} diff --git a/schematic/exceptions.py b/schematic/exceptions.py index 200ffe7bc..52ccc45ff 100644 --- a/schematic/exceptions.py +++ b/schematic/exceptions.py @@ -1,5 +1,5 @@ """Schematic Exceptions""" -from typing import Optional, Any, Sequence +from typing import Any, Optional, Sequence class MissingConfigValueError(Exception): diff --git a/schematic/loader.py b/schematic/loader.py index be9158002..ac20dabbd 100644 --- a/schematic/loader.py +++ b/schematic/loader.py @@ -1,15 +1,16 @@ """Loader""" -from typing import Any from collections.abc import Iterable from errno import ENOENT from os import pathsep from re import split +from typing import Any + from pkg_resources import ( resource_exists, resource_filename, + resource_listdir, resource_stream, resource_string, - resource_listdir, ) diff --git a/schematic/schemas/__init__.py b/schematic/schemas/__init__.py index 9fce15edf..f24df9932 100644 --- a/schematic/schemas/__init__.py +++ b/schematic/schemas/__init__.py @@ -1,9 +1,9 @@ "Schema init file" from schematic.schemas.data_model_edges import DataModelEdges -from schematic.schemas.data_model_nodes import DataModelNodes from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_json_schema import DataModelJSONSchema from schematic.schemas.data_model_jsonld import DataModelJsonLD +from schematic.schemas.data_model_nodes import DataModelNodes from schematic.schemas.data_model_relationships import DataModelRelationships from schematic.schemas.data_model_validator import DataModelValidator diff --git a/schematic/schemas/commands.py b/schematic/schemas/commands.py index 5b143f640..cf53cdd48 100644 --- a/schematic/schemas/commands.py +++ b/schematic/schemas/commands.py @@ -1,24 +1,23 @@ """Schema Commands""" import logging -import time import re -from typing import get_args, Optional, Any +import time +from typing import Any, Optional, get_args import click import click_log # type: ignore -# pylint: disable=logging-fstring-interpolation - -from schematic.schemas.data_model_parser import DataModelParser +from schematic.help import schema_commands from schematic.schemas.data_model_graph import DataModelGraph -from schematic.schemas.data_model_validator import DataModelValidator from schematic.schemas.data_model_jsonld import convert_graph_to_jsonld - -from schematic.utils.schema_utils import DisplayLabelType +from schematic.schemas.data_model_parser import DataModelParser +from schematic.schemas.data_model_validator import DataModelValidator from schematic.utils.cli_utils import query_dict -from schematic.utils.schema_utils import export_schema -from schematic.help import schema_commands +from schematic.utils.schema_utils import DisplayLabelType, export_schema + +# pylint: disable=logging-fstring-interpolation + logger = logging.getLogger("schematic") click_log.basic_config(logger) diff --git a/schematic/schemas/data_model_edges.py b/schematic/schemas/data_model_edges.py index 02df4e16e..42105bf7d 100644 --- a/schematic/schemas/data_model_edges.py +++ b/schematic/schemas/data_model_edges.py @@ -1,5 +1,6 @@ """Data Model Edges""" -from typing import Union, Any +from typing import Any, Union + from schematic.schemas.data_model_relationships import DataModelRelationships diff --git a/schematic/schemas/data_model_json_schema.py b/schematic/schemas/data_model_json_schema.py index cb58102c7..e48f6b5ec 100644 --- a/schematic/schemas/data_model_json_schema.py +++ b/schematic/schemas/data_model_json_schema.py @@ -9,8 +9,8 @@ from schematic.schemas.data_model_graph import DataModelGraphExplorer from schematic.schemas.data_model_relationships import DataModelRelationships -from schematic.utils.validate_utils import rule_in_rule_list from schematic.utils.schema_utils import get_json_schema_log_file_path +from schematic.utils.validate_utils import rule_in_rule_list logger = logging.getLogger(__name__) diff --git a/schematic/schemas/data_model_jsonld.py b/schematic/schemas/data_model_jsonld.py index 0a2464ecb..bc0f49cab 100644 --- a/schematic/schemas/data_model_jsonld.py +++ b/schematic/schemas/data_model_jsonld.py @@ -1,20 +1,19 @@ """Data Model Jsonld""" +import copy import json import logging -import copy -from typing import Union - from dataclasses import dataclass, field -from dataclasses_json import config, dataclass_json +from typing import Union import networkx as nx # type: ignore +from dataclasses_json import config, dataclass_json from schematic.schemas.data_model_graph import DataModelGraphExplorer from schematic.schemas.data_model_relationships import DataModelRelationships from schematic.utils.schema_utils import ( - get_label_from_display_name, convert_bool_to_str, + get_label_from_display_name, ) logging.basicConfig() diff --git a/schematic/schemas/data_model_nodes.py b/schematic/schemas/data_model_nodes.py index a8ea4fd09..711ddf771 100644 --- a/schematic/schemas/data_model_nodes.py +++ b/schematic/schemas/data_model_nodes.py @@ -1,20 +1,19 @@ """Data model Nodes""" -from typing import Optional, Callable, Any - from inspect import isfunction +from typing import Any, Callable, Optional + import networkx as nx # type: ignore from rdflib import Namespace from schematic.schemas.data_model_relationships import DataModelRelationships - from schematic.utils.schema_utils import ( - get_label_from_display_name, - get_attribute_display_name_from_label, - convert_bool_to_str, - parse_validation_rules, DisplayLabelType, EntryType, + convert_bool_to_str, + get_attribute_display_name_from_label, + get_label_from_display_name, + parse_validation_rules, ) diff --git a/schematic/schemas/data_model_relationships.py b/schematic/schemas/data_model_relationships.py index c222285da..94369f288 100644 --- a/schematic/schemas/data_model_relationships.py +++ b/schematic/schemas/data_model_relationships.py @@ -1,9 +1,9 @@ """Data Model Relationships""" from schematic.utils.schema_utils import ( - get_label_from_display_name, - get_attribute_display_name_from_label, convert_bool_to_str, + get_attribute_display_name_from_label, + get_label_from_display_name, parse_validation_rules, ) diff --git a/schematic/schemas/data_model_validator.py b/schematic/schemas/data_model_validator.py index 2abc3b67e..00411405d 100644 --- a/schematic/schemas/data_model_validator.py +++ b/schematic/schemas/data_model_validator.py @@ -1,12 +1,12 @@ """Data Model Validator""" -import time -import logging -from typing import Tuple, Any, Iterable import itertools +import logging +import multiprocessing +import time from dataclasses import dataclass +from typing import Any, Iterable, Tuple -import multiprocessing import networkx as nx # type: ignore from schematic.schemas.data_model_relationships import DataModelRelationships diff --git a/schematic/schemas/json_schema_validator.py b/schematic/schemas/json_schema_validator.py index 0a1760c79..b6fbf0f9f 100644 --- a/schematic/schemas/json_schema_validator.py +++ b/schematic/schemas/json_schema_validator.py @@ -5,12 +5,12 @@ from jsonschema import validate -from schematic.utils.io_utils import load_schemaorg, load_json -from schematic.utils.general import str2list, dict2list, find_duplicates from schematic.utils.curie_utils import ( expand_curies_in_schema, extract_name_from_uri_or_curie, ) +from schematic.utils.general import dict2list, find_duplicates, str2list +from schematic.utils.io_utils import load_json, load_schemaorg class SchemaValidator: diff --git a/schematic/store/database/README.md b/schematic/store/database/README.md index 347f0cafd..115a00159 100644 --- a/schematic/store/database/README.md +++ b/schematic/store/database/README.md @@ -1,5 +1,5 @@ The python scripts stored here are sourced from . -This logic was extracted out of `schematic_db` as there were a new of required +This logic was extracted out of `schematic_db` as there were a new of required dependency updates that prevented using the updated `schematic_db` code. Those dependency updates included: diff --git a/schematic/store/database/synapse_database_wrapper.py b/schematic/store/database/synapse_database_wrapper.py index ba0ed2dc9..70b140bd5 100644 --- a/schematic/store/database/synapse_database_wrapper.py +++ b/schematic/store/database/synapse_database_wrapper.py @@ -1,4 +1,4 @@ -"""Wrapper class for interacting with Synapse database objects. Eventually this will +"""Wrapper class for interacting with Synapse database objects. Eventually this will be replaced with a more database/table class that exists within the SYNPY project.""" from typing import Optional diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 93a7140f2..fd35f706d 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -2018,7 +2018,7 @@ def annotate_upload_manifest_table(self, manifest, datasetId, metadataManifestPa rather than the manifest path """ - + # Add uuid for table updates and fill. if not "Uuid" in manifest.columns: manifest["Uuid"] = '' @@ -2048,12 +2048,12 @@ def annotate_upload_manifest_table(self, manifest, datasetId, metadataManifestPa # Upload manifest as a table and get the SynID and manifest manifest_synapse_table_id, manifest, table_manifest = self.upload_format_manifest_table( dmge, manifest, datasetId, table_name, restrict = restrict_manifest, useSchemaLabel=useSchemaLabel,) - + # Iterate over manifest rows, create Synapse entities and store corresponding entity IDs in manifest if needed # also set metadata for each synapse entity as Synapse annotations for idx, row in manifest.iterrows(): if not row["entityId"]: - # If not using entityIds, fill with manifest_table_id so + # If not using entityIds, fill with manifest_table_id so row["entityId"] = manifest_synapse_table_id entityId = '' else: @@ -2062,14 +2062,14 @@ def annotate_upload_manifest_table(self, manifest, datasetId, metadataManifestPa # Load manifest to synapse as a CSV File manifest_synapse_file_id = self.upload_manifest_file(manifest, metadataManifestPath, datasetId, restrict_manifest) - + # Get annotations for the file manifest. manifest_annotations = self.format_manifest_annotations(manifest, manifest_synapse_file_id) - + self.syn.set_annotations(manifest_annotations) logger.info("Associated manifest file with dataset on Synapse.") - + # Update manifest Synapse table with new entity id column. self.make_synapse_table( table_to_load = table_manifest, @@ -2079,7 +2079,7 @@ def annotate_upload_manifest_table(self, manifest, datasetId, metadataManifestPa update_col = 'Uuid', specify_schema = False, ) - + # Get annotations for the table manifest manifest_annotations = self.format_manifest_annotations(manifest, manifest_synapse_table_id) self.syn.set_annotations(manifest_annotations) diff --git a/schematic/utils/schema_utils.py b/schematic/utils/schema_utils.py index bbf34940f..77baf0798 100644 --- a/schematic/utils/schema_utils.py +++ b/schematic/utils/schema_utils.py @@ -6,11 +6,10 @@ import logging import os import string -from typing import Literal, Union, Optional +from typing import Literal, Optional, Union import inflection - logger = logging.getLogger(__name__) DisplayLabelType = Literal["class_label", "display_label"] diff --git a/schematic/utils/viz_utils.py b/schematic/utils/viz_utils.py index 48dff6867..ce0ec3012 100644 --- a/schematic/utils/viz_utils.py +++ b/schematic/utils/viz_utils.py @@ -1,6 +1,7 @@ """viz utils""" -from typing import Optional, Iterable, Sequence +from typing import Iterable, Optional, Sequence + import graphviz # type: ignore diff --git a/schematic/visualization/commands.py b/schematic/visualization/commands.py index 6b302856a..d64649830 100644 --- a/schematic/visualization/commands.py +++ b/schematic/visualization/commands.py @@ -6,18 +6,17 @@ import logging import sys -from typing import Any, get_args, cast +from typing import Any, cast, get_args import click import click_log # type: ignore -from schematic.visualization.attributes_explorer import AttributesExplorer -from schematic.visualization.tangled_tree import TangledTree, FigureType, TextType +from schematic.configuration.configuration import CONFIG +from schematic.help import model_commands, viz_commands from schematic.utils.cli_utils import log_value_from_config, query_dict from schematic.utils.schema_utils import DisplayLabelType -from schematic.help import viz_commands -from schematic.help import model_commands -from schematic.configuration.configuration import CONFIG +from schematic.visualization.attributes_explorer import AttributesExplorer +from schematic.visualization.tangled_tree import FigureType, TangledTree, TextType logger = logging.getLogger(__name__) click_log.basic_config(logger) diff --git a/schematic_api/Dockerfile b/schematic_api/Dockerfile index cfcc6cab7..c51344201 100644 --- a/schematic_api/Dockerfile +++ b/schematic_api/Dockerfile @@ -13,7 +13,7 @@ ENV PYTHONFAULTHANDLER=1 \ PIP_DISABLE_PIP_VERSION_CHECK=on \ PIP_DEFAULT_TIMEOUT=200 \ POETRY_VERSION=1.7.1 \ - APP_PARENT_DIR=/app \ + APP_PARENT_DIR=/app \ NGINX_CONFIG=/etc/nginx/conf.d \ APP_DIR=/app/app \ ROOT=/ \ @@ -24,16 +24,16 @@ ENV PYTHONFAULTHANDLER=1 \ LABEL maintainer="Lingling Peng Andrew Lamb Gianna Jordan " -LABEL version=$TAG +LABEL version=$TAG - -# Note: + +# Note: # The starting number of uWSGI processes is controlled by the variable UWSGI_CHEAPER, by default set to 2. # The maximum number of uWSGI processes is controlled by the variable UWSGI_PROCESSES, by default set to 16 # By default, the image starts with 2 uWSGI processes running. When the server is experiencing a high load, it creates up to 16 uWSGI processes to handle it on demand. # NGINX_MAX_UPLOAD is set to 0 by default that allows unlimited upload file sizes # NGINX_WORKER_CONNECTIONS is set to 1024 by default that allows a maximum limit of 1024 connections per worker. -# NGINX_WORKER_OPEN_FILES is set to 2048 by default that allows 2048 open files +# NGINX_WORKER_OPEN_FILES is set to 2048 by default that allows 2048 open files # run open ssl and generate certificate RUN apt update && \ @@ -74,11 +74,11 @@ RUN chown www-data:www-data ${APP_PARENT_DIR} RUN rm -rf ${APP_PARENT_DIR}/main.py RUN rm -rf ${APP_PARENT_DIR}/uwsgi.ini -# copy to use custom uwsgi.ini +# copy to use custom uwsgi.ini COPY ./uwsgi.ini ./ # create a separate folder called app -RUN mkdir app +RUN mkdir app WORKDIR ${APP_DIR} # copy other files to app/app @@ -101,13 +101,13 @@ COPY tests ./tests # change permission RUN chown -R www-data:www-data ${APP_DIR} -# allow downloading to synapse cache +# allow downloading to synapse cache RUN chown -R www-data:www-data /root # copy schematic COPY schematic ./schematic -# change permission +# change permission WORKDIR /var/www/ #The -R option: make the command recursive, so it will change the owner of all files and subdirectories within a given folder. RUN chown -R www-data:www-data /var/www/ diff --git a/schematic_api/api/README.md b/schematic_api/api/README.md index 7a0896761..2211af509 100644 --- a/schematic_api/api/README.md +++ b/schematic_api/api/README.md @@ -1,60 +1,60 @@ # Directions to run APIs locally ## Run Schematic without uWSGI (preferred approach for external users) -### Run Schematic APIs locally -1) To run Schematic API locally, simply install dependencies by doing: +### Run Schematic APIs locally +1) To run Schematic API locally, simply install dependencies by doing: ``` -poetry install +poetry install ``` 2) Get `service_account_creds.json` by doing `schematic init --config /path/to/config.yml`. -3) Run the APIs by doing: +3) Run the APIs by doing: ``` poetry run python3 run_api.py ``` You should be able to see swagger UI interface when visiting `localhost:3001/v1/ui` -### Run Schematic APIs in a docker container -To run Schematic in a docker container, check out `docker-compose.yml` file in this repository. Please comment out the second part related to building `schematic-aws` container. You could start Schematic docker container by running: +### Run Schematic APIs in a docker container +To run Schematic in a docker container, check out `docker-compose.yml` file in this repository. Please comment out the second part related to building `schematic-aws` container. You could start Schematic docker container by running: ``` docker compose up --build --remove-orphans ``` ## Run Schematic APIs with uWSGI and nginx in a docker container (preferred approach for developers) ### install uWSGI -Install uWSGI by doing: +Install uWSGI by doing: ``` poetry install --all-extras ``` Note: this approach only works for unix OSs users or windows user with WSL ### Run Schematic APIs with uWSGI and nginx in a docker container -See steps below: +See steps below: 1) Comment out the first part of `docker-compose.yml` and focus only on building container `schematic-aws`. 2) Get `schematic_service_account_creds.json` by following these [instructions](https://scribehow.com/shared/Enable_Google_Drive_and_Google_Sheets_APIs_for_project__yqfcJz_rQVeyTcg0KQCINA) -3) Make a copy of `env.example` and rename it as `.env` and keep it in the same directory as `env.example`. By default, schematic uses port 81. If port 81 is not available, please update `USE_LISTEN_PORT` in `.env` file. +3) Make a copy of `env.example` and rename it as `.env` and keep it in the same directory as `env.example`. By default, schematic uses port 81. If port 81 is not available, please update `USE_LISTEN_PORT` in `.env` file. 4) Copy the content of `service_account_creds.json` and put it in `.env` file after key `SERVICE_ACCOUNT_CREDS`. Remember to wrap around the credentials with single quotes. -5) Build a docker image and spin up docker container `schematic-api-aws` by running: +5) Build a docker image and spin up docker container `schematic-api-aws` by running: ```bash docker compose up --build --remove-orphans ``` -You should be able to view your application when visit: `https://127.0.0.1/v1/ui/`. You might receive an notification like this in your browser: +You should be able to view your application when visit: `https://127.0.0.1/v1/ui/`. You might receive an notification like this in your browser: Screen Shot 2023-05-23 at 3 31 46 PM -Please click on "show details" and "visit this website" to proceed. Note that the instructions might be slightly different for different browsers. +Please click on "show details" and "visit this website" to proceed. Note that the instructions might be slightly different for different browsers. -By default, this command builds up two containers (`schematic` and `schematic-aws`). You could spin up two containers if you want. But only `schematic-aws` runs flask with uWSGI. +By default, this command builds up two containers (`schematic` and `schematic-aws`). You could spin up two containers if you want. But only `schematic-aws` runs flask with uWSGI. ## Notes on installation -1. The warning message: "connexion.options - The swagger_ui directory could not be found." could be addressed by pip installing connexion[swagger-ui]. For Mac users, the command should be: +1. The warning message: "connexion.options - The swagger_ui directory could not be found." could be addressed by pip installing connexion[swagger-ui]. For Mac users, the command should be: ```bash pip install connexion['swagger=ui'] ``` -2. Due to security concerns, running `schematic init` is no longer supported. Please follow these [instructions](https://scribehow.com/shared/Enable_Google_Drive_and_Google_Sheets_APIs_for_project__yqfcJz_rQVeyTcg0KQCINA) to obtain `schematic_service_account_creds.json`. Please specify the path of `schematic_service_account_creds.json` in config.yml (see google_sheets > service_acct_creds) if you intend to use the configuration file. +2. Due to security concerns, running `schematic init` is no longer supported. Please follow these [instructions](https://scribehow.com/shared/Enable_Google_Drive_and_Google_Sheets_APIs_for_project__yqfcJz_rQVeyTcg0KQCINA) to obtain `schematic_service_account_creds.json`. Please specify the path of `schematic_service_account_creds.json` in config.yml (see google_sheets > service_acct_creds) if you intend to use the configuration file. ## Access Results Access the Swagger UI docs at this location: @@ -63,49 +63,49 @@ http://localhost:3001/v1/ui/ ``` ## Notes for trouble shooting -1. ImportError: cannot import name 'soft_unicode' from 'markupsafe' -2. ImportError: cannot import name 'json' from 'itsdangerous' +1. ImportError: cannot import name 'soft_unicode' from 'markupsafe' +2. ImportError: cannot import name 'json' from 'itsdangerous' -To resolve: +To resolve: -Make sure that the following libraries have the correct version: +Make sure that the following libraries have the correct version: * itsdangerous version: ^2.0.0 * jinja2 version: >2.11.3 * markupsafe version: ^2.1.0 -## Notes for using schematic features and API endpoints utilizing Google Services (e.g. manifest generation): -Before trying out the API endpoints, please make sure that you have obtained `schematic_service_account_creds.json` (Instructions can be found in schematic/README.md) +## Notes for using schematic features and API endpoints utilizing Google Services (e.g. manifest generation): +Before trying out the API endpoints, please make sure that you have obtained `schematic_service_account_creds.json` (Instructions can be found in schematic/README.md) ### GET /manifest/generate -This endpoint functions similarly to the following: +This endpoint functions similarly to the following: ```bash schematic manifest -c ~/path/to/config.yml get -d ``` -Examples: +Examples: 1) Generate a patient manifest by using the sample data model. - Simply click on "Try it out" on swagger UI + Simply click on "Try it out" on swagger UI -2) Get an existing manifest: +2) Get an existing manifest: -* Step 1: Make sure you have credentials to download the desired manifest from Synapse. The "download" button should be disabled if you don't have credentials. +* Step 1: Make sure you have credentials to download the desired manifest from Synapse. The "download" button should be disabled if you don't have credentials. -* Step 2: Make sure you set asset_view to the right value. For Synapse, asset_view is the same as master_fileview_id in config.yml. +* Step 2: Make sure you set asset_view to the right value. For Synapse, asset_view is the same as master_fileview_id in config.yml. * Step 3: Use parent id of the manifest for "dataset_id" parameter - -Note: if the dataset_id you provided is invalid, it will generate an empty manifest based on the data model. + +Note: if the dataset_id you provided is invalid, it will generate an empty manifest based on the data model. ### POST /model/submit - + * For authorization, please use the value of `auth token` in your `.synapseConfig` * For the dataset_id parameter, please create a test folder on synapse and use its synapse ID -* For uploading a csv file, please generate an empty manifest by using `GET /manifest/generate` endpoint and fill it out. +* For uploading a csv file, please generate an empty manifest by using `GET /manifest/generate` endpoint and fill it out. For the patient manifest, "Family History" column only accepts a list. The details of the validation rules (based on the example data model) could be found [here](https://github.com/Sage-Bionetworks/schematic/blob/develop/tests/data/example.model.csv) diff --git a/schematic_api/api/__main__.py b/schematic_api/api/__main__.py index afc24b44a..dac669566 100644 --- a/schematic_api/api/__main__.py +++ b/schematic_api/api/__main__.py @@ -1,4 +1,5 @@ import os + from schematic_api.api import app diff --git a/schematic_api/api/openapi/api.yaml b/schematic_api/api/openapi/api.yaml index 7c32e8d90..a3b28266a 100644 --- a/schematic_api/api/openapi/api.yaml +++ b/schematic_api/api/openapi/api.yaml @@ -5,7 +5,7 @@ info: description: >- This service exposes core functionalities from schematic as REST API endpoints - + servers: - url: /v1 @@ -40,7 +40,7 @@ paths: schema: type: string description: > - Title of Manifest or Title Prefix, + Title of Manifest or Title Prefix, if making multiple manifests example: Example required: false @@ -49,13 +49,13 @@ paths: style: form schema: type: array - items: + items: type: string nullable: true description: > - Data Model Component(s). + Data Model Component(s). To make all manifests, enter "all manifests". - example: + example: - Patient - Biospecimen required: true @@ -71,7 +71,7 @@ paths: style: form schema: type: array - items: + items: type: string nullable: true description: > @@ -89,7 +89,7 @@ paths: schema: type: string enum: ["excel", "google_sheet", "dataframe (only if getting existing manifests)"] - description: If "excel" gets selected, this approach would avoid sending metadata to Google sheet APIs; if "google_sheet" gets selected, this would return a Google sheet URL. This parameter could potentially override sheet_url parameter. + description: If "excel" gets selected, this approach would avoid sending metadata to Google sheet APIs; if "google_sheet" gets selected, this would return a Google sheet URL. This parameter could potentially override sheet_url parameter. required: false - in: query name: strict_validation @@ -105,7 +105,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -162,11 +162,11 @@ paths: schema: type: string "403": - description: You don't have access to the requested resources. + description: You don't have access to the requested resources. "500": description: Please check schematic log tags: - - Manifest Operations + - Manifest Operations /dataset/manifest/download: get: summary: Endpoint to download an existing manifest by using a dataset id @@ -227,7 +227,7 @@ paths: # file_name will be the field name in # this multipart request file_name: - description: Upload a json or a csv file. + description: Upload a json or a csv file. type: string format: binary security: @@ -257,7 +257,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -297,7 +297,7 @@ paths: name: project_scope schema: type: array - items: + items: type: string nullable: false description: List, a subset of the projects contained within the asset view that are relevant for the current operation. Speeds up some operations that interact with Synapse. Relevant for validating manifests involving cross-manifest validation, but optional. @@ -342,7 +342,7 @@ paths: type: object properties: file_name: - description: Upload a json or a csv file. + description: Upload a json or a csv file. type: string format: binary security: @@ -363,7 +363,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -455,13 +455,13 @@ paths: schema: type: boolean default: true - description: if false, do not add annotations when submitting file-based manifests. + description: if false, do not add annotations when submitting file-based manifests. required: false - in: query name: project_scope schema: type: array - items: + items: type: string nullable: false description: List, a subset of the projects contained within the asset view that are relevant for the current operation. Speeds up some operations that interact with Synapse. @@ -521,7 +521,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -539,8 +539,8 @@ paths: - Manifest Operations /manifest/populate: post: - summary: Create a Google sheet link based on an existing manifest. - description: Create a Google sheet link based on an existing manifest. + summary: Create a Google sheet link based on an existing manifest. + description: Create a Google sheet link based on an existing manifest. requestBody: content: multipart/form-data: @@ -566,7 +566,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -633,7 +633,7 @@ paths: "200": description: A list of json "500": - description: Check schematic log. + description: Check schematic log. tags: - Manifest Operations /storage/projects: @@ -721,7 +721,7 @@ paths: items: type: string nullable: true - description: a list of files with particular names (i.e. Sample_A.txt). If you leave it empty, it will return all dataset files under the dataset ID. + description: a list of files with particular names (i.e. Sample_A.txt). If you leave it empty, it will return all dataset files under the dataset ID. required: false - in: query name: full_path @@ -765,7 +765,7 @@ paths: "200": description: csv file path or json "500": - description: Check schematic log. + description: Check schematic log. tags: - Synapse Storage /storage/project/manifests: @@ -794,7 +794,7 @@ paths: required: true responses: "200": - description: A list of tuples(json). + description: A list of tuples(json). content: application/json: schema: @@ -807,7 +807,7 @@ paths: ] ] "500": - description: Check schematic log. + description: Check schematic log. tags: - Synapse Storage /storage/entity/type: @@ -816,7 +816,7 @@ paths: description: Check type of an entity and return entity type operationId: schematic_api.api.routes.check_entity_type security: - - access_token: [] + - access_token: [] parameters: - in: query name: entity_id @@ -830,7 +830,7 @@ paths: "200": description: return type of a schematic entity "500": - description: Check schematic log. + description: Check schematic log. tags: - Synapse Storage @@ -840,7 +840,7 @@ paths: description: Check if a file or a folder is in a given asset view. Return "true" or "false" operationId: schematic_api.api.routes.check_if_files_in_assetview security: - - access_token: [] + - access_token: [] parameters: - in: query name: asset_view @@ -855,14 +855,14 @@ paths: schema: type: string nullable: false - description: ID of file or folder. For example, for Synapse this would be the Synapse ID. + description: ID of file or folder. For example, for Synapse this would be the Synapse ID. example: syn30988314 required: true responses: "200": description: return "true" or "false" "500": - description: Check schematic log. + description: Check schematic log. tags: - Synapse Storage /schemas/get/schema: @@ -886,7 +886,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -899,7 +899,7 @@ paths: schema: type: string "500": - description: Check schematic log. + description: Check schematic log. tags: - Schema Operation @@ -932,16 +932,16 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. required: false responses: "200": - description: A list of properties of a given class. + description: A list of properties of a given class. "500": - description: Check schematic log. + description: Check schematic log. tags: - Schema Operation /schemas/get/graph_by_edge_type: @@ -973,7 +973,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -992,7 +992,7 @@ paths: ] ] "500": - description: Check schematic log. + description: Check schematic log. tags: - Schema Operation /schemas/is_node_required: @@ -1028,9 +1028,9 @@ paths: required: false responses: "200": - description: return a boolean + description: return a boolean "500": - description: Check schematic log. + description: Check schematic log. tags: - Schema Operation @@ -1052,7 +1052,7 @@ paths: name: node_list schema: type: array - items: + items: type: string nullable: false description: List of node labels. @@ -1065,7 +1065,7 @@ paths: nullable: false enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -1074,7 +1074,7 @@ paths: "200": description: return List[str] "500": - description: Check schematic log. + description: Check schematic log. tags: - Schema Operation @@ -1108,7 +1108,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -1117,7 +1117,7 @@ paths: "200": description: return a list "500": - description: Check schematic log. + description: Check schematic log. tags: - Schema Operation @@ -1150,7 +1150,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -1178,7 +1178,7 @@ paths: description: Check schematic log. tags: - Schema Operation - + /utils/get_property_label_from_display_name: get: summary: Converts a given display name string into a proper property label string @@ -1204,10 +1204,10 @@ paths: "200": description: The property label of the display name. "500": - description: Check schematic log. + description: Check schematic log. tags: - Schema Operation - + /schemas/get_node_range: get: summary: Get all the valid values that are associated with a node label. @@ -1237,7 +1237,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -1287,7 +1287,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -1339,7 +1339,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -1376,7 +1376,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -1428,7 +1428,7 @@ paths: nullable: true enum: ["display_label", "class_label"] default: 'class_label' - description: Choose how to set the label in the data model. + description: Choose how to set the label in the data model. display_label, use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label. class_label, default, use standard class or property label. Do not change from default unless there is a real need, using 'display_label' can have consequences if not used properly. @@ -1443,18 +1443,18 @@ paths: example: Attribute,Label,Description,Required,Cond_Req,Valid Values,Conditional Requirements,Component Component,Component,TBD,False,,,,Patient - + Patient ID,PatientID,TBD,True,,,,Patient - + Year of Birth,YearofBirth,TBD,False,,,,Patient - Sex,Sex,TBD,True,,"['Female', 'Other', 'Male']",,Patient + Sex,Sex,TBD,True,,"['Female', 'Other', 'Male']",,Patient Diagnosis,Diagnosis,TBD,True,,"['Cancer', 'Healthy']",,Patient Cancer Type,CancerType,TBD,False,True,"['Lung', 'Skin', 'Colorectal', 'Breast', 'Prostate', '']"," If Diagnosis is ""Cancer"" then ""Cancer Type"" is required",Patient - Family History,FamilyHistory,TBD,False,True,," If Diagnosis is ""Cancer"" then ""Family History"" is required",Patient + Family History,FamilyHistory,TBD,False,True,," If Diagnosis is ""Cancer"" then ""Family History"" is required",Patient tags: - Visualization Operations @@ -1474,4 +1474,4 @@ paths: "500": description: Schematic version was not able to be identified. tags: - - Version \ No newline at end of file + - Version diff --git a/self-signed.conf b/self-signed.conf index 75a8db70a..46713b50e 100644 --- a/self-signed.conf +++ b/self-signed.conf @@ -1,2 +1,2 @@ ssl_certificate /etc/ssl/certs/localhost.crt; -ssl_certificate_key /etc/ssl/private/localhost.key; \ No newline at end of file +ssl_certificate_key /etc/ssl/private/localhost.key; diff --git a/ssl-params.conf b/ssl-params.conf index 8ae920484..a0cc44b41 100644 --- a/ssl-params.conf +++ b/ssl-params.conf @@ -15,4 +15,4 @@ resolver_timeout 5s; # add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; -add_header X-XSS-Protection "1; mode=block"; \ No newline at end of file +add_header X-XSS-Protection "1; mode=block"; diff --git a/tests/data/example.model.jsonld b/tests/data/example.model.jsonld index 7e4560d50..8fe991da0 100644 --- a/tests/data/example.model.jsonld +++ b/tests/data/example.model.jsonld @@ -1769,4 +1769,4 @@ } ], "@id": "http://schema.biothings.io/#0.1" -} \ No newline at end of file +} diff --git a/tests/data/example.single_rule.model.jsonld b/tests/data/example.single_rule.model.jsonld index 24da12094..c3c5c5f88 100644 --- a/tests/data/example.single_rule.model.jsonld +++ b/tests/data/example.single_rule.model.jsonld @@ -1467,4 +1467,4 @@ } ], "@id": "http://schema.biothings.io/#0.1" -} \ No newline at end of file +} diff --git a/tests/data/example_required_vr_test.model.csv b/tests/data/example_required_vr_test.model.csv index c7b6aecde..616deba46 100644 --- a/tests/data/example_required_vr_test.model.csv +++ b/tests/data/example_required_vr_test.model.csv @@ -18,4 +18,4 @@ BAM,,,Genome Build,,FALSE,ValidValue,,, CRAM,,,"Genome Build, Genome FASTA",,FALSE,ValidValue,,, CSV/TSV,,,Genome Build,,FALSE,ValidValue,,, Genome Build,,"GRCh37, GRCh38, GRCm38, GRCm39",,,TRUE,DataProperty,,, -Genome FASTA,,,,,TRUE,DataProperty,,, \ No newline at end of file +Genome FASTA,,,,,TRUE,DataProperty,,, diff --git a/tests/data/example_required_vr_test.model.jsonld b/tests/data/example_required_vr_test.model.jsonld index 47821c20e..de09e8390 100644 --- a/tests/data/example_required_vr_test.model.jsonld +++ b/tests/data/example_required_vr_test.model.jsonld @@ -862,4 +862,4 @@ } ], "@id": "http://schema.biothings.io/#0.1" -} \ No newline at end of file +} diff --git a/tests/data/example_test_nones.model.csv b/tests/data/example_test_nones.model.csv index e39ec8e5d..4a4100ac8 100644 --- a/tests/data/example_test_nones.model.csv +++ b/tests/data/example_test_nones.model.csv @@ -49,4 +49,4 @@ Check Date,date,,,,,FALSE,DataProperty,, Check NA,int::IsNA,,,,,FALSE,DataProperty,, MockRDB,,,,"Component, MockRDB_id, SourceManifest",,FALSE,DataType,, MockRDB_id,int,,,,,TRUE,DataProperty,, -SourceManifest,,,,,,TRUE,DataProperty,, \ No newline at end of file +SourceManifest,,,,,,TRUE,DataProperty,, diff --git a/tests/data/example_test_nones.model.jsonld b/tests/data/example_test_nones.model.jsonld index c47804236..52b96cc62 100644 --- a/tests/data/example_test_nones.model.jsonld +++ b/tests/data/example_test_nones.model.jsonld @@ -1470,4 +1470,4 @@ } ], "@id": "http://schema.biothings.io/#0.1" -} \ No newline at end of file +} diff --git a/tests/data/mock_manifests/Biospecimen_required_vr_test_fail.manifest.csv b/tests/data/mock_manifests/Biospecimen_required_vr_test_fail.manifest.csv index bfacc736a..1ee83f467 100644 --- a/tests/data/mock_manifests/Biospecimen_required_vr_test_fail.manifest.csv +++ b/tests/data/mock_manifests/Biospecimen_required_vr_test_fail.manifest.csv @@ -1,2 +1,2 @@ -Sample ID,Patient ID,Tissue Status,Component -jldkj,,Healthy,Biospecimen \ No newline at end of file +Sample ID,Patient ID,Tissue Status,Component +jldkj,,Healthy,Biospecimen diff --git a/tests/data/mock_manifests/Biospecimen_required_vr_test_pass.manifest.csv b/tests/data/mock_manifests/Biospecimen_required_vr_test_pass.manifest.csv index 31557c4f6..174030e9a 100644 --- a/tests/data/mock_manifests/Biospecimen_required_vr_test_pass.manifest.csv +++ b/tests/data/mock_manifests/Biospecimen_required_vr_test_pass.manifest.csv @@ -1,2 +1,2 @@ -Sample ID,Patient ID,Tissue Status,Component -jldkj,lkjdkl,Healthy,Biospecimen \ No newline at end of file +Sample ID,Patient ID,Tissue Status,Component +jldkj,lkjdkl,Healthy,Biospecimen diff --git a/tests/data/mock_manifests/Example.Patient.manifest.json b/tests/data/mock_manifests/Example.Patient.manifest.json index a9d28a977..c66aa5c8f 100644 --- a/tests/data/mock_manifests/Example.Patient.manifest.json +++ b/tests/data/mock_manifests/Example.Patient.manifest.json @@ -26,4 +26,4 @@ "Cancer Type": "Colorectal", "Family History": "Lung,Breast" } -] \ No newline at end of file +] diff --git a/tests/data/mock_manifests/Invalid_Biospecimen_Missing_Column_Manifest.csv b/tests/data/mock_manifests/Invalid_Biospecimen_Missing_Column_Manifest.csv index f84922de2..3fd4df8e0 100644 --- a/tests/data/mock_manifests/Invalid_Biospecimen_Missing_Column_Manifest.csv +++ b/tests/data/mock_manifests/Invalid_Biospecimen_Missing_Column_Manifest.csv @@ -1,2 +1,2 @@ -Sample ID,Patient ID,Component -123,123,Biospecimen \ No newline at end of file +Sample ID,Patient ID,Component +123,123,Biospecimen diff --git a/tests/data/mock_manifests/Invalid_Test_Manifest.csv b/tests/data/mock_manifests/Invalid_Test_Manifest.csv index a685acca7..9385558c6 100644 --- a/tests/data/mock_manifests/Invalid_Test_Manifest.csv +++ b/tests/data/mock_manifests/Invalid_Test_Manifest.csv @@ -2,4 +2,4 @@ Component,Check List,Check List Enum,Check List Like,Check List Like Enum,Check MockComponent,"ab,cd","ab,cd",ab,ab,"ab,cd","ab,cd","a,c,f","a,c,f","a,c,f","ab,cd","ab,cd,ef",a,a,5.4,6,99.65,7,valid,https://www.google.com/,1738,1738,8085,98085,1911,334,,6549,str1,70,32-984,7 MockComponent,9,a,c,c,ab cd,ab,a c f,a c f,a c f,invalid list values,ab cd ef,q,m,0,c,99,5.63,94,http://googlef.com/,7163,51100,9965,71738,123,717,,32851,str1,30,notADate,9.5 MockComponent,ab,ab,"cd, ab","cd, ab",ab,"a, b",a,a,a,"ab,cd","ab,cd,ef",b,b,683902,6.5,62.3,2,valid,https://github.com/Sage-Bionetworks/schematic,8085,8085,1738,210065,1427,123,,6550,str1,90,84-43-094,Not Applicable -,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file +,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/tests/data/mock_manifests/MockComponent-cross-manifest-1.csv b/tests/data/mock_manifests/MockComponent-cross-manifest-1.csv index ea543e9b9..bc4be842b 100644 --- a/tests/data/mock_manifests/MockComponent-cross-manifest-1.csv +++ b/tests/data/mock_manifests/MockComponent-cross-manifest-1.csv @@ -12,4 +12,4 @@ MockComponent,"a,m,f","ab,cd","ab,ef",ab,"ab,cd","ab,cd,ef","d,e","a,c","d,e",or MockComponent,"a,m,f","ab,cd","ab,ef",ef,"ab,cd","ab,cd,ef","d,e","a,c","d,e",test,"a,b,c,d",9,9,16.71666667,9,test,https://www.google.com/,test,2100,300,300,110,110,,6571,110,60,02/01/2005,1 MockComponent,"a,m,f","ab,cd","ab,ef",cd,"ab,cd","ab,cd,ef","d,e","a,c","d,e",test,"a,b,c,d",0,0,18.36666667,0,test,https://www.google.com/,test,2200,300,300,111,111,,6571,111,61,02/02/2005,1 MockComponent,"a,m,f","ab,cd","ab,ef",ab,"ab,cd","ab,cd,ef","d,e","a,c","d,e",test,"a,b,c,d",1,1,20.01666667,1,test,https://www.google.com/,test,2300,300,300,120,120,,6571,120,62,02/03/2005,1 -MockComponent,"a,m,f","ab,cd","ab,ef",ef,"ab,cd","ab,cd,ef","d,e","a,c","d,e",apple,"a,b,c,d",2,2,21.66666667,2,apple,https://www.google.com/,apple,2400,300,300,130,130,,6571,130,63,02/04/2005,1 \ No newline at end of file +MockComponent,"a,m,f","ab,cd","ab,ef",ef,"ab,cd","ab,cd,ef","d,e","a,c","d,e",apple,"a,b,c,d",2,2,21.66666667,2,apple,https://www.google.com/,apple,2400,300,300,130,130,,6571,130,63,02/04/2005,1 diff --git a/tests/data/mock_manifests/MockComponent-cross-manifest-2.csv b/tests/data/mock_manifests/MockComponent-cross-manifest-2.csv index 2b391fd39..5d6aa8ad8 100644 --- a/tests/data/mock_manifests/MockComponent-cross-manifest-2.csv +++ b/tests/data/mock_manifests/MockComponent-cross-manifest-2.csv @@ -12,4 +12,4 @@ MockComponent,"a,m,f","ab,cd","ab,ef",ab,"ab,cd","ab,cd,ef","d,e","a,c","d,e",or MockComponent,"a,m,f","ab,cd","ab,ef",ef,"ab,cd","ab,cd,ef","d,e","a,c","d,e",test,"a,b,c,d",9,9,16.71666667,9,test,https://www.google.com/,test,1200,300,9001,7,12,,6571,110,60,02/01/2005,1 MockComponent,"a,m,f","ab,cd","ab,ef",cd,"ab,cd","ab,cd,ef","d,e","a,c","d,e",test,"a,b,c,d",0,0,18.36666667,0,test,https://www.google.com/,test,1200,300,9001,6,12,,6571,111,61,02/02/2005,1 MockComponent,"a,m,f","ab,cd","ab,ef",ab,"ab,cd","ab,cd,ef","d,e","a,c","d,e",test,"a,b,c,d",1,1,20.01666667,1,test,https://www.google.com/,test,1200,300,9001,9,12,,6571,120,62,02/03/2005,1 -MockComponent,"a,m,f","ab,cd","ab,ef",ef,"ab,cd","ab,cd,ef","d,e","a,c","d,e",apple,"a,b,c,d",2,2,21.66666667,2,apple,https://www.google.com/,apple,1400,300,9001,7,12,,6571,130,63,02/04/2005,1 \ No newline at end of file +MockComponent,"a,m,f","ab,cd","ab,ef",ef,"ab,cd","ab,cd,ef","d,e","a,c","d,e",apple,"a,b,c,d",2,2,21.66666667,2,apple,https://www.google.com/,apple,1400,300,9001,7,12,,6571,130,63,02/04/2005,1 diff --git a/tests/data/mock_manifests/Mock_Component_rule_combination.csv b/tests/data/mock_manifests/Mock_Component_rule_combination.csv index 08f1babf7..7784965f8 100644 --- a/tests/data/mock_manifests/Mock_Component_rule_combination.csv +++ b/tests/data/mock_manifests/Mock_Component_rule_combination.csv @@ -12,4 +12,4 @@ MockComponent,"a,m,f","ab,cd","ab,ef",ab,"ab,cd","ab,cd,ef","d,e,a","a,c","d,e,f MockComponent,"a,m,f","ab,cd","ab,ef",ef,"ab,cd","ab,cd,ef","d,e","a,c","d,e",test,"a,b,c,d",9,9,16.71666667,9,test,https://www.google.com/,test,2100,300,300,110,110,,6571,110,60,02/01/2005,1 MockComponent,"a,m,f","ab,cd","ab,ef",cd,"ab,cd","ab,cd,ef","d,e","a,c","d,e,f",test,"a,b,c,d",0,0,18.36666667,0,test,https://www.google.com/,test,2200,300,300,111,111,,6571,111,61,02/02/2005,1 MockComponent,"a,m,f","ab,cd","ab,ef",ab,"ab,cd","ab,cd,ef","d,e","a,c","d,e",test,"a,b,c,d",1,1,20.01666667,1,test,https://www.google.com/,test,2300,300,300,120,120,,6571,120,62,02/03/2005,1 -MockComponent,"a,m,f","ab,cd","ab,ef",ef,"ab,cd","ab,cd,ef","d,e","a,c","d,e,f",apple,"a,b,c,d",2,2,21.66666667,2,apple,https://www.google.com/,apple,2400,300,300,130,130,,6571,130,63,02/04/2005,1 \ No newline at end of file +MockComponent,"a,m,f","ab,cd","ab,ef",ef,"ab,cd","ab,cd,ef","d,e","a,c","d,e,f",apple,"a,b,c,d",2,2,21.66666667,2,apple,https://www.google.com/,apple,2400,300,300,130,130,,6571,130,63,02/04/2005,1 diff --git a/tests/data/mock_manifests/Patient_required_vr_test_pass.manifest.csv b/tests/data/mock_manifests/Patient_required_vr_test_pass.manifest.csv index f9c68cd58..66db18b44 100644 --- a/tests/data/mock_manifests/Patient_required_vr_test_pass.manifest.csv +++ b/tests/data/mock_manifests/Patient_required_vr_test_pass.manifest.csv @@ -1,2 +1,2 @@ -Patient ID,Sex,Year of Birth,Diagnosis,Component,Cancer Type,Family History -,Female,,Cancer,Patient,Colorectal,"Colorectal, Breast" \ No newline at end of file +Patient ID,Sex,Year of Birth,Diagnosis,Component,Cancer Type,Family History +,Female,,Cancer,Patient,Colorectal,"Colorectal, Breast" diff --git a/tests/data/mock_manifests/Patient_test_no_entry_for_cond_required_column.manifest.csv b/tests/data/mock_manifests/Patient_test_no_entry_for_cond_required_column.manifest.csv index 7452c9660..19ccfc895 100644 --- a/tests/data/mock_manifests/Patient_test_no_entry_for_cond_required_column.manifest.csv +++ b/tests/data/mock_manifests/Patient_test_no_entry_for_cond_required_column.manifest.csv @@ -1,4 +1,4 @@ Patient ID,Sex,Year of Birth,Diagnosis,Component,Cancer Type,Family History Kdj;lkjdl,Male,,Healthy,Patient,, kdie,Female,,Healthy,Patient,, -kjdl,Male,,Healthy,Patient,, \ No newline at end of file +kjdl,Male,,Healthy,Patient,, diff --git a/tests/data/mock_manifests/TestManifestOperation_test_submit_nested_manifest_table_and_file_replace.csv b/tests/data/mock_manifests/TestManifestOperation_test_submit_nested_manifest_table_and_file_replace.csv index 6bcb468c6..11dade64a 100644 --- a/tests/data/mock_manifests/TestManifestOperation_test_submit_nested_manifest_table_and_file_replace.csv +++ b/tests/data/mock_manifests/TestManifestOperation_test_submit_nested_manifest_table_and_file_replace.csv @@ -1,2 +1,2 @@ -Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA,Year of Birth,author,confidence,date,eTag,IsImportantBool,IsImportantText,impact,entityId,RandomizedAnnotation -schematic - main/TestDatasets/TestDataset-Annotations-nested-submit/Sample_C.txt,some sample id,FASTQ,BulkRNA-seqAssay,,,,,,,0bf00691-a6e4-4487-9cab-851e22416ed2,FALSE,FALSE,,syn63646199, \ No newline at end of file +Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA,Year of Birth,author,confidence,date,eTag,IsImportantBool,IsImportantText,impact,entityId,RandomizedAnnotation +schematic - main/TestDatasets/TestDataset-Annotations-nested-submit/Sample_C.txt,some sample id,FASTQ,BulkRNA-seqAssay,,,,,,,0bf00691-a6e4-4487-9cab-851e22416ed2,FALSE,FALSE,,syn63646199, diff --git a/tests/data/mock_manifests/TestManifestSubmission_test_submit_manifest_with_blacklisted_characters.csv b/tests/data/mock_manifests/TestManifestSubmission_test_submit_manifest_with_blacklisted_characters.csv index 351a267ac..e37c0e8f4 100644 --- a/tests/data/mock_manifests/TestManifestSubmission_test_submit_manifest_with_blacklisted_characters.csv +++ b/tests/data/mock_manifests/TestManifestSubmission_test_submit_manifest_with_blacklisted_characters.csv @@ -1,2 +1,2 @@ -Filename,Sample-ID,File-Format,Component,Genome Build,Genome FASTA,entityId -Manifest Submission - Manual test - file-based manifest submission/test-annotation-key-table-column-name/sample A.txt,100,FASTQ,BulkRNA-seqAssay,GRCh38,,syn63607043 \ No newline at end of file +Filename,Sample-ID,File-Format,Component,Genome Build,Genome FASTA,entityId +Manifest Submission - Manual test - file-based manifest submission/test-annotation-key-table-column-name/sample A.txt,100,FASTQ,BulkRNA-seqAssay,GRCh38,,syn63607043 diff --git a/tests/data/mock_manifests/TestManifestSubmission_test_submit_manifest_with_hide_blanks.csv b/tests/data/mock_manifests/TestManifestSubmission_test_submit_manifest_with_hide_blanks.csv index 18bad92c0..b809c6961 100644 --- a/tests/data/mock_manifests/TestManifestSubmission_test_submit_manifest_with_hide_blanks.csv +++ b/tests/data/mock_manifests/TestManifestSubmission_test_submit_manifest_with_hide_blanks.csv @@ -1,2 +1,2 @@ -Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA,entityId -Manifest Submission - Manual test - file-based manifest submission/hide-blanks-true/sample A.txt,1,FASTQ,BulkRNA-seqAssay,,,syn63606862 \ No newline at end of file +Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA,entityId +Manifest Submission - Manual test - file-based manifest submission/hide-blanks-true/sample A.txt,1,FASTQ,BulkRNA-seqAssay,,,syn63606862 diff --git a/tests/data/mock_manifests/TestManifestValidation_test_manifest_validation_basic_valid.csv b/tests/data/mock_manifests/TestManifestValidation_test_manifest_validation_basic_valid.csv index dee60aaf9..fcbf309c4 100644 --- a/tests/data/mock_manifests/TestManifestValidation_test_manifest_validation_basic_valid.csv +++ b/tests/data/mock_manifests/TestManifestValidation_test_manifest_validation_basic_valid.csv @@ -10,4 +10,4 @@ Patient ID,Sex,Year of Birth,Diagnosis,Component,Cancer Type,Family History 9,Male,,Cancer,Patient,Colorectal,"Breast,Colorectal" 10,Male,,Cancer,Patient,Colorectal,"Breast,Colorectal" 11,Male,,Cancer,Patient,Prostate,"Breast,Colorectal" -12,Male,,Cancer,Patient,Skin,"Breast,Colorectal" \ No newline at end of file +12,Male,,Cancer,Patient,Skin,"Breast,Colorectal" diff --git a/tests/data/mock_manifests/Valid_Test_Manifest.csv b/tests/data/mock_manifests/Valid_Test_Manifest.csv index f47b4364c..a9fe98876 100644 --- a/tests/data/mock_manifests/Valid_Test_Manifest.csv +++ b/tests/data/mock_manifests/Valid_Test_Manifest.csv @@ -2,4 +2,4 @@ Component,Check List,Check List Enum,Check List Like,Check List Like Enum,Check MockComponent,"ab,cd","ab,cd",ab,ab,"ab,cd","ab,cd","a,c,f",a,"a,c,f",a,a,0,6,99.65,7,valid,https://www.google.com/,1738,1738,8085,8085,1911,334,,6571,str1,75,10/21/2022,Not Applicable MockComponent,"ab,cd","ab,cd",cd,cd,"ab,cd","ab,cd","a,c,f","a,c,f","a,c,f",e,b,1234,71,58.4,3,valid,https://www.google.com/,9965,9965,9965,9965,1655,717,,6571,str2,80,October 21 2022,8 MockComponent,"ab,cd","ab, cd","cd,ab",ab,"ab,cd","ab,cd","a,b",c,"a,b",b,c,683902,6.5,62.3,2,valid,https://www.google.com/,8085,8085,1738,1738,1427,206,present,32849,str3,95,10/21/2022,Not Applicable -MockComponent,"ab,cd","ab,cd","ab,cd","ab,cd","ab,cd","ab,cd","b,d,f","b,d,f","b,d,f",b,c,0,6.5,62.3,2,valid,https://www.google.com/,79,79,7,7,6635,608,,32849,str4,55,21/10/2022,695 \ No newline at end of file +MockComponent,"ab,cd","ab,cd","ab,cd","ab,cd","ab,cd","ab,cd","b,d,f","b,d,f","b,d,f",b,c,0,6.5,62.3,2,valid,https://www.google.com/,79,79,7,7,6635,608,,32849,str4,55,21/10/2022,695 diff --git a/tests/data/mock_manifests/Valid_Test_Manifest_with_nones.csv b/tests/data/mock_manifests/Valid_Test_Manifest_with_nones.csv index 0bbc8c71f..e64f7baee 100644 --- a/tests/data/mock_manifests/Valid_Test_Manifest_with_nones.csv +++ b/tests/data/mock_manifests/Valid_Test_Manifest_with_nones.csv @@ -2,4 +2,4 @@ Component,Check List,Check List Enum,Check List Like,Check List Like Enum,Check MockComponent,"ab,cd","ab,cd",ab,ab,"ab,cd","ab,cd","a,c,f",a,"a,c,f",a,a,,6,99.65,7,valid,https://www.google.com/,1738,1738,8085,8085,,6571,str1,75,10/21/2022,Not Applicable MockComponent,"ab,cd","ab,cd",,,"ab,cd","ab,cd","a,c,f","a,c,f","a,c,f",,b,1234,71,58.4,3,valid,https://www.google.com/,9965,9965,9965,9965,,6571,str2,80,October 21 2022,8 MockComponent,,,,,,,,,,b,,683902,,,,,,,,,,present,,,,, -MockComponent,"ab,cd","ab,cd","ab,cd","ab,cd","ab,cd","ab,cd","b,d,f","b,d,f","b,d,f",b,c,0,6.5,62.3,2,valid,https://www.google.com/,79,79,7,7,,32849,str4,55,21/10/2022,695 \ No newline at end of file +MockComponent,"ab,cd","ab,cd","ab,cd","ab,cd","ab,cd","ab,cd","b,d,f","b,d,f","b,d,f",b,c,0,6.5,62.3,2,valid,https://www.google.com/,79,79,7,7,,32849,str4,55,21/10/2022,695 diff --git a/tests/data/mock_manifests/Valid_none_value_test_manifest.csv b/tests/data/mock_manifests/Valid_none_value_test_manifest.csv index b7a0666d6..f3c009e47 100644 --- a/tests/data/mock_manifests/Valid_none_value_test_manifest.csv +++ b/tests/data/mock_manifests/Valid_none_value_test_manifest.csv @@ -3,4 +3,4 @@ Sample ID,Patient ID,Tissue Status,Component 2,2,Malignant,Biospecimen 3,3,None,Biospecimen 4,4,None,Biospecimen -5,5,None,Biospecimen \ No newline at end of file +5,5,None,Biospecimen diff --git a/tests/data/mock_manifests/example.biospecimen_component_rule.manifest.csv b/tests/data/mock_manifests/example.biospecimen_component_rule.manifest.csv index 8ac07cd1d..ce6688a56 100644 --- a/tests/data/mock_manifests/example.biospecimen_component_rule.manifest.csv +++ b/tests/data/mock_manifests/example.biospecimen_component_rule.manifest.csv @@ -1,4 +1,4 @@ Sample ID,Patient ID,Tissue Status,Component 123,123,Healthy,Biospecimen 456,123,Healthy,Biospecimen -789,syn465,Healthy,Biospecimen \ No newline at end of file +789,syn465,Healthy,Biospecimen diff --git a/tests/data/mock_manifests/example.patient_component_rule.manifest.csv b/tests/data/mock_manifests/example.patient_component_rule.manifest.csv index d0d26035b..f4a0b0e76 100644 --- a/tests/data/mock_manifests/example.patient_component_rule.manifest.csv +++ b/tests/data/mock_manifests/example.patient_component_rule.manifest.csv @@ -1,4 +1,4 @@ Patient ID,Sex,Year of Birth,Diagnosis,Component,Cancer Type,Family History 123,Female,,Cancer,Patient,Breast,"Breast, Colorectal" 123,Male,,Healthy,Patient,,"Breast, Colorectal" -789,Other,,Healthy,Patient,,"Breast, Colorectal" \ No newline at end of file +789,Other,,Healthy,Patient,,"Breast, Colorectal" diff --git a/tests/data/mock_manifests/example_biospecimen_test.csv b/tests/data/mock_manifests/example_biospecimen_test.csv index b8197a6ce..b6e820ef3 100644 --- a/tests/data/mock_manifests/example_biospecimen_test.csv +++ b/tests/data/mock_manifests/example_biospecimen_test.csv @@ -1,2 +1,2 @@ Sample ID,Patient ID,Tissue Status,Component -123,1,Healthy,Biospecimen \ No newline at end of file +123,1,Healthy,Biospecimen diff --git a/tests/data/mock_manifests/mock_example_biospecimen_manifest.csv b/tests/data/mock_manifests/mock_example_biospecimen_manifest.csv index c4926b694..6da4ddfa3 100644 --- a/tests/data/mock_manifests/mock_example_biospecimen_manifest.csv +++ b/tests/data/mock_manifests/mock_example_biospecimen_manifest.csv @@ -1,2 +1,2 @@ Sample ID,Patient ID,Tissue Status,Component -1,1,Healthy,Biospecimen \ No newline at end of file +1,1,Healthy,Biospecimen diff --git a/tests/data/test_configs/invalid_config2.yml b/tests/data/test_configs/invalid_config2.yml index e2f710b48..142a4bb5d 100644 --- a/tests/data/test_configs/invalid_config2.yml +++ b/tests/data/test_configs/invalid_config2.yml @@ -1,3 +1,3 @@ # This is a invalid config, but has fields in asset store that are not supported asset_store: - invalid_field: "xxx" \ No newline at end of file + invalid_field: "xxx" diff --git a/tests/data/test_configs/invalid_config3.yml b/tests/data/test_configs/invalid_config3.yml index 7cb2bc0f8..a322d9c90 100644 --- a/tests/data/test_configs/invalid_config3.yml +++ b/tests/data/test_configs/invalid_config3.yml @@ -2,4 +2,3 @@ asset_store: synapse: invalid_field: "xxx" - diff --git a/tests/data/validator_dag_test.model.csv b/tests/data/validator_dag_test.model.csv index 3184250ee..3a81f91be 100644 --- a/tests/data/validator_dag_test.model.csv +++ b/tests/data/validator_dag_test.model.csv @@ -19,7 +19,7 @@ CSV/TSV,,,Genome Build,,FALSE,ValidValue,,, Genome Build,,"GRCh37, GRCh38, GRCm38, GRCm39",,,TRUE,DataProperty,,, Genome FASTA,,,,,TRUE,DataProperty,,, MockComponent,,,"Component, Check List, Check Regex List, Check Regex Single, Check Regex Format, Check Regex Integer, Check Num, Check Float, Check Int, Check String, Check URL,Check Match at Least, Check Match at Least values, Check Match Exactly, Check Match Exactly values, Check Recommended, Check Ages, Check Unique, Check Range, Check Date, Check NA",,FALSE,DataType,,, -Check List,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list strict +Check List,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list strict Check Regex List,,,,,TRUE,DataProperty,,,list strict::regex match [a-f] Check Regex Single,,,,,TRUE,DataProperty,,,regex search [a-f] Check Regex Format,,,,,TRUE,DataProperty,,,regex match [a-f] diff --git a/tests/data/validator_test.model.csv b/tests/data/validator_test.model.csv index b5b84760f..bf901006b 100644 --- a/tests/data/validator_test.model.csv +++ b/tests/data/validator_test.model.csv @@ -19,7 +19,7 @@ CSV/TSV,,,Genome Build,,FALSE,ValidValue,,, Genome Build,,"GRCh37, GRCh38, GRCm38, GRCm39",,,TRUE,DataProperty,,, Genome FASTA,,,,,TRUE,DataProperty,,, MockComponent,,,"Component, Check List, Check Regex List, Check Regex Single, Check Regex Format, Check Regex Integer, Check Num, Check Float, Check Int, Check String, Check URL,Check Match at Least, Check Match at Least values, Check Match Exactly, Check Match Exactly values, Check Recommended, Check Ages, Check Unique, Check Range, Check Date, Check NA",,FALSE,DataType,,, -Check List,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list strict +Check List,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list strict Check Regex List,,,,,TRUE,DataProperty,,,list strict::regex match [a-f] Check Regex Single,,,,,TRUE,DataProperty,,,regex search [a-f] Check Regex Format,,,,,TRUE,DataProperty,,,regex match [a-f] @@ -42,4 +42,4 @@ Check NA,,,,,TRUE,DataProperty,,,int::IsNA MockRDB,,,"Component, MockRDB_id, SourceManifest",,FALSE,DataType,,, MockRDB_id,,,,,TRUE,DataProperty,,,int SourceManifest,,,,,TRUE,DataProperty,,, -entityId,,,,,TRUE,DataProperty,,, \ No newline at end of file +entityId,,,,,TRUE,DataProperty,,, diff --git a/tests/integration/test_manifest_generation.py b/tests/integration/test_manifest_generation.py index 2fe8c4896..6b7290d5b 100644 --- a/tests/integration/test_manifest_generation.py +++ b/tests/integration/test_manifest_generation.py @@ -13,8 +13,8 @@ from flask.testing import FlaskClient from openpyxl import load_workbook -from tests.conftest import ConfigurationForTesting from schematic.configuration.configuration import CONFIG +from tests.conftest import ConfigurationForTesting LIGHT_BLUE = "FFEAF7F9" # Required cell GRAY = "FFE0E0E0" # Header cell diff --git a/tests/integration/test_manifest_validation.py b/tests/integration/test_manifest_validation.py index 67d4fa9d1..667e7156b 100644 --- a/tests/integration/test_manifest_validation.py +++ b/tests/integration/test_manifest_validation.py @@ -10,8 +10,8 @@ import requests from flask.testing import FlaskClient -from tests.conftest import ConfigurationForTesting, Helpers from schematic.configuration.configuration import CONFIG +from tests.conftest import ConfigurationForTesting, Helpers EXAMPLE_SCHEMA_URL = "https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.model.jsonld" diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 7a27c7a34..493ce2c35 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1,18 +1,19 @@ """Testing for Configuration module""" import os + import pytest from pydantic import ValidationError +from schematic.configuration.configuration import ( + ConfigNonAllowedFieldError, + Configuration, +) from schematic.configuration.dataclasses import ( - SynapseConfig, + GoogleSheetsConfig, ManifestConfig, ModelConfig, - GoogleSheetsConfig, -) -from schematic.configuration.configuration import ( - Configuration, - ConfigNonAllowedFieldError, + SynapseConfig, ) diff --git a/tests/unit/test_data_model_validator.py b/tests/unit/test_data_model_validator.py index 33ea63575..e7550f37e 100644 --- a/tests/unit/test_data_model_validator.py +++ b/tests/unit/test_data_model_validator.py @@ -1,23 +1,23 @@ """Unit testing for the ValidateAttribute class""" -from typing import Generator, Any, Tuple, Iterable -import pytest +from typing import Any, Generator, Iterable, Tuple +import pytest from networkx import MultiDiGraph # type: ignore +from schematic.schemas.data_model_graph import DataModelGraph +from schematic.schemas.data_model_parser import DataModelParser from schematic.schemas.data_model_validator import ( DataModelValidator, Node, - get_node_labels_from, - get_missing_fields_from, check_characters_in_node_display_name, - match_node_names_with_reserved_names, - create_reserve_name_error_messages, create_blacklisted_characters_error_message, create_missing_fields_error_messages, + create_reserve_name_error_messages, + get_missing_fields_from, + get_node_labels_from, + match_node_names_with_reserved_names, ) -from schematic.schemas.data_model_parser import DataModelParser -from schematic.schemas.data_model_graph import DataModelGraph # pylint: disable=protected-access diff --git a/tests/unit/test_validate_attribute.py b/tests/unit/test_validate_attribute.py index 4e05ddc98..d25ecf7a1 100644 --- a/tests/unit/test_validate_attribute.py +++ b/tests/unit/test_validate_attribute.py @@ -3,10 +3,10 @@ from typing import Generator from unittest.mock import Mock, patch -from jsonschema import ValidationError -from pandas import Series, DataFrame, concat import numpy as np import pytest +from jsonschema import ValidationError +from pandas import DataFrame, Series, concat import schematic.models.validate_attribute from schematic.models.validate_attribute import GenerateError, ValidateAttribute diff --git a/uwsgi-nginx-entrypoint.sh b/uwsgi-nginx-entrypoint.sh index 722d49941..f90ed7532 100644 --- a/uwsgi-nginx-entrypoint.sh +++ b/uwsgi-nginx-entrypoint.sh @@ -54,4 +54,4 @@ if [ -n "$ALPINEPYTHON" ] ; then export PYTHONPATH=$PYTHONPATH:/usr/local/lib/$ALPINEPYTHON/site-packages:/usr/lib/$ALPINEPYTHON/site-packages fi -exec "$@" \ No newline at end of file +exec "$@" diff --git a/uwsgi.ini b/uwsgi.ini index 50b5b14d6..95402dec6 100644 --- a/uwsgi.ini +++ b/uwsgi.ini @@ -22,4 +22,4 @@ buffer-size = 32768 # for dealing with long token in DCA and DFA # for dealing with OSError: write error ignore-sigpipe=true ignore-write-errors=true -disable-write-exception=true \ No newline at end of file +disable-write-exception=true