diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 413047d..6c665b3 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -26,30 +26,19 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' # caching pip dependencies - - name: Cache test data - id: cache_test_data - uses: actions/cache@v3 - with: - path: | - tests/data - data - key: venv-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/tests/scripts/fetch_test_data.sh') }} - - name: Install dependencies run: | python -m pip install --upgrade pip pip install . pip install pytest omegaconf - + - name: Get test data from OSF - if: ${{ steps.cache_test_data.outputs.cache-hit != 'true' }} run: | sh tests/scripts/fetch_test_data.sh - + - name: Test with pytest run: | pytest tests/test_preprocessing.py pytest tests/test_svd.py pytest tests/test_map_to_map.py pytest tests/test_distribution_to_distribution.py - diff --git a/config_files/config_preproc.yaml b/config_files/config_preproc.yaml index 4c503cc..0a087c9 100644 --- a/config_files/config_preproc.yaml +++ b/config_files/config_preproc.yaml @@ -1,5 +1,4 @@ submission_config_file: submission_config.json -seed_flavor_assignment: 0 thresh_percentile: 93.0 BOT_box_size: 32 BOT_loss: wemd diff --git a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py index 0fddb0b..0e6d330 100644 --- a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py +++ b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py @@ -1,5 +1,4 @@ import torch -import numpy as np import json import os @@ -40,10 +39,7 @@ def save_submission(volumes, populations, submission_id, submission_index, confi def preprocess_submissions(submission_dataset, config): - - n_subs = max(submission_dataset.subs_index) + 1 hash_table = {} - box_size_gt = submission_dataset.submission_config["gt"]["box_size"] pixel_size_gt = submission_dataset.submission_config["gt"]["pixel_size"] vol_gt_ref = submission_dataset.vol_gt_ref @@ -51,9 +47,12 @@ def preprocess_submissions(submission_dataset, config): for i in range(len(submission_dataset)): idx = submission_dataset.subs_index[i] - hash_table[submission_dataset.submission_config[str(idx)]["name"]] = ( - submission_dataset.submission_config[str(idx)]["flavor_name"] + " " + str(submission_dataset.submission_config[str(idx)]["submission_version"]) - ) + sub_flavor = submission_dataset.submission_config[str(idx)]["flavor_name"] + " " + str(submission_dataset.submission_config[str(idx)]["submission_version"]) + sub_name = submission_dataset.submission_config[str(idx)]["name"] + hash_table[sub_flavor] = { + "name": sub_name, + "filename": f"submission_{idx}.pt", + } print(f"Preprocessing submission {idx}...") @@ -103,7 +102,10 @@ def preprocess_submissions(submission_dataset, config): else: submission_version = f" {submission_version}" print(f" SUBMISSION VERSION {submission_version}") - submission_id = submission_dataset.submission_config[str(idx)]["flavor_name"] + submission_version + submission_id = ( + submission_dataset.submission_config[str(idx)]["flavor_name"] + + submission_version + ) print(f"SUBMISSION ID {submission_id}") save_submission( diff --git a/src/cryo_challenge/data/_validation/config_validators.py b/src/cryo_challenge/data/_validation/config_validators.py index 93316a0..b2fa933 100644 --- a/src/cryo_challenge/data/_validation/config_validators.py +++ b/src/cryo_challenge/data/_validation/config_validators.py @@ -1,7 +1,7 @@ from numbers import Number import pandas as pd import os -from typing import List + def validate_generic_config(config: dict, reference: dict) -> None: """ @@ -48,7 +48,6 @@ def validate_config_preprocessing(config_data: dict) -> None: "BOT_loss": str, "BOT_iter": Number, "BOT_refine": bool, - "seed_flavor_assignment": int, } validate_generic_config(config_data, keys_and_types) return diff --git a/tests/config_files/test_config_preproc.yaml b/tests/config_files/test_config_preproc.yaml index 0a27d7e..0d0fec5 100644 --- a/tests/config_files/test_config_preproc.yaml +++ b/tests/config_files/test_config_preproc.yaml @@ -1,5 +1,4 @@ submission_config_file: tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json -seed_flavor_assignment: 0 thresh_percentile: 93.0 BOT_box_size: 32 BOT_loss: wemd