merged dev changes and added versions to flavor names in hash table

flatironinstitute · Aug 9, 2024 · 7f231bf · 7f231bf
2 parents 8191486 + 4a24b6d
commit 7f231bf
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 25 deletions.
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -26,30 +26,19 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: 'pip' # caching pip dependencies
 
-      - name: Cache test data
-        id: cache_test_data
-        uses: actions/cache@v3
-        with:
-          path: |
-            tests/data
-            data
-          key: venv-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/tests/scripts/fetch_test_data.sh') }}
-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install .
           pip install pytest omegaconf
-          
+
       - name: Get test data from OSF
-        if: ${{ steps.cache_test_data.outputs.cache-hit != 'true' }}
         run: |
           sh tests/scripts/fetch_test_data.sh
-          
+
       - name: Test with pytest
         run: |
           pytest tests/test_preprocessing.py
           pytest tests/test_svd.py
           pytest tests/test_map_to_map.py
           pytest tests/test_distribution_to_distribution.py
-          
diff --git a/config_files/config_preproc.yaml b/config_files/config_preproc.yaml
@@ -1,5 +1,4 @@
 submission_config_file: submission_config.json
-seed_flavor_assignment: 0
 thresh_percentile: 93.0
 BOT_box_size: 32
 BOT_loss: wemd

diff --git a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py
@@ -1,5 +1,4 @@
 import torch
-import numpy as np
 import json
 import os
 
@@ -40,20 +39,20 @@ def save_submission(volumes, populations, submission_id, submission_index, confi
 
 
 def preprocess_submissions(submission_dataset, config):
-
-    n_subs = max(submission_dataset.subs_index) + 1
     hash_table = {}
-
     box_size_gt = submission_dataset.submission_config["gt"]["box_size"]
     pixel_size_gt = submission_dataset.submission_config["gt"]["pixel_size"]
     vol_gt_ref = submission_dataset.vol_gt_ref
 
     for i in range(len(submission_dataset)):
         idx = submission_dataset.subs_index[i]
 
-        hash_table[submission_dataset.submission_config[str(idx)]["name"]] = (
-            submission_dataset.submission_config[str(idx)]["flavor_name"]  + " " + str(submission_dataset.submission_config[str(idx)]["submission_version"])
-        )
+        sub_flavor = submission_dataset.submission_config[str(idx)]["flavor_name"] + " " + str(submission_dataset.submission_config[str(idx)]["submission_version"])
+        sub_name = submission_dataset.submission_config[str(idx)]["name"]
+        hash_table[sub_flavor] = {
+            "name": sub_name,
+            "filename": f"submission_{idx}.pt",
+        }
 
         print(f"Preprocessing submission {idx}...")
 
@@ -103,7 +102,10 @@ def preprocess_submissions(submission_dataset, config):
         else:
             submission_version = f" {submission_version}"
         print(f" SUBMISSION VERSION {submission_version}")
-        submission_id = submission_dataset.submission_config[str(idx)]["flavor_name"] + submission_version
+        submission_id = (
+            submission_dataset.submission_config[str(idx)]["flavor_name"]
+            + submission_version
+        )
         print(f"SUBMISSION ID {submission_id}")
 
         save_submission(

diff --git a/src/cryo_challenge/data/_validation/config_validators.py b/src/cryo_challenge/data/_validation/config_validators.py
@@ -1,7 +1,7 @@
 from numbers import Number
 import pandas as pd
 import os
-from typing import List
+
 
 def validate_generic_config(config: dict, reference: dict) -> None:
     """
@@ -48,7 +48,6 @@ def validate_config_preprocessing(config_data: dict) -> None:
         "BOT_loss": str,
         "BOT_iter": Number,
         "BOT_refine": bool,
-        "seed_flavor_assignment": int,
     }
     validate_generic_config(config_data, keys_and_types)
     return

diff --git a/tests/config_files/test_config_preproc.yaml b/tests/config_files/test_config_preproc.yaml
@@ -1,5 +1,4 @@
 submission_config_file: tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json
-seed_flavor_assignment: 0
 thresh_percentile: 93.0
 BOT_box_size: 32
 BOT_loss: wemd