From 0fcc4ed91297796699c4914183e6ea8e5d3dafeb Mon Sep 17 00:00:00 2001 From: Miro Astore Date: Thu, 8 Aug 2024 18:02:34 -0400 Subject: [PATCH 1/6] added flavor keys --- .../_preprocessing/preprocessing_pipeline.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py index 589239c..a9201b7 100644 --- a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py +++ b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py @@ -65,7 +65,7 @@ def preprocess_submissions(submission_dataset, config): ] n_subs = max(submission_dataset.subs_index) + 1 - random_mapping = np.random.choice(len(ice_cream_flavors), n_subs, replace=False) + random_mapping = np.random.choice(len(ice_cream_flavors), len(ice_cream_flavors), replace=False) hash_table = {} box_size_gt = submission_dataset.submission_config["gt"]["box_size"] @@ -76,8 +76,9 @@ def preprocess_submissions(submission_dataset, config): idx = submission_dataset.subs_index[i] hash_table[submission_dataset.submission_config[str(idx)]["name"]] = ( - ice_cream_flavors[random_mapping[idx]] + ice_cream_flavors[submission_dataset.submission_config[str(idx)]["flavor_key"]] ) + print(submission_dataset.submission_config[str(idx)]["flavor_key"]) print(f"Preprocessing submission {idx}...") @@ -126,8 +127,8 @@ def preprocess_submissions(submission_dataset, config): submission_version = "" else: submission_version = f" {submission_version}" - print(f" SUBMISSIION VERSION {submission_version}") - submission_id = ice_cream_flavors[random_mapping[idx]] + submission_version + print(f" SUBMISSION VERSION {submission_version}") + submission_id = ice_cream_flavors[random_mapping[submission_dataset.submission_config[str(idx)]["flavor_key"]]] + submission_version print(f"SUBMISSION ID {submission_id}") save_submission( From ea60e55f87638458c3d0e991d05ea9e63e99b5ec Mon Sep 17 00:00:00 2001 From: Miro Astore Date: Thu, 8 Aug 2024 18:08:42 -0400 Subject: [PATCH 2/6] added flavor_key to test json --- .../submission_x/submission_config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json b/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json index 5671514..e7c96e2 100644 --- a/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json +++ b/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json @@ -9,6 +9,7 @@ "0": { "name": "raw_submission_in_testdata", "align": 1, + "flavor_key": 1, "box_size": 244, "pixel_size": 2.146, "path": "tests/data/unprocessed_dataset_2_submissions/submission_x", From a5f84dd87d6ae23c81e0f31ba066dd2d4ef57bc0 Mon Sep 17 00:00:00 2001 From: Miro Astore Date: Fri, 9 Aug 2024 10:55:38 -0400 Subject: [PATCH 3/6] added direct naming of flavors from json file --- .../_preprocessing/preprocessing_pipeline.py | 29 ++----------------- .../submission_x/submission_config.json | 2 +- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py index a9201b7..2646b39 100644 --- a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py +++ b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py @@ -40,32 +40,8 @@ def save_submission(volumes, populations, submission_id, submission_index, confi def preprocess_submissions(submission_dataset, config): - np.random.seed(config["seed_flavor_assignment"]) - ice_cream_flavors = [ - "Chocolate", - "Vanilla", - "Cookies N' Cream", - "Mint Chocolate Chip", - "Strawberry", - "Butter Pecan", - "Salted Caramel", - "Pistachio", - "Rocky Road", - "Coffee", - "Cookie Dough", - "Chocolate Chip", - "Neapolitan", - "Cherry", - "Rainbow Sherbet", - "Peanut Butter", - "Cotton Candy", - "Lemon Sorbet", - "Mango", - "Black Raspberry", - ] n_subs = max(submission_dataset.subs_index) + 1 - random_mapping = np.random.choice(len(ice_cream_flavors), len(ice_cream_flavors), replace=False) hash_table = {} box_size_gt = submission_dataset.submission_config["gt"]["box_size"] @@ -76,9 +52,8 @@ def preprocess_submissions(submission_dataset, config): idx = submission_dataset.subs_index[i] hash_table[submission_dataset.submission_config[str(idx)]["name"]] = ( - ice_cream_flavors[submission_dataset.submission_config[str(idx)]["flavor_key"]] + submission_dataset.submission_config[str(idx)]["flavor_name"] ) - print(submission_dataset.submission_config[str(idx)]["flavor_key"]) print(f"Preprocessing submission {idx}...") @@ -128,7 +103,7 @@ def preprocess_submissions(submission_dataset, config): else: submission_version = f" {submission_version}" print(f" SUBMISSION VERSION {submission_version}") - submission_id = ice_cream_flavors[random_mapping[submission_dataset.submission_config[str(idx)]["flavor_key"]]] + submission_version + submission_id = submission_dataset.submission_config[str(idx)]["flavor_name"] + " " + submission_version print(f"SUBMISSION ID {submission_id}") save_submission( diff --git a/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json b/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json index e7c96e2..b8318b9 100644 --- a/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json +++ b/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json @@ -9,7 +9,7 @@ "0": { "name": "raw_submission_in_testdata", "align": 1, - "flavor_key": 1, + "flavor_name": "test flavor", "box_size": 244, "pixel_size": 2.146, "path": "tests/data/unprocessed_dataset_2_submissions/submission_x", From 13a94c0f62edf0e78ad0a810ff40104e5f9580da Mon Sep 17 00:00:00 2001 From: Miro Astore Date: Fri, 9 Aug 2024 11:34:12 -0400 Subject: [PATCH 4/6] removed ice cream dict and random assignment in favor of direct assignment in json config file for ice cream falvors --- src/cryo_challenge/_preprocessing/preprocessing_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py index 2646b39..718c433 100644 --- a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py +++ b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py @@ -103,7 +103,7 @@ def preprocess_submissions(submission_dataset, config): else: submission_version = f" {submission_version}" print(f" SUBMISSION VERSION {submission_version}") - submission_id = submission_dataset.submission_config[str(idx)]["flavor_name"] + " " + submission_version + submission_id = submission_dataset.submission_config[str(idx)]["flavor_name"] + submission_version print(f"SUBMISSION ID {submission_id}") save_submission( From 3008c779248ef3b9a0b699f93f0d53b28d17c936 Mon Sep 17 00:00:00 2001 From: DSilva27 Date: Fri, 9 Aug 2024 12:44:53 -0400 Subject: [PATCH 5/6] edit how the hash table is saved --- .../_preprocessing/preprocessing_pipeline.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py index 718c433..90ccc51 100644 --- a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py +++ b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py @@ -1,5 +1,4 @@ import torch -import numpy as np import json import os @@ -40,10 +39,7 @@ def save_submission(volumes, populations, submission_id, submission_index, confi def preprocess_submissions(submission_dataset, config): - - n_subs = max(submission_dataset.subs_index) + 1 hash_table = {} - box_size_gt = submission_dataset.submission_config["gt"]["box_size"] pixel_size_gt = submission_dataset.submission_config["gt"]["pixel_size"] vol_gt_ref = submission_dataset.vol_gt_ref @@ -51,9 +47,12 @@ def preprocess_submissions(submission_dataset, config): for i in range(len(submission_dataset)): idx = submission_dataset.subs_index[i] - hash_table[submission_dataset.submission_config[str(idx)]["name"]] = ( - submission_dataset.submission_config[str(idx)]["flavor_name"] - ) + sub_flavor = submission_dataset.submission_config[str(idx)]["flavor_name"] + sub_name = submission_dataset.submission_config[str(idx)]["name"] + hash_table[sub_flavor] = { + "name": sub_name, + "filename": f"submission_{idx}.pt", + } print(f"Preprocessing submission {idx}...") @@ -103,7 +102,10 @@ def preprocess_submissions(submission_dataset, config): else: submission_version = f" {submission_version}" print(f" SUBMISSION VERSION {submission_version}") - submission_id = submission_dataset.submission_config[str(idx)]["flavor_name"] + submission_version + submission_id = ( + submission_dataset.submission_config[str(idx)]["flavor_name"] + + submission_version + ) print(f"SUBMISSION ID {submission_id}") save_submission( From 3e8388eef6f45cbd889d2ab0525eda2e54caf28b Mon Sep 17 00:00:00 2001 From: DSilva27 Date: Fri, 9 Aug 2024 13:03:14 -0400 Subject: [PATCH 6/6] remove seed assignment from config files --- config_files/config_preproc.yaml | 1 - src/cryo_challenge/data/_validation/config_validators.py | 3 +-- tests/config_files/test_config_preproc.yaml | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/config_files/config_preproc.yaml b/config_files/config_preproc.yaml index 4c503cc..0a087c9 100644 --- a/config_files/config_preproc.yaml +++ b/config_files/config_preproc.yaml @@ -1,5 +1,4 @@ submission_config_file: submission_config.json -seed_flavor_assignment: 0 thresh_percentile: 93.0 BOT_box_size: 32 BOT_loss: wemd diff --git a/src/cryo_challenge/data/_validation/config_validators.py b/src/cryo_challenge/data/_validation/config_validators.py index 93316a0..b2fa933 100644 --- a/src/cryo_challenge/data/_validation/config_validators.py +++ b/src/cryo_challenge/data/_validation/config_validators.py @@ -1,7 +1,7 @@ from numbers import Number import pandas as pd import os -from typing import List + def validate_generic_config(config: dict, reference: dict) -> None: """ @@ -48,7 +48,6 @@ def validate_config_preprocessing(config_data: dict) -> None: "BOT_loss": str, "BOT_iter": Number, "BOT_refine": bool, - "seed_flavor_assignment": int, } validate_generic_config(config_data, keys_and_types) return diff --git a/tests/config_files/test_config_preproc.yaml b/tests/config_files/test_config_preproc.yaml index 0a27d7e..0d0fec5 100644 --- a/tests/config_files/test_config_preproc.yaml +++ b/tests/config_files/test_config_preproc.yaml @@ -1,5 +1,4 @@ submission_config_file: tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json -seed_flavor_assignment: 0 thresh_percentile: 93.0 BOT_box_size: 32 BOT_loss: wemd