diff --git a/.gitignore b/.gitignore index 68bc17f..5ede44a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,15 @@ +# downloaded data +data/dataset_2_submissions +data/dataset_1_submissions +data/dataset_2_ground_truth + +# data for testing and resulting outputs +tests/data/Ground_truth +tests/data/dataset_2_submissions/ +tests/data/unprocessed_dataset_2_submissions/submission_x/ +tests/results/ + + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README.md b/README.md index 9dfb755..48a15a2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

Cryo-EM Heterogeneity Challenge

- + Supported Python versions GitHub Downloads (all assets, all releases) GitHub branch check runs @@ -10,13 +10,13 @@

- + Cryo-EM Heterogeneity Challenge

- + This repository contains the code used to analyse the submissions for the [Inaugural Flatiron Cryo-EM Heterogeneity Challenge](https://www.simonsfoundation.org/flatiron/center-for-computational-biology/structural-and-molecular-biophysics-collaboration/heterogeneity-in-cryo-electron-microscopy/). # Scope @@ -26,13 +26,13 @@ This repository explains how to preprocess a submission (80 maps and correspondi This is a work in progress, while the code will probably not change, we are still writting better tutorials, documentation, and other ideas for analyzing the data. We are also in the process of making it easier for other people to contribute with their own metrics and methods. We are also in the process of distributing the code to PyPi. # Accesing the data -The data is available via the Open Science Foundation project [The Inaugural Flatiron Institute Cryo-EM Heterogeneity Community Challenge](https://osf.io/8h6fz/). You can download via a web browser, or programatically with wget as per [this script](https://github.com/flatironinstitute/Cryo-EM-Heterogeneity-Challenge-1/blob/main/tests/scripts/fetch_test_data.sh). +The data is available via the Open Science Foundation project [The Inaugural Flatiron Institute Cryo-EM Heterogeneity Community Challenge](https://osf.io/8h6fz/). You can download via a web browser, or programatically with wget as per [this script](https://github.com/flatironinstitute/Cryo-EM-Heterogeneity-Challenge-1/blob/main/data/fetch_data.sh). **_NOTE_**: We recommend downloadaing the data with the script and wget as the downloads from the web browser might be unstable. # Installation -## Stable installation +## Stable installation Installing this repository is simply. We recommend creating a virtual environment (using conda or pyenv), since we have dependencies such as PyTorch or Aspire, which are better dealt with in an isolated environment. After creating your environment, make sure to activate it and run ```bash @@ -63,7 +63,7 @@ pytest tests/test_distribution_to_distribution.py If you want to run our code on the full challenge data, or you own local data, please complete the following steps ### 1. Download the full challenge data from [The Inaugural Flatiron Institute Cryo-EM Heterogeneity Community Challenge](https://osf.io/8h6fz/) -You can do this through the web browser, or programatically with wget (you can get inspiration from [this script](https://github.com/flatironinstitute/Cryo-EM-Heterogeneity-Challenge-1/blob/main/tests/scripts/fetch_test_data.sh), which is just for the test data, not the full datasets) +You can do this through the web browser, or programatically with wget (you can use [this script](https://github.com/flatironinstitute/Cryo-EM-Heterogeneity-Challenge-1/blob/main/data/fetch_data.sh, this will download around 220 GB of data) ### 2. Modify the config files and run the commands on the full challenge data Point to the path where the data is locally diff --git a/data/fetch_data.sh b/data/fetch_data.sh new file mode 100644 index 0000000..54ecb0d --- /dev/null +++ b/data/fetch_data.sh @@ -0,0 +1,21 @@ +mkdir -p data/dataset_2_submissions data/dataset_1_submissions data/dataset_2_ground_truth + +# dataset 1 submissions +for i in {0..10} +do + wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/dataset_1_submissions/submission_${i}.pt?download=true -O data/dataset_1_submissions/submission_${i}.pt +done + +# dataset 2 submissions +for i in {0..11} +do + wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/dataset_2_submissions/submission_${i}.pt?download=true -O data/dataset_2_submissions/submission_${i}.pt +done + +# ground truth + +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/Ground_truth/maps_gt_flat.pt?download=true -O data/dataset_2_ground_truth/maps_gt_flat.pt + +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/Ground_truth/metadata.csv?download=true -O data/dataset_2_ground_truth/metadata.csv + +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/Ground_truth/mask_dilated_wide_224x224.mrc?download=true -O data/dataset_2_ground_truth/mask_dilated_wide_224x224.mrc diff --git a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py index 5fc3db9..5994e6e 100644 --- a/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py +++ b/src/cryo_challenge/_preprocessing/preprocessing_pipeline.py @@ -124,7 +124,10 @@ def preprocess_submissions(submission_dataset, config): print(f" submission saved as submission_{idx}.pt") print(f"Preprocessing submission {idx} complete") - with open("hash_table.json", "w") as f: + hash_table_path = os.path.join( + config["output_path"], "submission_to_icecream_table.json" + ) + with open(hash_table_path, "w") as f: json.dump(hash_table, f, indent=4) return diff --git a/tests/config_files/test_config_distribution_to_distribution.yaml b/tests/config_files/test_config_distribution_to_distribution.yaml index a1f03af..05b6317 100644 --- a/tests/config_files/test_config_distribution_to_distribution.yaml +++ b/tests/config_files/test_config_distribution_to_distribution.yaml @@ -9,4 +9,4 @@ cvxpy_solver: ECOS optimal_q_kl: n_iter: 100000 break_atol: 0.0001 -output_fname: results/test_distribution_to_distribution_submission_0.pkl \ No newline at end of file +output_fname: tests/results/test_distribution_to_distribution_submission_0.pkl diff --git a/tests/config_files/test_config_map_to_map.yaml b/tests/config_files/test_config_map_to_map.yaml index 85d663d..c70d5aa 100644 --- a/tests/config_files/test_config_map_to_map.yaml +++ b/tests/config_files/test_config_map_to_map.yaml @@ -1,17 +1,17 @@ data: n_pix: 224 - psize: 2.146 + psize: 2.146 submission: fname: tests/data/dataset_2_submissions/test_submission_0_n8.pt volume_key: volumes metadata_key: populations label_key: id ground_truth: - volumes: tests/data/Ground_truth/test_maps_gt_flat_10.pt - metadata: tests/data/Ground_truth/test_metadata_10.csv - mask: + volumes: tests/data/Ground_truth/test_maps_gt_flat_10.pt + metadata: tests/data/Ground_truth/test_metadata_10.csv + mask: do: true - volume: data/Ground_truth/mask_dilated_wide_224x224.mrc + volume: tests/data/Ground_truth/mask_dilated_wide_224x224.mrc analysis: metrics: - l2 @@ -20,4 +20,4 @@ analysis: normalize: do: true method: median_zscore -output: tests/results/test_map_to_map_distance_matrix_submission_0.pkl \ No newline at end of file +output: tests/results/test_map_to_map_distance_matrix_submission_0.pkl diff --git a/tests/scripts/fetch_test_data.sh b/tests/scripts/fetch_test_data.sh index 5b58f23..c252871 100644 --- a/tests/scripts/fetch_test_data.sh +++ b/tests/scripts/fetch_test_data.sh @@ -1,11 +1,11 @@ -mkdir -p tests/data/dataset_2_submissions data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x tests/data/Ground_truth/ data/Ground_truth +mkdir -p tests/data/dataset_2_submissions tests/data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x tests/data/Ground_truth/ tests/data/Ground_truth wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/dataset_2_submissions/test_submission_0_n8.pt?download=true -O tests/data/dataset_2_submissions/test_submission_0_n8.pt ADIR=$(pwd) ln -s $ADIR/tests/data/dataset_2_submissions/test_submission_0_n8.pt $ADIR/tests/data/dataset_2_submissions/submission_0.pt # symlink for svd which needs submission_0.pt for filename wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_maps_gt_flat_10.pt?download=true -O tests/data/Ground_truth/test_maps_gt_flat_10.pt -wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_metadata_10.csv?download=true -O tests/data/Ground_truth/test_metadata_10.csv -wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/1.mrc?download=true -O tests/data/Ground_truth/1.mrc -wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/Ground_truth/mask_dilated_wide_224x224.mrc?download=true -O data/Ground_truth/mask_dilated_wide_224x224.mrc +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_metadata_10.csv?download=true -O tests/data/Ground_truth/test_metadata_10.csv +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/1.mrc?download=true -O tests/data/Ground_truth/1.mrc +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/Ground_truth/mask_dilated_wide_224x224.mrc?download=true -O tests/data/Ground_truth/mask_dilated_wide_224x224.mrc for FILE in 1.mrc 2.mrc 3.mrc 4.mrc populations.txt do wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/unprocessed_dataset_2_submissions/submission_x/${FILE}?download=true -O tests/data/unprocessed_dataset_2_submissions/submission_x/${FILE}