From 50396b4415b9e69699e86cd5ea7cc51b8251f5f6 Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Tue, 25 Jun 2024 16:51:31 -0400 Subject: [PATCH 01/17] configs, test data --- .github/workflows/testing.yml | 2 ++ tests/config_files/test_config_preproc.yaml | 8 ++++++++ .../submission_x/submission_config.json | 16 ++++++++++++++++ tests/scripts/fetch_test_data.sh | 7 +++++-- tests/test_preprocessing.py | 7 +++++++ 5 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 tests/config_files/test_config_preproc.yaml create mode 100644 tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json create mode 100644 tests/test_preprocessing.py diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 45538ae..130585c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -55,5 +55,7 @@ jobs: - name: Test with pytest run: | + pytest tests/test_preprocess.py pytest tests/test_map_to_map.py pytest tests/test_distribution_to_distribution.py + diff --git a/tests/config_files/test_config_preproc.yaml b/tests/config_files/test_config_preproc.yaml new file mode 100644 index 0000000..0a27d7e --- /dev/null +++ b/tests/config_files/test_config_preproc.yaml @@ -0,0 +1,8 @@ +submission_config_file: tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json +seed_flavor_assignment: 0 +thresh_percentile: 93.0 +BOT_box_size: 32 +BOT_loss: wemd +BOT_iter: 200 +BOT_refine: True +output_path: tests/results/preproccessed_submission_x/ diff --git a/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json b/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json new file mode 100644 index 0000000..87184aa --- /dev/null +++ b/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json @@ -0,0 +1,16 @@ +{ + "gt": { + "name": "gt", + "path": "tests/data/unprocessed_dataset_2_submissions/submission_x", + "box_size": 224, + "pixel_size": 2.146, + "ref_align_fname": "1.mrc" + }, + "0": { + "name": "raw_submission_in_testdata", + "align": 1, + "box_size": 244, + "pixel_size": 2.146, + "path": "tests/data/unprocessed_dataset_2_submissions/submission_x" + } +} \ No newline at end of file diff --git a/tests/scripts/fetch_test_data.sh b/tests/scripts/fetch_test_data.sh index 740d895..d90327e 100644 --- a/tests/scripts/fetch_test_data.sh +++ b/tests/scripts/fetch_test_data.sh @@ -1,5 +1,8 @@ -mkdir -p data/dataset_2_submissions tests/data/dataset_2_submissions tests/results +mkdir -p data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/dataset_2_submissions/test_submission_0_n8.pt?download=true -O tests/data/dataset_2_submissions/test_submission_0_n8.pt wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/Ground_truth/test_maps_gt_flat_10.pt?download=true -O tests/data/test_maps_gt_flat_10.pt wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/Ground_truth/test_metadata_10.csv?download=true -O tests/data/test_metadata_10.csv -wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/Ground_truth/mask_dilated_wide_224x224.mrc?download=true -O data/mask_dilated_wide_224x224.mrc \ No newline at end of file +for FILE in 1.mrc 2.mrc 3.mrc 4.mrc populations.txt +do + wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/unprocessed_dataset_2_submissions/submission_x/${FILE}?download=true -O tests/data/unprocessed_dataset_2_submissions/submission_x/${FILE} +done \ No newline at end of file diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py new file mode 100644 index 0000000..cbf54e4 --- /dev/null +++ b/tests/test_preprocessing.py @@ -0,0 +1,7 @@ +from omegaconf import OmegaConf +from cryo_challenge._commands import run_preprocessing + + +def test_run_preprocessing(): + args = OmegaConf.create({'config': 'tests/config_files/test_config_preproc.yaml'}) + run_preprocessing.main(args) \ No newline at end of file From 7d222550161453f658357084ac69ef52b42f9a4d Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Tue, 25 Jun 2024 16:58:33 -0400 Subject: [PATCH 02/17] fix name of .py test --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 130585c..4d98e66 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -55,7 +55,7 @@ jobs: - name: Test with pytest run: | - pytest tests/test_preprocess.py + pytest tests/test_preprocessing.py pytest tests/test_map_to_map.py pytest tests/test_distribution_to_distribution.py From 607d7fe0b8ca6df0dfd7cb842ae9119d31069a7f Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Tue, 25 Jun 2024 17:05:15 -0400 Subject: [PATCH 03/17] update organization in osf --- tests/scripts/fetch_test_data.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/scripts/fetch_test_data.sh b/tests/scripts/fetch_test_data.sh index d90327e..613db91 100644 --- a/tests/scripts/fetch_test_data.sh +++ b/tests/scripts/fetch_test_data.sh @@ -1,7 +1,7 @@ mkdir -p data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x -wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/dataset_2_submissions/test_submission_0_n8.pt?download=true -O tests/data/dataset_2_submissions/test_submission_0_n8.pt -wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/Ground_truth/test_maps_gt_flat_10.pt?download=true -O tests/data/test_maps_gt_flat_10.pt -wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/Ground_truth/test_metadata_10.csv?download=true -O tests/data/test_metadata_10.csv +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/dataset_2_submissions/test_submission_0_n8.pt?download=true -O tests/data/dataset_2_submissions/test_submission_0_n8.pt +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_maps_gt_flat_10.pt?download=true -O tests/data/test_maps_gt_flat_10.pt +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_metadata_10.csv?download=true -O tests/data/test_metadata_10.csv for FILE in 1.mrc 2.mrc 3.mrc 4.mrc populations.txt do wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/unprocessed_dataset_2_submissions/submission_x/${FILE}?download=true -O tests/data/unprocessed_dataset_2_submissions/submission_x/${FILE} From c74a3a8d302f3d631f8ba3407709769030289322 Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Tue, 25 Jun 2024 17:12:50 -0400 Subject: [PATCH 04/17] updated paths with Ground_truth folder on osf and locally" --- .../test_config_distribution_to_distribution.yaml | 2 +- tests/config_files/test_config_map_to_map.yaml | 6 +++--- tests/scripts/fetch_test_data.sh | 8 +++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/config_files/test_config_distribution_to_distribution.yaml b/tests/config_files/test_config_distribution_to_distribution.yaml index 9359e58..a1f03af 100644 --- a/tests/config_files/test_config_distribution_to_distribution.yaml +++ b/tests/config_files/test_config_distribution_to_distribution.yaml @@ -1,7 +1,7 @@ input_fname: tests/results/test_map_to_map_distance_matrix_submission_0.pkl metrics: - l2 -gt_metadata_fname: tests/data/test_metadata_10.csv +gt_metadata_fname: tests/data/Ground_truth/test_metadata_10.csv n_replicates: 3 n_pool_microstate: 2 replicate_fraction: 0.9 diff --git a/tests/config_files/test_config_map_to_map.yaml b/tests/config_files/test_config_map_to_map.yaml index eed5fb0..85d663d 100644 --- a/tests/config_files/test_config_map_to_map.yaml +++ b/tests/config_files/test_config_map_to_map.yaml @@ -7,11 +7,11 @@ data: metadata_key: populations label_key: id ground_truth: - volumes: tests/data/test_maps_gt_flat_10.pt - metadata: tests/data/test_metadata_10.csv + volumes: tests/data/Ground_truth/test_maps_gt_flat_10.pt + metadata: tests/data/Ground_truth/test_metadata_10.csv mask: do: true - volume: data/mask_dilated_wide_224x224.mrc + volume: data/Ground_truth/mask_dilated_wide_224x224.mrc analysis: metrics: - l2 diff --git a/tests/scripts/fetch_test_data.sh b/tests/scripts/fetch_test_data.sh index 613db91..2938c56 100644 --- a/tests/scripts/fetch_test_data.sh +++ b/tests/scripts/fetch_test_data.sh @@ -1,8 +1,10 @@ mkdir -p data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/dataset_2_submissions/test_submission_0_n8.pt?download=true -O tests/data/dataset_2_submissions/test_submission_0_n8.pt -wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_maps_gt_flat_10.pt?download=true -O tests/data/test_maps_gt_flat_10.pt -wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_metadata_10.csv?download=true -O tests/data/test_metadata_10.csv +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_maps_gt_flat_10.pt?download=true -O tests/data/Ground_truth/test_maps_gt_flat_10.pt +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_metadata_10.csv?download=true -O tests/data/Ground_truth/test_metadata_10.csv +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/1.mrc?download=true -O tests/data/Ground_truth/1.mrc +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/mask_dilated_wide_224x224.mrc?download=true -O tests/data/Ground_truth/mask_dilated_wide_224x224.mrc for FILE in 1.mrc 2.mrc 3.mrc 4.mrc populations.txt do wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/unprocessed_dataset_2_submissions/submission_x/${FILE}?download=true -O tests/data/unprocessed_dataset_2_submissions/submission_x/${FILE} -done \ No newline at end of file +done From 3137d9dada31ca042303e06acb8ec4f3cf717364 Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Tue, 25 Jun 2024 17:57:34 -0400 Subject: [PATCH 05/17] osf paths and local data dirs --- tests/scripts/fetch_test_data.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/fetch_test_data.sh b/tests/scripts/fetch_test_data.sh index 2938c56..4dde369 100644 --- a/tests/scripts/fetch_test_data.sh +++ b/tests/scripts/fetch_test_data.sh @@ -1,9 +1,9 @@ -mkdir -p data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x +mkdir -p data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x tests/data/Ground_truth/ wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/dataset_2_submissions/test_submission_0_n8.pt?download=true -O tests/data/dataset_2_submissions/test_submission_0_n8.pt wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_maps_gt_flat_10.pt?download=true -O tests/data/Ground_truth/test_maps_gt_flat_10.pt wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_metadata_10.csv?download=true -O tests/data/Ground_truth/test_metadata_10.csv wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/1.mrc?download=true -O tests/data/Ground_truth/1.mrc -wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/mask_dilated_wide_224x224.mrc?download=true -O tests/data/Ground_truth/mask_dilated_wide_224x224.mrc +wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/Ground_truth/mask_dilated_wide_224x224.mrc?download=true -O data/Ground_truth/mask_dilated_wide_224x224.mrc for FILE in 1.mrc 2.mrc 3.mrc 4.mrc populations.txt do wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/unprocessed_dataset_2_submissions/submission_x/${FILE}?download=true -O tests/data/unprocessed_dataset_2_submissions/submission_x/${FILE} From f3b1a5eb070f342b811c6407edb07517208ff51d Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Tue, 25 Jun 2024 18:26:34 -0400 Subject: [PATCH 06/17] force testing --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4d98e66..364f6db 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -49,7 +49,7 @@ jobs: pip install pytest omegaconf - name: Get test data from OSF - if: ${{ steps.cache_test_data.outputs.cache-hit != 'true' }} + # if: ${{ steps.cache_test_data.outputs.cache-hit != 'true' }} run: | sh tests/scripts/fetch_test_data.sh From 204eb4bdbc93ba456a59cefb24f7af145f10def3 Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Tue, 25 Jun 2024 18:28:58 -0400 Subject: [PATCH 07/17] update paths --- tests/scripts/fetch_test_data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/fetch_test_data.sh b/tests/scripts/fetch_test_data.sh index 4dde369..6286e52 100644 --- a/tests/scripts/fetch_test_data.sh +++ b/tests/scripts/fetch_test_data.sh @@ -1,4 +1,4 @@ -mkdir -p data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x tests/data/Ground_truth/ +mkdir -p data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x tests/data/Ground_truth/ data/Ground_truth wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/dataset_2_submissions/test_submission_0_n8.pt?download=true -O tests/data/dataset_2_submissions/test_submission_0_n8.pt wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_maps_gt_flat_10.pt?download=true -O tests/data/Ground_truth/test_maps_gt_flat_10.pt wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_metadata_10.csv?download=true -O tests/data/Ground_truth/test_metadata_10.csv From a0ac5473e3eb8c5ebe61d3e866e2ae918823e839 Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Tue, 25 Jun 2024 18:35:44 -0400 Subject: [PATCH 08/17] update paths --- tests/scripts/fetch_test_data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/fetch_test_data.sh b/tests/scripts/fetch_test_data.sh index 6286e52..2975718 100644 --- a/tests/scripts/fetch_test_data.sh +++ b/tests/scripts/fetch_test_data.sh @@ -1,4 +1,4 @@ -mkdir -p data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x tests/data/Ground_truth/ data/Ground_truth +mkdir -p tests/data/dataset_2_submissions data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x tests/data/Ground_truth/ data/Ground_truth wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/dataset_2_submissions/test_submission_0_n8.pt?download=true -O tests/data/dataset_2_submissions/test_submission_0_n8.pt wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_maps_gt_flat_10.pt?download=true -O tests/data/Ground_truth/test_maps_gt_flat_10.pt wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_metadata_10.csv?download=true -O tests/data/Ground_truth/test_metadata_10.csv From 904e0080d6b6c7081421404b346ba0635dd2cab4 Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Tue, 25 Jun 2024 18:39:48 -0400 Subject: [PATCH 09/17] revert --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 364f6db..4d98e66 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -49,7 +49,7 @@ jobs: pip install pytest omegaconf - name: Get test data from OSF - # if: ${{ steps.cache_test_data.outputs.cache-hit != 'true' }} + if: ${{ steps.cache_test_data.outputs.cache-hit != 'true' }} run: | sh tests/scripts/fetch_test_data.sh From b7813cef639d39c509c00e9243481b19688e8bf7 Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Wed, 26 Jun 2024 09:57:31 -0400 Subject: [PATCH 10/17] svd test --- tests/scripts/fetch_test_data.sh | 1 + tutorials/2_tutorial_svd.ipynb | 23 +++++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/scripts/fetch_test_data.sh b/tests/scripts/fetch_test_data.sh index 2975718..b5c9500 100644 --- a/tests/scripts/fetch_test_data.sh +++ b/tests/scripts/fetch_test_data.sh @@ -1,5 +1,6 @@ mkdir -p tests/data/dataset_2_submissions data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x tests/data/Ground_truth/ data/Ground_truth wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/dataset_2_submissions/test_submission_0_n8.pt?download=true -O tests/data/dataset_2_submissions/test_submission_0_n8.pt +ln -s tests/data/dataset_2_submissions/test_submission_0_n8.pt tests/data/dataset_2_submissions/submission_0.pt # symlink for svd which needs submission_0.pt for filename wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_maps_gt_flat_10.pt?download=true -O tests/data/Ground_truth/test_maps_gt_flat_10.pt wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_metadata_10.csv?download=true -O tests/data/Ground_truth/test_metadata_10.csv wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/1.mrc?download=true -O tests/data/Ground_truth/1.mrc diff --git a/tutorials/2_tutorial_svd.ipynb b/tutorials/2_tutorial_svd.ipynb index e0f33ac..b41bfba 100644 --- a/tutorials/2_tutorial_svd.ipynb +++ b/tutorials/2_tutorial_svd.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -40,9 +40,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c199472d76994ca7b2639a2709536f6d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FileChooser(path='/mnt/home/gwoollard', filename='', title='', show_hidden=False, select_desc='Select', change…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Select path to SVD config file\n", "# An example of this file is available in the path ../config_files/config_svd.yaml\n", From ff1d25c5d787e31bfbbd46506c2a510ebe880ead Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Wed, 26 Jun 2024 10:10:12 -0400 Subject: [PATCH 11/17] svd test in github workflows script --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4d98e66..829a237 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -56,6 +56,7 @@ jobs: - name: Test with pytest run: | pytest tests/test_preprocessing.py + pytest tests/test_svd.py pytest tests/test_map_to_map.py pytest tests/test_distribution_to_distribution.py From aa59911dd889d4278c509a202abf6b28c6b203b6 Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Wed, 26 Jun 2024 10:17:09 -0400 Subject: [PATCH 12/17] test svd --- tests/test_svd.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/test_svd.py diff --git a/tests/test_svd.py b/tests/test_svd.py new file mode 100644 index 0000000..568370e --- /dev/null +++ b/tests/test_svd.py @@ -0,0 +1,7 @@ +from omegaconf import OmegaConf +from cryo_challenge._commands import run_svd + + +def test_run_preprocessing(): + args = OmegaConf.create({'config': 'tests/config_files/test_config_svd.yaml'}) + run_svd.main(args) \ No newline at end of file From fc4f0be4f102ab7f33b8c5417ea7ee8ca06f6e4c Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Wed, 26 Jun 2024 10:17:35 -0400 Subject: [PATCH 13/17] test svd config --- tests/config_files/test_config_svd.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/config_files/test_config_svd.yaml diff --git a/tests/config_files/test_config_svd.yaml b/tests/config_files/test_config_svd.yaml new file mode 100644 index 0000000..460a194 --- /dev/null +++ b/tests/config_files/test_config_svd.yaml @@ -0,0 +1,14 @@ +path_to_volumes: tests/data/dataset_2_submissions/ +box_size_ds: 32 +submission_list: [1] +experiment_mode: "all_vs_ref" # options are "all_vs_all", "all_vs_ref" +# optional unless experiment_mode is "all_vs_ref" +path_to_reference: tests/data/Ground_truth/test_maps_gt_flat_10.pt +dtype: "float32" # options are "float32", "float64" +output_options: + # path will be created if it does not exist + output_path: tests/results/svd + # whether or not to save the processed volumes (downsampled, normalized, etc.) + save_volumes: True + # whether or not to save the SVD matrices (U, S, V) + save_svd_matrices: True From 265fb56edf6f6c1034b72136c3ebbc22c641c3cd Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Wed, 26 Jun 2024 10:45:57 -0400 Subject: [PATCH 14/17] refer to submission_0.pt --- tests/config_files/test_config_svd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config_files/test_config_svd.yaml b/tests/config_files/test_config_svd.yaml index 460a194..c392525 100644 --- a/tests/config_files/test_config_svd.yaml +++ b/tests/config_files/test_config_svd.yaml @@ -1,6 +1,6 @@ path_to_volumes: tests/data/dataset_2_submissions/ box_size_ds: 32 -submission_list: [1] +submission_list: [0] experiment_mode: "all_vs_ref" # options are "all_vs_all", "all_vs_ref" # optional unless experiment_mode is "all_vs_ref" path_to_reference: tests/data/Ground_truth/test_maps_gt_flat_10.pt From e24ba64b221b6c1cf22b4eeb4076a7c233b4d1aa Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Wed, 26 Jun 2024 10:51:49 -0400 Subject: [PATCH 15/17] manuallyrun osf fetch --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 829a237..8c21f83 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -49,7 +49,7 @@ jobs: pip install pytest omegaconf - name: Get test data from OSF - if: ${{ steps.cache_test_data.outputs.cache-hit != 'true' }} + # if: ${{ steps.cache_test_data.outputs.cache-hit != 'true' }} run: | sh tests/scripts/fetch_test_data.sh From ef7e50de6b5ce6e8b2fb9d092b5c14a79b1ee710 Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Wed, 26 Jun 2024 11:04:19 -0400 Subject: [PATCH 16/17] full path for symlink --- tests/scripts/fetch_test_data.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/scripts/fetch_test_data.sh b/tests/scripts/fetch_test_data.sh index b5c9500..5b58f23 100644 --- a/tests/scripts/fetch_test_data.sh +++ b/tests/scripts/fetch_test_data.sh @@ -1,6 +1,7 @@ mkdir -p tests/data/dataset_2_submissions data/dataset_2_submissions tests/results tests/data/unprocessed_dataset_2_submissions/submission_x tests/data/Ground_truth/ data/Ground_truth wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/dataset_2_submissions/test_submission_0_n8.pt?download=true -O tests/data/dataset_2_submissions/test_submission_0_n8.pt -ln -s tests/data/dataset_2_submissions/test_submission_0_n8.pt tests/data/dataset_2_submissions/submission_0.pt # symlink for svd which needs submission_0.pt for filename +ADIR=$(pwd) +ln -s $ADIR/tests/data/dataset_2_submissions/test_submission_0_n8.pt $ADIR/tests/data/dataset_2_submissions/submission_0.pt # symlink for svd which needs submission_0.pt for filename wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_maps_gt_flat_10.pt?download=true -O tests/data/Ground_truth/test_maps_gt_flat_10.pt wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/test_metadata_10.csv?download=true -O tests/data/Ground_truth/test_metadata_10.csv wget https://files.osf.io/v1/resources/8h6fz/providers/dropbox/tests/Ground_truth/1.mrc?download=true -O tests/data/Ground_truth/1.mrc From fde0f13db5cd37e5f9035aa3f553f8d6ed1e872a Mon Sep 17 00:00:00 2001 From: Geoffrey Woollard Date: Wed, 26 Jun 2024 11:10:17 -0400 Subject: [PATCH 17/17] revert --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 8c21f83..829a237 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -49,7 +49,7 @@ jobs: pip install pytest omegaconf - name: Get test data from OSF - # if: ${{ steps.cache_test_data.outputs.cache-hit != 'true' }} + if: ${{ steps.cache_test_data.outputs.cache-hit != 'true' }} run: | sh tests/scripts/fetch_test_data.sh