Merge pull request #78 from flatironinstitute/75-reduce-size-of-testi…

…ng-files 75 reduce size of testing files
flatironinstitute · Aug 12, 2024 · 3a588c6 · 3a588c6
2 parents fc99c96 + 410fe89
commit 3a588c6
Show file tree

Hide file tree

Showing 22 changed files with 29 additions and 39 deletions.
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -32,13 +32,6 @@ jobs:
           pip install .
           pip install pytest omegaconf
 
-      - name: Get test data from OSF
-        run: |
-          sh tests/scripts/fetch_test_data.sh
-
       - name: Test with pytest
         run: |
-          pytest tests/test_preprocessing.py
-          pytest tests/test_svd.py
-          pytest tests/test_map_to_map.py
-          pytest tests/test_distribution_to_distribution.py
+          pytest tests
diff --git a/.gitignore b/.gitignore
@@ -3,10 +3,7 @@ data/dataset_2_submissions
 data/dataset_1_submissions
 data/dataset_2_ground_truth
 
-# data for testing and resulting outputs
-tests/data/Ground_truth
-tests/data/dataset_2_submissions/
-tests/data/unprocessed_dataset_2_submissions/submission_x/
+# testing results
 tests/results/
 
 

diff --git a/tests/config_files/test_config_distribution_to_distribution.yaml b/tests/config_files/test_config_distribution_to_distribution.yaml
@@ -1,4 +1,4 @@
-input_fname: tests/results/test_map_to_map_distance_matrix_submission_0.pkl
+input_fname: tests/data/data_for_dist_to_dist/test_map_to_map_distance_matrix_submission_0.pkl
 metrics:
   - l2
 gt_metadata_fname: tests/data/Ground_truth/test_metadata_10.csv

diff --git a/tests/config_files/test_config_map_to_map.yaml b/tests/config_files/test_config_map_to_map.yaml
@@ -1,8 +1,8 @@
 data:
-  n_pix: 224
-  psize: 2.146
+  n_pix: 16
+  psize: 30.044
   submission:
-    fname: tests/data/dataset_2_submissions/test_submission_0_n8.pt
+    fname: tests/data/dataset_2_submissions/submission_1000.pt
     volume_key: volumes
     metadata_key: populations
     label_key: id
@@ -11,7 +11,7 @@ data:
     metadata: tests/data/Ground_truth/test_metadata_10.csv
   mask:
     do: true
-    volume: tests/data/Ground_truth/mask_dilated_wide_224x224.mrc
+    volume: tests/data/Ground_truth/test_mask_dilated_wide.mrc
 analysis:
   metrics:
     - l2

diff --git a/tests/config_files/test_config_svd.yaml b/tests/config_files/test_config_svd.yaml
@@ -1,6 +1,6 @@
 path_to_volumes: tests/data/dataset_2_submissions/
-box_size_ds: 32
-submission_list: [0]
+box_size_ds: 16
+submission_list: [1000]
 experiment_mode: "all_vs_ref" # options are "all_vs_all", "all_vs_ref"
 # optional unless experiment_mode is "all_vs_ref"
 path_to_reference: tests/data/Ground_truth/test_maps_gt_flat_10.pt

diff --git a/tests/data/Ground_truth/1.mrc b/tests/data/Ground_truth/1.mrc
diff --git a/tests/data/Ground_truth/mask_dilated_wide_224x224.mrc b/tests/data/Ground_truth/mask_dilated_wide_224x224.mrc
diff --git a/tests/data/Ground_truth/test_maps_gt_flat_10.npy b/tests/data/Ground_truth/test_maps_gt_flat_10.npy
diff --git a/tests/data/Ground_truth/test_maps_gt_flat_10.pt b/tests/data/Ground_truth/test_maps_gt_flat_10.pt
diff --git a/tests/data/Ground_truth/test_mask_dilated_wide.mrc b/tests/data/Ground_truth/test_mask_dilated_wide.mrc
diff --git a/tests/data/Ground_truth/test_metadata_10.csv b/tests/data/Ground_truth/test_metadata_10.csv
@@ -0,0 +1,11 @@
+index,volumes,populations_count,pc1,populations
+3238,13396.mrc,1,-231.62100638454024,2.9636654614427123e-05
+3020,10063.mrc,13,-179.257841640357,0.0003852765099875
+1113,01421.mrc,12,-141.8237629192062,0.0003556398553731
+3592,21858.mrc,4,-101.62462603216916,0.0001185466184577
+1947,03298.mrc,5,-34.99878410436052,0.0001481832730721
+2097,03764.mrc,6,3.946553364334135,0.0001778199276865
+1574,02336.mrc,5,44.70670231717438,0.0001481832730721
+2813,08011.mrc,8,108.6308222660271,0.0002370932369154
+88,00090.mrc,21,147.70416251702042,0.0006223697469029
+771,00906.mrc,11,186.3446095998357,0.0003260032007586
diff --git a/tests/data/data_for_dist_to_dist/test_map_to_map_distance_matrix_submission_0.pkl b/tests/data/data_for_dist_to_dist/test_map_to_map_distance_matrix_submission_0.pkl
diff --git a/tests/data/dataset_2_submissions/submission_1000.pt b/tests/data/dataset_2_submissions/submission_1000.pt
diff --git a/tests/data/test_maps_gt_flat_2.pt b/tests/data/test_maps_gt_flat_2.pt
diff --git a/tests/data/test_metadata_2.csv b/tests/data/test_metadata_2.csv
diff --git a/tests/data/unprocessed_dataset_2_submissions/submission_x/1.mrc b/tests/data/unprocessed_dataset_2_submissions/submission_x/1.mrc
diff --git a/tests/data/unprocessed_dataset_2_submissions/submission_x/2.mrc b/tests/data/unprocessed_dataset_2_submissions/submission_x/2.mrc
diff --git a/tests/data/unprocessed_dataset_2_submissions/submission_x/3.mrc b/tests/data/unprocessed_dataset_2_submissions/submission_x/3.mrc
diff --git a/tests/data/unprocessed_dataset_2_submissions/submission_x/4.mrc b/tests/data/unprocessed_dataset_2_submissions/submission_x/4.mrc
diff --git a/tests/data/unprocessed_dataset_2_submissions/submission_x/populations.txt b/tests/data/unprocessed_dataset_2_submissions/submission_x/populations.txt
@@ -0,0 +1,4 @@
+0.25
+0.25
+0.25
+0.25
diff --git a/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json b/tests/data/unprocessed_dataset_2_submissions/submission_x/submission_config.json
@@ -2,16 +2,16 @@
     "gt": {
         "name": "gt",
         "path": "tests/data/unprocessed_dataset_2_submissions/submission_x",
-        "box_size": 224,
-        "pixel_size": 2.146,
+        "box_size": 16,
+        "pixel_size": 30.044,
         "ref_align_fname": "1.mrc"
     },
     "0": {
         "name": "raw_submission_in_testdata",
         "align": 1,
-        "flavor_name": "test flavor", 
-        "box_size": 244,
-        "pixel_size": 2.146,
+        "flavor_name": "test flavor",
+        "box_size": 32,
+        "pixel_size": 15.022,
         "path": "tests/data/unprocessed_dataset_2_submissions/submission_x",
         "flip": 1,
         "populations_file": "tests/data/unprocessed_dataset_2_submissions/submission_x/populations.txt",

diff --git a/tests/scripts/fetch_test_data.sh b/tests/scripts/fetch_test_data.sh
-Original file line number
+Diff line change
@@ -0,0 +1,4 @@
+.25
+.25
+.25
+.25