diff --git a/src/aind_data_transfer_service/configs/job_configs.py b/src/aind_data_transfer_service/configs/job_configs.py index 72c4ce9..c59ef37 100644 --- a/src/aind_data_transfer_service/configs/job_configs.py +++ b/src/aind_data_transfer_service/configs/job_configs.py @@ -131,11 +131,6 @@ class BasicUploadJobConfigs(BaseSettings): aws_param_store_name: Optional[str] = Field(None) - processor_full_name: str = Field( - ..., - description="Name of person uploading data", - title="Processor Full Name", - ) project_name: str = Field( ..., description="Name of project", title="Project Name" ) diff --git a/src/aind_data_transfer_service/configs/job_upload_template.py b/src/aind_data_transfer_service/configs/job_upload_template.py index bd8762f..2debb7a 100644 --- a/src/aind_data_transfer_service/configs/job_upload_template.py +++ b/src/aind_data_transfer_service/configs/job_upload_template.py @@ -19,7 +19,6 @@ class JobUploadTemplate: NUM_TEMPLATE_ROWS = 20 XLSX_DATETIME_FORMAT = "YYYY-MM-DDTHH:mm:ss" HEADERS = [ - "processor_full_name", "project_name", "process_capsule_id", "platform", @@ -33,7 +32,6 @@ class JobUploadTemplate: ] SAMPLE_JOBS = [ [ - "Anna Apple", "Behavior Platform", "1f999652-00a0-4c4b-99b5-64c2985ad070", Platform.BEHAVIOR.abbreviation, @@ -46,7 +44,6 @@ class JobUploadTemplate: "/allen/aind/stage/fake/dir", ], [ - "John Smith", "Ophys Platform - SLAP2", None, Platform.SMARTSPIM.abbreviation, @@ -57,7 +54,6 @@ class JobUploadTemplate: "/allen/aind/stage/fake/dir", ], [ - "Anna Apple", "Ephys Platform", None, Platform.ECEPHYS.abbreviation, diff --git a/tests/resources/job_upload_template.xlsx b/tests/resources/job_upload_template.xlsx index 9203d33..39ec6f7 100644 Binary files a/tests/resources/job_upload_template.xlsx and b/tests/resources/job_upload_template.xlsx differ diff --git a/tests/resources/sample.csv b/tests/resources/sample.csv index 2305c47..d62bab1 100644 --- a/tests/resources/sample.csv +++ b/tests/resources/sample.csv @@ -1,4 +1,4 @@ -processor_full_name, project_name, process_capsule_id, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime -Anna Apple, Ephys Platform, , ECEPHYS, dir/data_set_1, ,, some_bucket, 123454, ecephys, 2020-10-10 14:10:10 -John Smith, Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, BEHAVIOR_VIDEOS, dir/data_set_2, MRI, dir/data_set_3, open, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM -Anna Apple, Behavior Platform, , BEHAVIOR_VIDEOS, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, scratch, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +project_name, process_capsule_id, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime +Ephys Platform, , ECEPHYS, dir/data_set_1, ,, some_bucket, 123454, ecephys, 2020-10-10 14:10:10 +Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, BEHAVIOR_VIDEOS, dir/data_set_2, MRI, dir/data_set_3, open, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +Behavior Platform, , BEHAVIOR_VIDEOS, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, scratch, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM diff --git a/tests/resources/sample.xlsx b/tests/resources/sample.xlsx index 94578d7..e7ab762 100644 Binary files a/tests/resources/sample.xlsx and b/tests/resources/sample.xlsx differ diff --git a/tests/resources/sample_alt_modality_case.csv b/tests/resources/sample_alt_modality_case.csv index c58f227..35f31da 100644 --- a/tests/resources/sample_alt_modality_case.csv +++ b/tests/resources/sample_alt_modality_case.csv @@ -1,4 +1,4 @@ -processor_full_name, project_name, process_capsule_id, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime -Anna Apple, Ephys Platform, , ecephys, dir/data_set_1, ,, some_bucket, 123454, ecephys, 2020-10-10 14:10:10 -John Smith, Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM -Anna Apple, Behavior Platform, , behavior-videos, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, scratch, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +project_name, process_capsule_id, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime +Ephys Platform, , ecephys, dir/data_set_1, ,, some_bucket, 123454, ecephys, 2020-10-10 14:10:10 +Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +Behavior Platform, , behavior-videos, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, scratch, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM diff --git a/tests/resources/sample_empty_rows.csv b/tests/resources/sample_empty_rows.csv index 9013962..48b3a14 100644 --- a/tests/resources/sample_empty_rows.csv +++ b/tests/resources/sample_empty_rows.csv @@ -1,7 +1,7 @@ -processor_full_name, project_name, process_capsule_id, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime -Anna Apple, Ephys Platform, , ECEPHYS, dir/data_set_1, ,, some_bucket, 123454, ecephys, 2020-10-10 14:10:10 -John Smith, Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, BEHAVIOR_VIDEOS, dir/data_set_2, MRI, dir/data_set_3, open, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM -Anna Apple, Behavior Platform, , BEHAVIOR_VIDEOS, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, scratch, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +project_name, process_capsule_id, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime +Ephys Platform, , ECEPHYS, dir/data_set_1, ,, some_bucket, 123454, ecephys, 2020-10-10 14:10:10 +Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, BEHAVIOR_VIDEOS, dir/data_set_2, MRI, dir/data_set_3, open, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +Behavior Platform, , BEHAVIOR_VIDEOS, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, scratch, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM ,,,,,, ,,,,,, ,,,,,, diff --git a/tests/resources/sample_empty_rows.xlsx b/tests/resources/sample_empty_rows.xlsx index 62817e5..0274f9d 100644 Binary files a/tests/resources/sample_empty_rows.xlsx and b/tests/resources/sample_empty_rows.xlsx differ diff --git a/tests/resources/sample_invalid_ext.txt b/tests/resources/sample_invalid_ext.txt index 2305c47..d62bab1 100644 --- a/tests/resources/sample_invalid_ext.txt +++ b/tests/resources/sample_invalid_ext.txt @@ -1,4 +1,4 @@ -processor_full_name, project_name, process_capsule_id, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime -Anna Apple, Ephys Platform, , ECEPHYS, dir/data_set_1, ,, some_bucket, 123454, ecephys, 2020-10-10 14:10:10 -John Smith, Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, BEHAVIOR_VIDEOS, dir/data_set_2, MRI, dir/data_set_3, open, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM -Anna Apple, Behavior Platform, , BEHAVIOR_VIDEOS, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, scratch, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +project_name, process_capsule_id, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime +Ephys Platform, , ECEPHYS, dir/data_set_1, ,, some_bucket, 123454, ecephys, 2020-10-10 14:10:10 +Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, BEHAVIOR_VIDEOS, dir/data_set_2, MRI, dir/data_set_3, open, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +Behavior Platform, , BEHAVIOR_VIDEOS, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, scratch, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM diff --git a/tests/resources/sample_malformed.csv b/tests/resources/sample_malformed.csv index 690c713..f31058e 100644 --- a/tests/resources/sample_malformed.csv +++ b/tests/resources/sample_malformed.csv @@ -1,4 +1,4 @@ -processor_full_name, project_name, process_capsule_id, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime -Anna Apple, Ephys Platform, , ECEPHYS, dir/data_set_1, ,, some_bucket, 123454, ecephys, 2020-10-10 14:10:10 -John Smith, Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, WRONG_MODALITY_HERE, dir/data_set_2, MRI, dir/data_set_3, open, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM -Anna Apple, Behavior Platform, , BEHAVIOR_VIDEOS, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, scratch, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +project_name, process_capsule_id, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime +Ephys Platform, , ECEPHYS, dir/data_set_1, ,, some_bucket, 123454, ecephys, 2020-10-10 14:10:10 +Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, WRONG_MODALITY_HERE, dir/data_set_2, MRI, dir/data_set_3, open, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +Behavior Platform, , BEHAVIOR_VIDEOS, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, scratch, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM diff --git a/tests/resources/sample_malformed.xlsx b/tests/resources/sample_malformed.xlsx index 66afb63..417d5cd 100644 Binary files a/tests/resources/sample_malformed.xlsx and b/tests/resources/sample_malformed.xlsx differ diff --git a/tests/test_configs.py b/tests/test_configs.py index 0bbc4a2..7ce51eb 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -27,7 +27,6 @@ class TestJobConfigs(unittest.TestCase): expected_job_configs = [ BasicUploadJobConfigs( aws_param_store_name="/some/param/store", - processor_full_name="Anna Apple", project_name="Ephys Platform", process_capsule_id=None, s3_bucket="private", @@ -53,7 +52,6 @@ class TestJobConfigs(unittest.TestCase): ), BasicUploadJobConfigs( aws_param_store_name="/some/param/store", - processor_full_name="John Smith", project_name="Behavior Platform", process_capsule_id="1f999652-00a0-4c4b-99b5-64c2985ad070", s3_bucket="open", @@ -86,7 +84,6 @@ class TestJobConfigs(unittest.TestCase): ), BasicUploadJobConfigs( aws_param_store_name="/some/param/store", - processor_full_name="Anna Apple", project_name="Behavior Platform", process_capsule_id=None, s3_bucket="scratch", @@ -158,7 +155,6 @@ def test_parse_csv_file(self): # not formatted correctly with self.assertRaises(Exception) as e1: BasicUploadJobConfigs( - processor_full_name="Anna Apple", project_name="Behavior Platform", s3_bucket="", platform=Platform.BEHAVIOR, @@ -225,7 +221,6 @@ def test_malformed_platform(self): with self.assertRaises(AttributeError) as e: BasicUploadJobConfigs( aws_param_store_name="/some/param/store", - processor_full_name="Anna Apple", project_name="Behavior Platform", s3_bucket="some_bucket2", platform="MISSING", @@ -321,7 +316,6 @@ def test_from_job_and_server_configs(self): " python -m aind_data_transfer.jobs.basic_job" " --json-args ' " '{"aws_param_store_name":"/some/param/store",' - '"processor_full_name":"Anna Apple",' '"project_name":"Ephys Platform",' '"process_capsule_id":null,' '"s3_bucket":"private",' diff --git a/tests/test_server.py b/tests/test_server.py index 1b0f30d..a45e7f9 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -298,8 +298,7 @@ def test_submit_hpc_jobs( { "hpc_settings": '{"qos":"production", "name": "job1"}', "upload_job_settings": ( - '{"processor_full_name":"Anna Apple", ' - '"project_name":"Ephys Platform", ' + '{"project_name":"Ephys Platform", ' '"process_capsule_id":null, ' '"s3_bucket": "private", ' '"platform": {"name": "Behavior platform", ' @@ -367,8 +366,7 @@ def test_submit_hpc_jobs_open_data( { "hpc_settings": '{"qos":"production", "name": "job1"}', "upload_job_settings": ( - '{"processor_full_name":"Anna Apple", ' - '"project_name":"Ephys Platform", ' + '{"project_name":"Ephys Platform", ' '"process_capsule_id":null, ' '"s3_bucket": "open", ' '"platform": {"name": "Behavior platform", '