From ae79b84c1872ae68fe7272617f37333216391f14 Mon Sep 17 00:00:00 2001 From: Harsha Kethineni Date: Wed, 28 Feb 2018 12:28:58 -0600 Subject: [PATCH] Engine uploads convert ':' to '/' --- api/placer.py | 4 +- .../integration_tests/python/test_uploads.py | 48 +++++++++++++++---- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/api/placer.py b/api/placer.py index 2e1215092..cfe5a163f 100644 --- a/api/placer.py +++ b/api/placer.py @@ -299,6 +299,8 @@ def process_file_field(self, field, file_attrs): if not job_ticket['success']: file_attrs['from_failed_job'] = True + file_attrs['name'] = file_attrs['name'].replace(':', '/') + self.save_file(field, file_attrs) self.saved.append(file_attrs) @@ -318,7 +320,7 @@ def finalize(self): file_mds = self.metadata.get(self.container_type, {}).get('files', []) saved_file_names = [x.get('name') for x in self.saved] for file_md in file_mds: - if file_md['name'] not in saved_file_names: + if file_md['name'].replace(':', '/') not in saved_file_names: self.save_file(None, file_md) # save file_attrs update only self.saved.append(file_md) diff --git a/tests/integration_tests/python/test_uploads.py b/tests/integration_tests/python/test_uploads.py index 607ec86e8..d556057ca 100644 --- a/tests/integration_tests/python/test_uploads.py +++ b/tests/integration_tests/python/test_uploads.py @@ -701,10 +701,23 @@ def test_acquisition_engine_upload(data_builder, file_form, as_root): ) assert r.status_code == 404 + metadata['acquisition']['files'] = [ + { + 'name': 'one.csv', + 'type': 'engine type 0', + 'info': {'test': 'f0'} + }, + { + 'name': 'folder:two.csv', + 'type': 'engine type 1', + 'info': {'test': 'f1'} + } + ] + # engine upload r = as_root.post('/engine', params={'level': 'acquisition', 'id': acquisition, 'job': job}, - files=file_form('one.csv', 'two.csv', meta=metadata) + files=file_form('one.csv', 'folder:two.csv', meta=metadata) ) assert r.ok @@ -734,7 +747,7 @@ def test_acquisition_engine_upload(data_builder, file_form, as_root): assert a_timestamp == m_timestamp for mf in metadata['acquisition']['files']: - f = find_file_in_array(mf['name'], a['files']) + f = find_file_in_array(mf['name'].replace(':', '/'), a['files']) assert mf is not None assert f['type'] == mf['type'] assert f['info'] == mf['info'] @@ -764,6 +777,11 @@ def test_session_engine_upload(data_builder, file_form, as_root): 'name': 'two.csv', 'type': 'engine type 1', 'info': {'test': 'f1'} + }, + { + 'name': 'folder:three.csv', + 'type': 'engine type 2', + 'info': {'test': 'f2'} } ] } @@ -771,7 +789,7 @@ def test_session_engine_upload(data_builder, file_form, as_root): r = as_root.post('/engine', params={'level': 'session', 'id': session}, - files=file_form('one.csv', 'two.csv', meta=metadata) + files=file_form('one.csv', 'two.csv', 'folder:three.csv', meta=metadata) ) assert r.ok @@ -794,7 +812,11 @@ def test_session_engine_upload(data_builder, file_form, as_root): assert s_timestamp == m_timestamp for f in s['files']: - mf = find_file_in_array(f['name'], metadata['session']['files']) + if '/' in f['name']: + assert f['name'] == 'folder/three.csv' + mf = find_file_in_array(f['name'].replace('/', ':'), metadata['session']['files']) + else: + mf = find_file_in_array(f['name'], metadata['session']['files']) assert mf is not None assert f['type'] == mf['type'] assert f['info'] == mf['info'] @@ -816,6 +838,11 @@ def test_project_engine_upload(data_builder, file_form, as_root): 'name': 'two.csv', 'type': 'engine type 1', 'info': {'test': 'f1'} + }, + { + 'name': 'folder:three.csv', + 'type': 'engine type 2', + 'info': {'test': 'f2'} } ] } @@ -823,7 +850,7 @@ def test_project_engine_upload(data_builder, file_form, as_root): r = as_root.post('/engine', params={'level': 'project', 'id': project}, - files=file_form('one.csv', 'two.csv', meta=metadata) + files=file_form('one.csv', 'two.csv', 'folder:three.csv', meta=metadata) ) assert r.ok @@ -835,7 +862,11 @@ def test_project_engine_upload(data_builder, file_form, as_root): assert p['info'] == metadata['project']['info'] for f in p['files']: - mf = find_file_in_array(f['name'], metadata['project']['files']) + if '/' in f['name']: + assert f['name'] == 'folder/three.csv' + mf = find_file_in_array(f['name'].replace('/', ':'), metadata['project']['files']) + else: + mf = find_file_in_array(f['name'], metadata['project']['files']) assert mf is not None assert f['type'] == mf['type'] assert f['info'] == mf['info'] @@ -843,7 +874,8 @@ def test_project_engine_upload(data_builder, file_form, as_root): def test_acquisition_file_only_engine_upload(data_builder, file_form, as_root): acquisition = data_builder.create_acquisition() - file_names = ['one.csv', 'two.csv'] + file_names = ['one.csv', 'folder:two.csv'] + expected_file_names = ['one.csv', 'folder/two.csv'] r = as_root.post('/engine', params={'level': 'acquisition', 'id': acquisition}, @@ -853,7 +885,7 @@ def test_acquisition_file_only_engine_upload(data_builder, file_form, as_root): r = as_root.get('/acquisitions/' + acquisition) assert r.ok - assert set(f['name'] for f in r.json()['files']) == set(file_names) + assert set(f['name'] for f in r.json()['files']) == set(expected_file_names) def test_acquisition_subsequent_file_engine_upload(data_builder, file_form, as_root):