Skip to content

Commit

Permalink
Match on metadata fname basename on engine upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@hkethi002
hkethi002 committed Feb 28, 2018
1 parent e3d4568 commit 249d7ee
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 24 deletions.
7 changes: 3 additions & 4 deletions api/placer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import copy
import datetime
import dateutil
import os
import pymongo
import uuid
import zipfile
Expand Down Expand Up @@ -285,7 +286,7 @@ def process_file_field(self, field, file_attrs):
file_mds = self.metadata.get(self.container_type, {}).get('files', [])

for file_md in file_mds:
if file_md['name'] == file_attrs['name']:
if os.path.basename(file_md['name']) == file_attrs['name']:
file_attrs.update(file_md)
break

Expand All @@ -295,8 +296,6 @@ def process_file_field(self, field, file_attrs):
if not job_ticket['success']:
file_attrs['from_failed_job'] = True

file_attrs['name'] = file_attrs['name'].replace(':', '/')

self.save_file(field, file_attrs)
self.saved.append(file_attrs)

Expand All @@ -316,7 +315,7 @@ def finalize(self):
file_mds = self.metadata.get(self.container_type, {}).get('files', [])
saved_file_names = [x.get('name') for x in self.saved]
for file_md in file_mds:
if file_md['name'].replace(':', '/') not in saved_file_names:
if file_md['name'] not in saved_file_names:
self.save_file(None, file_md) # save file_attrs update only
self.saved.append(file_md)

Expand Down
31 changes: 11 additions & 20 deletions tests/integration_tests/python/test_uploads.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -708,7 +708,7 @@ def test_acquisition_engine_upload(data_builder, file_form, as_root):
'info': {'test': 'f0'}
},
{
'name': 'folder:two.csv',
'name': 'folder/two.csv',
'type': 'engine type 1',
'info': {'test': 'f1'}
}
Expand All @@ -717,7 +717,7 @@ def test_acquisition_engine_upload(data_builder, file_form, as_root):
# engine upload
r = as_root.post('/engine',
params={'level': 'acquisition', 'id': acquisition, 'job': job},
files=file_form('one.csv', 'folder:two.csv', meta=metadata)
files=file_form('one.csv', 'two.csv', meta=metadata)
)
assert r.ok

Expand Down Expand Up @@ -747,7 +747,7 @@ def test_acquisition_engine_upload(data_builder, file_form, as_root):
assert a_timestamp == m_timestamp

for mf in metadata['acquisition']['files']:
f = find_file_in_array(mf['name'].replace(':', '/'), a['files'])
f = find_file_in_array(mf['name'], a['files'])
assert mf is not None
assert f['type'] == mf['type']
assert f['info'] == mf['info']
Expand Down Expand Up @@ -779,7 +779,7 @@ def test_session_engine_upload(data_builder, file_form, as_root):
'info': {'test': 'f1'}
},
{
'name': 'folder:three.csv',
'name': 'folder/three.csv',
'type': 'engine type 2',
'info': {'test': 'f2'}
}
Expand All @@ -789,7 +789,7 @@ def test_session_engine_upload(data_builder, file_form, as_root):

r = as_root.post('/engine',
params={'level': 'session', 'id': session},
files=file_form('one.csv', 'two.csv', 'folder:three.csv', meta=metadata)
files=file_form('one.csv', 'two.csv', 'three.csv', meta=metadata)
)
assert r.ok

Expand All @@ -812,11 +812,7 @@ def test_session_engine_upload(data_builder, file_form, as_root):
assert s_timestamp == m_timestamp

for f in s['files']:
if '/' in f['name']:
assert f['name'] == 'folder/three.csv'
mf = find_file_in_array(f['name'].replace('/', ':'), metadata['session']['files'])
else:
mf = find_file_in_array(f['name'], metadata['session']['files'])
mf = find_file_in_array(f['name'], metadata['session']['files'])
assert mf is not None
assert f['type'] == mf['type']
assert f['info'] == mf['info']
Expand All @@ -840,7 +836,7 @@ def test_project_engine_upload(data_builder, file_form, as_root):
'info': {'test': 'f1'}
},
{
'name': 'folder:three.csv',
'name': 'folder/three.csv',
'type': 'engine type 2',
'info': {'test': 'f2'}
}
Expand All @@ -850,7 +846,7 @@ def test_project_engine_upload(data_builder, file_form, as_root):

r = as_root.post('/engine',
params={'level': 'project', 'id': project},
files=file_form('one.csv', 'two.csv', 'folder:three.csv', meta=metadata)
files=file_form('one.csv', 'two.csv', 'three.csv', meta=metadata)
)
assert r.ok

Expand All @@ -862,20 +858,15 @@ def test_project_engine_upload(data_builder, file_form, as_root):
assert p['info'] == metadata['project']['info']

for f in p['files']:
if '/' in f['name']:
assert f['name'] == 'folder/three.csv'
mf = find_file_in_array(f['name'].replace('/', ':'), metadata['project']['files'])
else:
mf = find_file_in_array(f['name'], metadata['project']['files'])
mf = find_file_in_array(f['name'], metadata['project']['files'])
assert mf is not None
assert f['type'] == mf['type']
assert f['info'] == mf['info']


def test_acquisition_file_only_engine_upload(data_builder, file_form, as_root):
acquisition = data_builder.create_acquisition()
file_names = ['one.csv', 'folder:two.csv']
expected_file_names = ['one.csv', 'folder/two.csv']
file_names = ['one.csv', 'two.csv']

r = as_root.post('/engine',
params={'level': 'acquisition', 'id': acquisition},
Expand All @@ -885,7 +876,7 @@ def test_acquisition_file_only_engine_upload(data_builder, file_form, as_root):

r = as_root.get('/acquisitions/' + acquisition)
assert r.ok
assert set(f['name'] for f in r.json()['files']) == set(expected_file_names)
assert set(f['name'] for f in r.json()['files']) == set(file_names)


def test_acquisition_subsequent_file_engine_upload(data_builder, file_form, as_root):
Expand Down

0 comments on commit 249d7ee

Please sign in to comment.