Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fwd slash #1089

Open
wants to merge 4 commits into
base: generic-storage
Choose a base branch
from
Open

Fwd slash #1089

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ae79b84
Engine uploads convert ':' to '/'
hkethi002 Feb 28, 2018
9f3ad43
Match on metadata fname basename on engine upload
hkethi002 Feb 28, 2018
6342cc4
One to one matching of metadata to file form
hkethi002 Mar 19, 2018
6bdfcb4
route file download and info to contain slashes
hkethi002 Mar 19, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions api/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
'tag': '[^/]{1,32}',

# Filename
'fname': '[^/]+',
'fname': '.+',

# Note ID
'nid': '[0-9a-f]{24}',
Expand Down Expand Up @@ -272,9 +272,9 @@ def prefix(path, routes):
route('/packfile', FileListHandler, h='packfile', m=['POST']),
route('/packfile-end', FileListHandler, h='packfile_end'),
route('/<list_name:files>', FileListHandler, m=['POST']),
route('/<list_name:files>/info/<name:{fname}>', FileListHandler, h='get_info', m=['GET']),
route('/<list_name:files>/info/<name:{fname}>', FileListHandler, h='modify_info', m=['POST']),
route('/<list_name:files>/<name:{fname}>', FileListHandler, m=['GET', 'PUT', 'DELETE']),
route('/<list_name:files>/<name:{fname}>/info', FileListHandler, h='get_info', m=['GET']),
route('/<list_name:files>/<name:{fname}>/info', FileListHandler, h='modify_info', m=['POST']),

route( '/<sub_cont_name:{cname}|all>/analyses', AnalysesHandler, h='get_all', m=['GET']),
route( '/analyses', AnalysesHandler, h='get_all', m=['GET']),
Expand Down
5 changes: 4 additions & 1 deletion api/files.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,10 +122,13 @@ class SingleFileFieldStorage(cgi.FieldStorage):
def make_file(self, binary=None):
# Sanitize form's filename (read: prevent malicious escapes, bad characters, etc)

self.filename = fs.path.basename(self.filename)
self.hasher = hashlib.new(DEFAULT_HASH_ALG)
if not isinstance(self.filename, unicode):
self.filename = six.u(self.filename)

# If the filepath doesn't exist, make it
if not file_system.exists(fs.path.dirname(self.filename)) and self.filename:
file_system.makedirs(fs.path.dirname(self.filename))
self.open_file = file_system.open(self.filename, 'wb')
return self.open_file

Expand Down
2 changes: 1 addition & 1 deletion api/handlers/listhandler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,7 @@ def get(self, cont_name, list_name, **kwargs):
self.response.headers['Content-Type'] = str(fileinfo.get('mimetype', 'application/octet-stream'))
else:
self.response.headers['Content-Type'] = 'application/octet-stream'
self.response.headers['Content-Disposition'] = 'attachment; filename="' + filename + '"'
self.response.headers['Content-Disposition'] = 'attachment; filename="' + os.path.basename(filename) + '"'
else:
self.response.status = 206
if len(ranges) > 1:
Expand Down
2 changes: 1 addition & 1 deletion tests/integration_tests/python/test_access_log.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ def test_access_log_succeeds(data_builder, as_admin, log_db):

log_records_count_before = log_db.access_log.count({})

r = as_admin.get('/projects/' + project + '/files/' + file_name + '/info')
r = as_admin.get('/projects/' + project + '/files/info/' + file_name)
assert r.ok
assert r.json()['name'] == file_name

Expand Down
40 changes: 20 additions & 20 deletions tests/integration_tests/python/test_containers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -612,7 +612,7 @@ def test_edit_file_attributes(data_builder, as_admin, file_form):

assert as_admin.put('/projects/' + project + '/files/' + file_name, json=payload).ok

r = as_admin.get('/projects/' + project + '/files/' + file_name + '/info')
r = as_admin.get('/projects/' + project + '/files/info/' + file_name)
assert r.ok

file_object = r.json()
Expand Down Expand Up @@ -786,33 +786,33 @@ def test_edit_file_info(data_builder, as_admin, file_form):


# Assert getting file info 404s properly
r = as_admin.get('/projects/' + project + '/files/' + 'not_real.txt' + '/info')
r = as_admin.get('/projects/' + project + '/files/info/' + 'not_real.txt')
assert r.status_code == 404
r = as_admin.get('/projects/' + '000000000000000000000000' + '/files/' + 'not_real.txt' + '/info')
r = as_admin.get('/projects/' + '000000000000000000000000' + '/files/info/' + 'not_real.txt')
assert r.status_code == 404

r = as_admin.post('/projects/' + project + '/files', files=file_form(file_name))
assert r.ok

r = as_admin.get('/projects/' + project + '/files/' + file_name + '/info')
r = as_admin.get('/projects/' + project + '/files/info/' + file_name)
assert r.ok
assert r.json()['info'] == {}

# Send improper payload
r = as_admin.post('/projects/' + project + '/files/' + file_name + '/info', json={
r = as_admin.post('/projects/' + project + '/files/info/' + file_name, json={
'delete': ['map'],
'replace': {'not_going': 'to_happen'}
})
assert r.status_code == 400

# Send improper payload
r = as_admin.post('/projects/' + project + '/files/' + file_name + '/info', json={
r = as_admin.post('/projects/' + project + '/files/info/' + file_name, json={
'delete': {'a': 'map'},
})
assert r.status_code == 400

# Send improper payload
r = as_admin.post('/projects/' + project + '/files/' + file_name + '/info', json={
r = as_admin.post('/projects/' + project + '/files/info/' + file_name, json={
'set': 'cannot do this',
})
assert r.status_code == 400
Expand All @@ -828,60 +828,60 @@ def test_edit_file_info(data_builder, as_admin, file_form):
}


r = as_admin.post('/projects/' + project + '/files/' + file_name + '/info', json={
r = as_admin.post('/projects/' + project + '/files/info/' + file_name, json={
'replace': file_info
})
assert r.ok

r = as_admin.get('/projects/' + project + '/files/' + file_name + '/info')
r = as_admin.get('/projects/' + project + '/files/info/' + file_name)
assert r.ok
assert r.json()['info'] == file_info


# Use 'set' to add new key
r = as_admin.post('/projects/' + project + '/files/' + file_name + '/info', json={
r = as_admin.post('/projects/' + project + '/files/info/' + file_name, json={
'set': {'map': 'no longer a map'}
})
assert r.ok

file_info['map'] = 'no longer a map'
r = as_admin.get('/projects/' + project + '/files/' + file_name + '/info')
r = as_admin.get('/projects/' + project + '/files/info/' + file_name)
assert r.ok
assert r.json()['info'] == file_info


# Use 'set' to do full replace of "map" key
r = as_admin.post('/projects/' + project + '/files/' + file_name + '/info', json={
r = as_admin.post('/projects/' + project + '/files/info/' + file_name, json={
'set': {'map': 'no longer a map'}
})
assert r.ok

file_info['map'] = 'no longer a map'
r = as_admin.get('/projects/' + project + '/files/' + file_name + '/info')
r = as_admin.get('/projects/' + project + '/files/info/' + file_name)
assert r.ok
assert r.json()['info'] == file_info


# Use 'delete' to unset "map" key
r = as_admin.post('/projects/' + project + '/files/' + file_name + '/info', json={
r = as_admin.post('/projects/' + project + '/files/info/' + file_name, json={
'delete': ['map', 'a']
})
assert r.ok

file_info.pop('map')
file_info.pop('a')
r = as_admin.get('/projects/' + project + '/files/' + file_name + '/info')
r = as_admin.get('/projects/' + project + '/files/info/' + file_name)
assert r.ok
assert r.json()['info'] == file_info


# Use 'delete' on keys that do not exist
r = as_admin.post('/projects/' + project + '/files/' + file_name + '/info', json={
r = as_admin.post('/projects/' + project + '/files/info/' + file_name, json={
'delete': ['madeup', 'keys']
})
assert r.ok

r = as_admin.get('/projects/' + project + '/files/' + file_name + '/info')
r = as_admin.get('/projects/' + project + '/files/info/' + file_name)
assert r.ok
assert r.json()['info'] == file_info

Expand All @@ -895,7 +895,7 @@ def test_edit_file_info(data_builder, as_admin, file_form):

# Add reserved key and ensure it is returned
BIDS_map = {'BIDS':{'project_label': 'TEST'}}
r = as_admin.post('/projects/' + project + '/files/' + file_name + '/info', json={
r = as_admin.post('/projects/' + project + '/files/info/' + file_name, json={
'set': BIDS_map
})
assert r.ok
Expand All @@ -909,12 +909,12 @@ def test_edit_file_info(data_builder, as_admin, file_form):


# Use 'replace' to set file info to {}
r = as_admin.post('/projects/' + project + '/files/' + file_name + '/info', json={
r = as_admin.post('/projects/' + project + '/files/info/' + file_name, json={
'replace': {}
})
assert r.ok

r = as_admin.get('/projects/' + project + '/files/' + file_name + '/info')
r = as_admin.get('/projects/' + project + '/files/info/' + file_name)
assert r.ok
assert r.json()['info'] == {}

Expand Down
35 changes: 31 additions & 4 deletions tests/integration_tests/python/test_uploads.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -701,10 +701,28 @@ def test_acquisition_engine_upload(data_builder, file_form, as_root):
)
assert r.status_code == 404

metadata['acquisition']['files'] = [
{
'name': 'one.csv',
'type': 'engine type 0',
'info': {'test': 'f0'}
},
{
'name': 'folderA/two.csv',
'type': 'engine type 1',
'info': {'test': 'f1'}
},
{
'name': 'folderB/two.csv',
'type': 'engine type 1',
'info': {'test': 'f1'}
}
]

# engine upload
r = as_root.post('/engine',
params={'level': 'acquisition', 'id': acquisition, 'job': job},
files=file_form('one.csv', 'two.csv', meta=metadata)
files=file_form('one.csv', 'folderA/two.csv', 'folderB/two.csv', meta=metadata)
)
assert r.ok

Expand Down Expand Up @@ -739,7 +757,6 @@ def test_acquisition_engine_upload(data_builder, file_form, as_root):
assert f['type'] == mf['type']
assert f['info'] == mf['info']


def test_session_engine_upload(data_builder, file_form, as_root):
project = data_builder.create_project()
session = data_builder.create_session()
Expand All @@ -764,14 +781,19 @@ def test_session_engine_upload(data_builder, file_form, as_root):
'name': 'two.csv',
'type': 'engine type 1',
'info': {'test': 'f1'}
},
{
'name': 'folder/three.csv',
'type': 'engine type 2',
'info': {'test': 'f2'}
}
]
}
}

r = as_root.post('/engine',
params={'level': 'session', 'id': session},
files=file_form('one.csv', 'two.csv', meta=metadata)
files=file_form('one.csv', 'two.csv', 'folder/three.csv', meta=metadata)
)
assert r.ok

Expand Down Expand Up @@ -816,14 +838,19 @@ def test_project_engine_upload(data_builder, file_form, as_root):
'name': 'two.csv',
'type': 'engine type 1',
'info': {'test': 'f1'}
},
{
'name': 'folder/three.csv',
'type': 'engine type 2',
'info': {'test': 'f2'}
}
]
}
}

r = as_root.post('/engine',
params={'level': 'project', 'id': project},
files=file_form('one.csv', 'two.csv', meta=metadata)
files=file_form('one.csv', 'two.csv', 'folder/three.csv', meta=metadata)
)
assert r.ok

Expand Down