Skip to content

Commit

Permalink
_check_filesystem() first check size before hash, add a flag to skip …
Browse files Browse the repository at this point in the history
…hash
  • Loading branch information
oliche committed Jan 10, 2024
1 parent 139e1b7 commit 78765f6
Showing 1 changed file with 6 additions and 6 deletions.
12 changes: 6 additions & 6 deletions one/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -523,7 +523,7 @@ def sort_fcn(itm):
else:
return eids

def _check_filesystem(self, datasets, offline=None, update_exists=True):
def _check_filesystem(self, datasets, offline=None, update_exists=True, check_hash=True):
"""Update the local filesystem for the given datasets.
Given a set of datasets, check whether records correctly reflect the filesystem.
Expand Down Expand Up @@ -579,15 +579,15 @@ def _check_filesystem(self, datasets, offline=None, update_exists=True):
if file.exists():
# Check if there's a hash mismatch
# If so, add this index to list of datasets that need downloading
if rec['hash'] is not None:
if rec['file_size'] and file.stat().st_size != rec['file_size']:
_logger.warning('local file size mismatch on dataset: %s',
PurePosixPath(rec.session_path, rec.rel_path))
indices_to_download.append(i)
elif check_hash and rec['hash'] is not None:
if hashfile.md5(file) != rec['hash']:
_logger.warning('local md5 mismatch on dataset: %s',
PurePosixPath(rec.session_path, rec.rel_path))
indices_to_download.append(i)
elif rec['file_size'] and file.stat().st_size != rec['file_size']:
_logger.warning('local file size mismatch on dataset: %s',
PurePosixPath(rec.session_path, rec.rel_path))
indices_to_download.append(i)
files.append(file) # File exists so add to file list
else:
# File doesn't exist so add None to output file list
Expand Down

0 comments on commit 78765f6

Please sign in to comment.