Create placeholder files for collection attachments, update the remov…

…e-cas script to show progress of the rollback
scitran · Jan 25, 2018 · 1de467c · 1de467c
1 parent 2d8b919
commit 1de467c
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 7 deletions.
diff --git a/bin/oneoffs/create_placeholders.py b/bin/oneoffs/create_placeholders.py
@@ -23,7 +23,8 @@ def create_placeholders():
                             ('acquisitions', 'files'),
                             ('analyses', 'files'),
                             ('sessions', 'files'),
-                            ('sessions', 'subject.files')]
+                            ('sessions', 'subject.files'),
+                            ('collections', 'files')]
 
     _files = []
 

diff --git a/bin/oneoffs/remove_cas.py b/bin/oneoffs/remove_cas.py
@@ -28,6 +28,14 @@ def copy_file(path, target_path):
     shutil.copy(path, target_path)
 
 
+def cleanup_empty_folders():
+    # Cleanup the empty folders
+    log.info('Cleanup empty folders')
+    for _dirpath, _, _ in os.walk(config.get_item('persistent', 'data_path'), topdown=False):
+        if not (os.listdir(_dirpath) or config.get_item('persistent', 'data_path') == _dirpath):
+            os.rmdir(_dirpath)
+
+
 def remove_cas():
     """
     Remove CAS logic, generate UUID for the files and rename them on the filesystem, make a copy of the file if more
@@ -103,7 +111,7 @@ def remove_cas():
         log.exception(e)
         log.info('Rollback...')
         base = config.get_item('persistent', 'data_path')
-        for f in _files:
+        for i, f in enumerate(_files):
             if f.get('_id', ''):
                 hash_path = os.path.join(base, util.path_from_hash(f['fileinfo']['hash']))
                 uuid_path = files.get_file_abs_path(f['_id'])
@@ -116,13 +124,13 @@ def remove_cas():
                     {'_id': f['collection_id'], f['prefix'] + '.name': f['fileinfo']['name']},
                     {'$unset': {f['prefix'] + '.$._id': ''}}
                 )
+            # Show progress
+            if i % (len(_files) / 10) == 0:
+                log.info('Processed %s files of total %s files ...' % (i, len(_files)))
+        cleanup_empty_folders()
         exit(1)
 
-    # Cleanup the empty folders
-    log.info('Cleanup empty folders')
-    for _dirpath, _, _ in os.walk(config.get_item('persistent', 'data_path'), topdown=False):
-        if not (os.listdir(_dirpath) or config.get_item('persistent', 'data_path') == _dirpath):
-            os.rmdir(_dirpath)
+    cleanup_empty_folders()
 
 
 if __name__ == '__main__':