-
Notifications
You must be signed in to change notification settings - Fork 1
/
6_copy_to_data.py
114 lines (72 loc) · 4.07 KB
/
6_copy_to_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
import os
from glob import glob
import shutil
import sys
import logging
#from git import Repo
pipeline_dir = '/scratch/AG_Preibisch/Ella/embryo/nd2totif_maskembryos_stagebin_pipeline'
scratch_csv_path = os.path.join(pipeline_dir, 'embryos.csv')
dir_path_scratch_finaldata = os.path.join(pipeline_dir, "finaldata_temp_files")
dir_path_scratch_preview = os.path.join(pipeline_dir, "preview_embryos")
dir_path_new_tif = os.path.join(pipeline_dir, 'tif_temp_files')
dir_path_new_nd2 = os.path.join(pipeline_dir, 'nd2_temp_files')
analysis_path = os.path.join('/data/preibisch/Laura_Microscopy/dosage_compensation/smFISH-analysis/fit')
data_csv_path = os.path.join(analysis_path, 'embryos_csv', 'embryos.csv')
dir_path_data_finaldata = os.path.join(analysis_path, "finaldata")
dir_path_data_preview = os.path.join(analysis_path, "preview_embryos")
dir_path_data_tifs = os.path.join(analysis_path, "tifs")
log_file_path = os.path.join(pipeline_dir, 'pipeline.log')
######################### Set up log file ###############################
def setup_logger():
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(lineno)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
setup_logger()
logging.info("\n\nStarting script copy_to_data\n *********************************************")
#########################################################################
shutil.copyfile(scratch_csv_path, data_csv_path)
os.chmod(data_csv_path, 0o664)
new_tifs_paths = glob(os.path.join(dir_path_scratch_finaldata, 'tifs', '*'))
all_old_tifs_names = [os.path.basename(f) for f in glob(os.path.join(dir_path_data_finaldata, 'tifs', '*'))]
tifs_names = [os.path.basename(f) for f in new_tifs_paths]
for i,n in enumerate(tifs_names):
if n not in all_old_tifs_names:
shutil.copyfile(new_tifs_paths[i], os.path.join(dir_path_data_finaldata, 'tifs', n))
shutil.copyfile(os.path.join(dir_path_scratch_finaldata, 'masks', f'{n[:-4]}.mask.tif'),
os.path.join(dir_path_data_finaldata, 'masks', f'{n[:-4]}.mask.tif'))
shutil.copyfile(os.path.join(dir_path_scratch_preview, f'{n[:-4]}.png'), os.path.join(dir_path_data_preview, f'{n[:-4]}.png'))
for c in range(3):
shutil.copyfile(os.path.join(dir_path_scratch_finaldata, 'medians', f'c{c}_{n}'),
os.path.join(dir_path_data_finaldata, 'medians', f'c{c}_{n}'))
os.chmod(os.path.join(dir_path_data_finaldata, 'medians', f'c{c}_{n}'), 0o664)
os.chmod(os.path.join(dir_path_data_finaldata, 'tifs', n), 0o664)
os.chmod(os.path.join(dir_path_data_finaldata, 'masks', f'{n[:-4]}.mask.tif'), 0o664)
os.chmod(os.path.join(dir_path_data_preview, f'{n[:-4]}.png'), 0o664)
else:
logging.warning(f'{n} is already in data, double name')
## Copy original tifs (from nd2 before cropping):
org_tifs_scratch_paths = glob(os.path.join(dir_path_new_tif, '*'))
all_org_old_tifs_names = [os.path.basename(f) for f in glob(os.path.join(dir_path_data_tifs, '*'))]
tifs_names = [os.path.basename(f) for f in org_tifs_scratch_paths]
for i,n in enumerate(tifs_names):
if n not in all_org_old_tifs_names:
shutil.copyfile(org_tifs_scratch_paths[i], os.path.join(dir_path_data_tifs, n))
os.chmod(os.path.join(dir_path_data_tifs, n), 0o664)
else:
logging.warning(f'{n} is already in data, double name')
shutil.rmtree(dir_path_new_tif, ignore_errors=True)
shutil.rmtree(dir_path_new_nd2, ignore_errors=True)
shutil.rmtree(dir_path_scratch_finaldata, ignore_errors=True)
## Git add, commit, push
############################### Log file output status ################################
with open(log_file_path,'r') as f:
curr_run_log = f.read().split('Starting script copy_to_data')[-1].split('\n')
permission_errors = [l.split("<class")[0] for l in curr_run_log if "Permission" in l]
if len(permission_errors)>0:
nl = '\n'
logging.warning(f'AY YAY YAY, permission errors: \n {nl.join(permission_errors)}')
logging.info("Finished script, yay!\n ********************************************************************")