diff --git a/config/config.yml b/config/config.yml index 2c50355..6f06925 100644 --- a/config/config.yml +++ b/config/config.yml @@ -140,6 +140,13 @@ report: slice_start: 0 slice_step: 50 colour_map: viridis + resources: + readme_md: resources/qc/README.md + vol_viewer_dir: resources/qc/volViewer + ff_html: resources/qc/ff_html_temp.html + report_html: resources/qc/qc_report_temp.html + subject_html: resources/qc/subject_html_temp.html + ws_html: resources/qc/ws_html_temp.html containers: diff --git a/qc/README.md b/resources/qc/README.md similarity index 58% rename from qc/README.md rename to resources/qc/README.md index 400b91e..a77594a 100644 --- a/qc/README.md +++ b/resources/qc/README.md @@ -2,18 +2,13 @@ ### 1: Run the workflow with personalized report generation configurations -### 2: Navigate to qc directory: - -```bash -cd ./qc -``` -### 3: Create a Python web server: +### 2: Navigate to qc directory, then create a Python web server: ```bash python -m http.server ``` -### 4: Open the link generated in a browser and open qc_report.html +### 3: Open the link generated in a browser and open `qc_report.html` ### Note: - Can view the whole slice images and the flatfield corrections without running the web server. However, to view the volume rendered brain, the web server is required. \ No newline at end of file + Can view the whole slice images and the flatfield corrections without running the web server. However, to view the volume rendered brain, the web server is required. diff --git a/qc/resources/ff_html_temp.html b/resources/qc/ff_html_temp.html similarity index 100% rename from qc/resources/ff_html_temp.html rename to resources/qc/ff_html_temp.html diff --git a/qc/resources/qc_report_temp.html b/resources/qc/qc_report_temp.html similarity index 100% rename from qc/resources/qc_report_temp.html rename to resources/qc/qc_report_temp.html diff --git a/qc/resources/subject_html_temp.html b/resources/qc/subject_html_temp.html similarity index 100% rename from qc/resources/subject_html_temp.html rename to resources/qc/subject_html_temp.html diff --git a/qc/resources/volViewer/cm_gray.png b/resources/qc/volViewer/cm_gray.png similarity index 100% rename from qc/resources/volViewer/cm_gray.png rename to resources/qc/volViewer/cm_gray.png diff --git a/qc/resources/volViewer/cm_viridis.png b/resources/qc/volViewer/cm_viridis.png similarity index 100% rename from qc/resources/volViewer/cm_viridis.png rename to resources/qc/volViewer/cm_viridis.png diff --git a/qc/resources/volViewer/volRender.html b/resources/qc/volViewer/volRender.html similarity index 100% rename from qc/resources/volViewer/volRender.html rename to resources/qc/volViewer/volRender.html diff --git a/qc/resources/volViewer/volRenderScript.js b/resources/qc/volViewer/volRenderScript.js similarity index 100% rename from qc/resources/volViewer/volRenderScript.js rename to resources/qc/volViewer/volRenderScript.js diff --git a/qc/resources/ws_html_temp.html b/resources/qc/ws_html_temp.html similarity index 100% rename from qc/resources/ws_html_temp.html rename to resources/qc/ws_html_temp.html diff --git a/workflow/Snakefile b/workflow/Snakefile index 85ecad7..155244d 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -33,6 +33,7 @@ rule all: input: get_all_targets(), get_bids_toplevel_targets(), + get_qc_targets(), localrule: True diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 4e52bc7..e6b7ad0 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -29,6 +29,20 @@ def expand_bids(expand_kwargs, **bids_kwargs): return files +def remote_file(filename): + if is_remote(filename): + return storage(str(filename)) + else: + return filename + + +def remote_directory(dirname): + if is_remote(dirname): + return storage(directory(str(dirname))) + else: + return directory(dirname) + + def directory_bids(root, *args, **kwargs): """Similar to expand_bids, this replacement function is needed to ensure storage() comes after directory() tags""" @@ -107,18 +121,25 @@ def get_all_targets(): ), ) ) - if config["report"]["create_report"]: - targets.extend( - expand( - "qc/sub-{subject}_sample-{sample}_acq-{acq}/subject.html", - subject=datasets.loc[i, "subject"], - sample=datasets.loc[i, "sample"], - acq=datasets.loc[i, "acq"], - ) - ) return targets +def get_all_subj_html(wildcards): + htmls = [] + + for i in range(len(datasets)): + + html = "{root}/qc/sub-{subject}_sample-{sample}_acq-{acq}/subject.html".format( + root=root, + subject=datasets.loc[i, "subject"], + sample=datasets.loc[i, "sample"], + acq=datasets.loc[i, "acq"], + ) + htmls.append(remote_file(html)) + + return htmls + + def get_bids_toplevel_targets(): targets = [] targets.append(bids_toplevel(root, "README.md")) @@ -129,6 +150,14 @@ def get_bids_toplevel_targets(): return targets +def get_qc_targets(): + targets = [] + if config["report"]["create_report"]: + targets.append(remote_file(Path(root) / "qc" / "qc_report.html")) + targets.append(remote_file(Path(root) / "qc" / "README.md")) + return targets + + def dataset_is_remote(wildcards): return is_remote_gcs(Path(get_dataset_path(wildcards))) diff --git a/workflow/rules/qc.smk b/workflow/rules/qc.smk index 415b3d7..cb44298 100644 --- a/workflow/rules/qc.smk +++ b/workflow/rules/qc.smk @@ -1,3 +1,16 @@ + +rule setup_qc_dir: + "Copies QC resources to the output bids folder" + input: + readme_md=config["report"]["resources"]["readme_md"], + output: + readme_md=remote_file(Path(root) / "qc" / "README.md"), + log: + "logs/setup_qc_dir_log.txt", + shell: + "cp {input.readme_md} {output.readme_md}" + + rule generate_flatfield_qc: "Generates an html file for comparing before and after flatfield correction" input: @@ -19,17 +32,30 @@ rule generate_flatfield_qc: desc="flatcorr", suffix="SPIM.zarr", ), + ff_html=config["report"]["resources"]["ff_html"], params: ff_s_start=config["report"]["flatfield_corrected"]["slice_start"], ff_s_step=config["report"]["flatfield_corrected"]["slice_step"], ff_cmap=config["report"]["flatfield_corrected"]["colour_map"], output: - html="qc/sub-{subject}_sample-{sample}_acq-{acq}/flatfieldqc.html", - corr_images_dir=directory( - "qc/sub-{subject}_sample-{sample}_acq-{acq}/images/corr" + html=remote_file( + Path(root) + / "qc" + / "sub-{subject}_sample-{sample}_acq-{acq}/flatfieldqc.html" ), - uncorr_images_dir=directory( - "qc/sub-{subject}_sample-{sample}_acq-{acq}/images/uncorr" + corr_images_dir=remote_directory( + Path(root) + / "qc" + / "sub-{subject}_sample-{sample}_acq-{acq}" + / "images" + / "corr" + ), + uncorr_images_dir=remote_directory( + Path(root) + / "qc" + / "sub-{subject}_sample-{sample}_acq-{acq}" + / "images" + / "uncorr" ), log: bids( @@ -49,6 +75,7 @@ rule generate_whole_slice_qc: input: **get_storage_creds(), ome=get_input_ome_zarr_to_nii(), + ws_html=config["report"]["resources"]["ws_html"], params: ws_s_start=config["report"]["whole_slice_viewer"]["slice_start"], ws_s_step=config["report"]["whole_slice_viewer"]["slice_step"], @@ -56,8 +83,19 @@ rule generate_whole_slice_qc: uri=get_output_ome_zarr_uri(), storage_provider_settings=workflow.storage_provider_settings, output: - html="qc/sub-{subject}_sample-{sample}_acq-{acq}/whole_slice_qc.html", - images_dir=directory("qc/sub-{subject}_sample-{sample}_acq-{acq}/images/whole"), + html=remote_file( + Path(root) + / "qc" + / "sub-{subject}_sample-{sample}_acq-{acq}" + / "whole_slice_qc.html" + ), + images_dir=remote_directory( + Path(root) + / "qc" + / "sub-{subject}_sample-{sample}_acq-{acq}" + / "images" + / "whole" + ), log: bids( root="logs", @@ -76,14 +114,23 @@ rule generate_volume_qc: input: **get_storage_creds(), ome=get_input_ome_zarr_to_nii(), + vol_viewer_dir=config["report"]["resources"]["vol_viewer_dir"], params: uri=get_output_ome_zarr_uri(), storage_provider_settings=workflow.storage_provider_settings, output: - resources=directory( - "qc/sub-{subject}_sample-{sample}_acq-{acq}/volume_resources" + resources=remote_directory( + Path(root) + / "qc" + / "sub-{subject}_sample-{sample}_acq-{acq}" + / "volume_resources" + ), + html=remote_file( + Path(root) + / "qc" + / "sub-{subject}_sample-{sample}_acq-{acq}" + / "volume_qc.html" ), - html="qc/sub-{subject}_sample-{sample}_acq-{acq}/volume_qc.html", log: bids( root="logs", @@ -100,11 +147,17 @@ rule generate_volume_qc: rule generate_subject_qc: "Generates html files to access all the subjects qc reports in one place" input: - ws_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/whole_slice_qc.html", - ff_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/flatfieldqc.html", - vol_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/volume_qc.html", + subject_html=config["report"]["resources"]["subject_html"], + ws_html=rules.generate_whole_slice_qc.output.html, + ff_html=rules.generate_flatfield_qc.output.html, + vol_html=rules.generate_volume_qc.output.html, output: - sub_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/subject.html", + sub_html=remote_file( + Path(root) + / "qc" + / "sub-{subject}_sample-{sample}_acq-{acq}" + / "subject.html" + ), log: bids( root="logs", @@ -116,3 +169,21 @@ rule generate_subject_qc: ), script: "../scripts/generate_subject_qc.py" + + +rule generate_aggregate_qc: + input: + report_html=config["report"]["resources"]["report_html"], + subj_htmls=get_all_subj_html, + params: + datasets=datasets, + output: + total_html=remote_file(Path(root) / "qc" / "qc_report.html"), + log: + bids( + root="logs", + datatype="generate_aggregate_qc", + suffix="log.txt", + ), + script: + "../scripts/generate_aggregate_qc.py" diff --git a/workflow/scripts/generate_flatfield_qc.py b/workflow/scripts/generate_flatfield_qc.py index d1a6200..ac9f9d7 100644 --- a/workflow/scripts/generate_flatfield_qc.py +++ b/workflow/scripts/generate_flatfield_qc.py @@ -9,7 +9,7 @@ # load the html template from jinja file_loader = FileSystemLoader(".") env = Environment(loader=file_loader) -template = env.get_template("qc/resources/ff_html_temp.html") +template = env.get_template(snakemake.input.ff_html) # User set configurations ff_s_start=snakemake.params.ff_s_start diff --git a/workflow/scripts/generate_subject_qc.py b/workflow/scripts/generate_subject_qc.py index b7f7493..6c75542 100644 --- a/workflow/scripts/generate_subject_qc.py +++ b/workflow/scripts/generate_subject_qc.py @@ -1,10 +1,11 @@ from jinja2 import Environment, FileSystemLoader import os.path as path +from pathlib import Path # load jinja template file_loader = FileSystemLoader(".") env = Environment(loader=file_loader) -template = env.get_template("qc/resources/subject_html_temp.html") +template = env.get_template(snakemake.input.subject_html) # input html files ws_html = snakemake.input.ws_html @@ -13,7 +14,6 @@ # output html files sub_html = snakemake.output.sub_html -total_html = "qc/qc_report.html" # Wildcards subject = snakemake.wildcards.subject @@ -21,9 +21,10 @@ acq = snakemake.wildcards.acq # Get relative path to the subjects QC htmls -ws_rel_path = path.relpath(path.dirname(sub_html), path.dirname(ws_html))+"/"+path.basename(ws_html) -ff_rel_path = path.relpath(path.dirname(sub_html), path.dirname(ff_html))+"/"+path.basename(ff_html) -vol_rel_path = path.relpath(path.dirname(sub_html), path.dirname(vol_html))+ "/" +path.basename(vol_html) + +ws_rel_path = Path(ws_html).relative_to(Path(sub_html).parent) +ff_rel_path = Path(ff_html).relative_to(Path(sub_html).parent) +vol_rel_path = Path(vol_html).relative_to(Path(sub_html).parent) # Fill in jinja template for subject html and write it out output = template.render(back_link="../qc_report.html",subject=subject,sample=sample,acq=acq, @@ -31,18 +32,3 @@ with open(sub_html, 'w') as f: f.write(output) -# Create line to add link to subject into final qc report combining all subjects -sub_link = f'\n\t\t{subject}-{sample}-{acq}
' - -# if not first sample just add the one link -if(path.exists(total_html)): - with open(total_html,'a') as f: - f.write(sub_link) - -# if it is the first sample write out the template -else: - template = env.get_template("qc/resources/qc_report_temp.html") - output = template.render() - output+=sub_link - with open(total_html, 'w') as f: - f.write(output) diff --git a/workflow/scripts/generate_volume_qc.py b/workflow/scripts/generate_volume_qc.py index 2015dd5..1fb1398 100644 --- a/workflow/scripts/generate_volume_qc.py +++ b/workflow/scripts/generate_volume_qc.py @@ -10,7 +10,7 @@ import zarr # directory containing the volume rendering files -resource_dir = Path(snakemake.output.resources) +resource_dir = snakemake.output.resources # where html file should be written html_dest = snakemake.output.html @@ -18,8 +18,8 @@ ome_data = snakemake.input.ome # move volume renderer into the subjects directory -copy_tree("qc/resources/volViewer", str(resource_dir)) -shutil.move(resource_dir / "volRender.html", html_dest) +copy_tree(snakemake.input.vol_viewer_dir, resource_dir) +shutil.move(Path(resource_dir) / "volRender.html", html_dest) uri = snakemake.params.uri if is_remote(uri): @@ -43,7 +43,7 @@ ds_z = ds_z.downsample(along_z=downsample_factor) # Write it to a JSON for js script to read -with open(resource_dir / "volumeData.json", 'w') as f: +with open(Path(resource_dir) / "volumeData.json", 'w') as f: json_data = json.dumps(ds_z.darr.compute().tolist()) f.write(json_data) diff --git a/workflow/scripts/generate_whole_slice_qc.py b/workflow/scripts/generate_whole_slice_qc.py index c01d555..ed64b21 100644 --- a/workflow/scripts/generate_whole_slice_qc.py +++ b/workflow/scripts/generate_whole_slice_qc.py @@ -12,7 +12,7 @@ # load jinja html template file_loader = FileSystemLoader(".") env = Environment(loader=file_loader) -template = env.get_template("qc/resources/ws_html_temp.html") +template = env.get_template(snakemake.input.ws_html) # user set configurations ws_s_step=snakemake.params.ws_s_step