diff --git a/config/config.yml b/config/config.yml
index 2c50355..6f06925 100644
--- a/config/config.yml
+++ b/config/config.yml
@@ -140,6 +140,13 @@ report:
slice_start: 0
slice_step: 50
colour_map: viridis
+ resources:
+ readme_md: resources/qc/README.md
+ vol_viewer_dir: resources/qc/volViewer
+ ff_html: resources/qc/ff_html_temp.html
+ report_html: resources/qc/qc_report_temp.html
+ subject_html: resources/qc/subject_html_temp.html
+ ws_html: resources/qc/ws_html_temp.html
containers:
diff --git a/qc/README.md b/resources/qc/README.md
similarity index 58%
rename from qc/README.md
rename to resources/qc/README.md
index 400b91e..a77594a 100644
--- a/qc/README.md
+++ b/resources/qc/README.md
@@ -2,18 +2,13 @@
### 1: Run the workflow with personalized report generation configurations
-### 2: Navigate to qc directory:
-
-```bash
-cd ./qc
-```
-### 3: Create a Python web server:
+### 2: Navigate to qc directory, then create a Python web server:
```bash
python -m http.server
```
-### 4: Open the link generated in a browser and open qc_report.html
+### 3: Open the link generated in a browser and open `qc_report.html`
### Note:
- Can view the whole slice images and the flatfield corrections without running the web server. However, to view the volume rendered brain, the web server is required.
\ No newline at end of file
+ Can view the whole slice images and the flatfield corrections without running the web server. However, to view the volume rendered brain, the web server is required.
diff --git a/qc/resources/ff_html_temp.html b/resources/qc/ff_html_temp.html
similarity index 100%
rename from qc/resources/ff_html_temp.html
rename to resources/qc/ff_html_temp.html
diff --git a/qc/resources/qc_report_temp.html b/resources/qc/qc_report_temp.html
similarity index 100%
rename from qc/resources/qc_report_temp.html
rename to resources/qc/qc_report_temp.html
diff --git a/qc/resources/subject_html_temp.html b/resources/qc/subject_html_temp.html
similarity index 100%
rename from qc/resources/subject_html_temp.html
rename to resources/qc/subject_html_temp.html
diff --git a/qc/resources/volViewer/cm_gray.png b/resources/qc/volViewer/cm_gray.png
similarity index 100%
rename from qc/resources/volViewer/cm_gray.png
rename to resources/qc/volViewer/cm_gray.png
diff --git a/qc/resources/volViewer/cm_viridis.png b/resources/qc/volViewer/cm_viridis.png
similarity index 100%
rename from qc/resources/volViewer/cm_viridis.png
rename to resources/qc/volViewer/cm_viridis.png
diff --git a/qc/resources/volViewer/volRender.html b/resources/qc/volViewer/volRender.html
similarity index 100%
rename from qc/resources/volViewer/volRender.html
rename to resources/qc/volViewer/volRender.html
diff --git a/qc/resources/volViewer/volRenderScript.js b/resources/qc/volViewer/volRenderScript.js
similarity index 100%
rename from qc/resources/volViewer/volRenderScript.js
rename to resources/qc/volViewer/volRenderScript.js
diff --git a/qc/resources/ws_html_temp.html b/resources/qc/ws_html_temp.html
similarity index 100%
rename from qc/resources/ws_html_temp.html
rename to resources/qc/ws_html_temp.html
diff --git a/workflow/Snakefile b/workflow/Snakefile
index 85ecad7..155244d 100644
--- a/workflow/Snakefile
+++ b/workflow/Snakefile
@@ -33,6 +33,7 @@ rule all:
input:
get_all_targets(),
get_bids_toplevel_targets(),
+ get_qc_targets(),
localrule: True
diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk
index 4e52bc7..e6b7ad0 100644
--- a/workflow/rules/common.smk
+++ b/workflow/rules/common.smk
@@ -29,6 +29,20 @@ def expand_bids(expand_kwargs, **bids_kwargs):
return files
+def remote_file(filename):
+ if is_remote(filename):
+ return storage(str(filename))
+ else:
+ return filename
+
+
+def remote_directory(dirname):
+ if is_remote(dirname):
+ return storage(directory(str(dirname)))
+ else:
+ return directory(dirname)
+
+
def directory_bids(root, *args, **kwargs):
"""Similar to expand_bids, this replacement function
is needed to ensure storage() comes after directory() tags"""
@@ -107,18 +121,25 @@ def get_all_targets():
),
)
)
- if config["report"]["create_report"]:
- targets.extend(
- expand(
- "qc/sub-{subject}_sample-{sample}_acq-{acq}/subject.html",
- subject=datasets.loc[i, "subject"],
- sample=datasets.loc[i, "sample"],
- acq=datasets.loc[i, "acq"],
- )
- )
return targets
+def get_all_subj_html(wildcards):
+ htmls = []
+
+ for i in range(len(datasets)):
+
+ html = "{root}/qc/sub-{subject}_sample-{sample}_acq-{acq}/subject.html".format(
+ root=root,
+ subject=datasets.loc[i, "subject"],
+ sample=datasets.loc[i, "sample"],
+ acq=datasets.loc[i, "acq"],
+ )
+ htmls.append(remote_file(html))
+
+ return htmls
+
+
def get_bids_toplevel_targets():
targets = []
targets.append(bids_toplevel(root, "README.md"))
@@ -129,6 +150,14 @@ def get_bids_toplevel_targets():
return targets
+def get_qc_targets():
+ targets = []
+ if config["report"]["create_report"]:
+ targets.append(remote_file(Path(root) / "qc" / "qc_report.html"))
+ targets.append(remote_file(Path(root) / "qc" / "README.md"))
+ return targets
+
+
def dataset_is_remote(wildcards):
return is_remote_gcs(Path(get_dataset_path(wildcards)))
diff --git a/workflow/rules/qc.smk b/workflow/rules/qc.smk
index 415b3d7..cb44298 100644
--- a/workflow/rules/qc.smk
+++ b/workflow/rules/qc.smk
@@ -1,3 +1,16 @@
+
+rule setup_qc_dir:
+ "Copies QC resources to the output bids folder"
+ input:
+ readme_md=config["report"]["resources"]["readme_md"],
+ output:
+ readme_md=remote_file(Path(root) / "qc" / "README.md"),
+ log:
+ "logs/setup_qc_dir_log.txt",
+ shell:
+ "cp {input.readme_md} {output.readme_md}"
+
+
rule generate_flatfield_qc:
"Generates an html file for comparing before and after flatfield correction"
input:
@@ -19,17 +32,30 @@ rule generate_flatfield_qc:
desc="flatcorr",
suffix="SPIM.zarr",
),
+ ff_html=config["report"]["resources"]["ff_html"],
params:
ff_s_start=config["report"]["flatfield_corrected"]["slice_start"],
ff_s_step=config["report"]["flatfield_corrected"]["slice_step"],
ff_cmap=config["report"]["flatfield_corrected"]["colour_map"],
output:
- html="qc/sub-{subject}_sample-{sample}_acq-{acq}/flatfieldqc.html",
- corr_images_dir=directory(
- "qc/sub-{subject}_sample-{sample}_acq-{acq}/images/corr"
+ html=remote_file(
+ Path(root)
+ / "qc"
+ / "sub-{subject}_sample-{sample}_acq-{acq}/flatfieldqc.html"
),
- uncorr_images_dir=directory(
- "qc/sub-{subject}_sample-{sample}_acq-{acq}/images/uncorr"
+ corr_images_dir=remote_directory(
+ Path(root)
+ / "qc"
+ / "sub-{subject}_sample-{sample}_acq-{acq}"
+ / "images"
+ / "corr"
+ ),
+ uncorr_images_dir=remote_directory(
+ Path(root)
+ / "qc"
+ / "sub-{subject}_sample-{sample}_acq-{acq}"
+ / "images"
+ / "uncorr"
),
log:
bids(
@@ -49,6 +75,7 @@ rule generate_whole_slice_qc:
input:
**get_storage_creds(),
ome=get_input_ome_zarr_to_nii(),
+ ws_html=config["report"]["resources"]["ws_html"],
params:
ws_s_start=config["report"]["whole_slice_viewer"]["slice_start"],
ws_s_step=config["report"]["whole_slice_viewer"]["slice_step"],
@@ -56,8 +83,19 @@ rule generate_whole_slice_qc:
uri=get_output_ome_zarr_uri(),
storage_provider_settings=workflow.storage_provider_settings,
output:
- html="qc/sub-{subject}_sample-{sample}_acq-{acq}/whole_slice_qc.html",
- images_dir=directory("qc/sub-{subject}_sample-{sample}_acq-{acq}/images/whole"),
+ html=remote_file(
+ Path(root)
+ / "qc"
+ / "sub-{subject}_sample-{sample}_acq-{acq}"
+ / "whole_slice_qc.html"
+ ),
+ images_dir=remote_directory(
+ Path(root)
+ / "qc"
+ / "sub-{subject}_sample-{sample}_acq-{acq}"
+ / "images"
+ / "whole"
+ ),
log:
bids(
root="logs",
@@ -76,14 +114,23 @@ rule generate_volume_qc:
input:
**get_storage_creds(),
ome=get_input_ome_zarr_to_nii(),
+ vol_viewer_dir=config["report"]["resources"]["vol_viewer_dir"],
params:
uri=get_output_ome_zarr_uri(),
storage_provider_settings=workflow.storage_provider_settings,
output:
- resources=directory(
- "qc/sub-{subject}_sample-{sample}_acq-{acq}/volume_resources"
+ resources=remote_directory(
+ Path(root)
+ / "qc"
+ / "sub-{subject}_sample-{sample}_acq-{acq}"
+ / "volume_resources"
+ ),
+ html=remote_file(
+ Path(root)
+ / "qc"
+ / "sub-{subject}_sample-{sample}_acq-{acq}"
+ / "volume_qc.html"
),
- html="qc/sub-{subject}_sample-{sample}_acq-{acq}/volume_qc.html",
log:
bids(
root="logs",
@@ -100,11 +147,17 @@ rule generate_volume_qc:
rule generate_subject_qc:
"Generates html files to access all the subjects qc reports in one place"
input:
- ws_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/whole_slice_qc.html",
- ff_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/flatfieldqc.html",
- vol_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/volume_qc.html",
+ subject_html=config["report"]["resources"]["subject_html"],
+ ws_html=rules.generate_whole_slice_qc.output.html,
+ ff_html=rules.generate_flatfield_qc.output.html,
+ vol_html=rules.generate_volume_qc.output.html,
output:
- sub_html="qc/sub-{subject}_sample-{sample}_acq-{acq}/subject.html",
+ sub_html=remote_file(
+ Path(root)
+ / "qc"
+ / "sub-{subject}_sample-{sample}_acq-{acq}"
+ / "subject.html"
+ ),
log:
bids(
root="logs",
@@ -116,3 +169,21 @@ rule generate_subject_qc:
),
script:
"../scripts/generate_subject_qc.py"
+
+
+rule generate_aggregate_qc:
+ input:
+ report_html=config["report"]["resources"]["report_html"],
+ subj_htmls=get_all_subj_html,
+ params:
+ datasets=datasets,
+ output:
+ total_html=remote_file(Path(root) / "qc" / "qc_report.html"),
+ log:
+ bids(
+ root="logs",
+ datatype="generate_aggregate_qc",
+ suffix="log.txt",
+ ),
+ script:
+ "../scripts/generate_aggregate_qc.py"
diff --git a/workflow/scripts/generate_flatfield_qc.py b/workflow/scripts/generate_flatfield_qc.py
index d1a6200..ac9f9d7 100644
--- a/workflow/scripts/generate_flatfield_qc.py
+++ b/workflow/scripts/generate_flatfield_qc.py
@@ -9,7 +9,7 @@
# load the html template from jinja
file_loader = FileSystemLoader(".")
env = Environment(loader=file_loader)
-template = env.get_template("qc/resources/ff_html_temp.html")
+template = env.get_template(snakemake.input.ff_html)
# User set configurations
ff_s_start=snakemake.params.ff_s_start
diff --git a/workflow/scripts/generate_subject_qc.py b/workflow/scripts/generate_subject_qc.py
index b7f7493..6c75542 100644
--- a/workflow/scripts/generate_subject_qc.py
+++ b/workflow/scripts/generate_subject_qc.py
@@ -1,10 +1,11 @@
from jinja2 import Environment, FileSystemLoader
import os.path as path
+from pathlib import Path
# load jinja template
file_loader = FileSystemLoader(".")
env = Environment(loader=file_loader)
-template = env.get_template("qc/resources/subject_html_temp.html")
+template = env.get_template(snakemake.input.subject_html)
# input html files
ws_html = snakemake.input.ws_html
@@ -13,7 +14,6 @@
# output html files
sub_html = snakemake.output.sub_html
-total_html = "qc/qc_report.html"
# Wildcards
subject = snakemake.wildcards.subject
@@ -21,9 +21,10 @@
acq = snakemake.wildcards.acq
# Get relative path to the subjects QC htmls
-ws_rel_path = path.relpath(path.dirname(sub_html), path.dirname(ws_html))+"/"+path.basename(ws_html)
-ff_rel_path = path.relpath(path.dirname(sub_html), path.dirname(ff_html))+"/"+path.basename(ff_html)
-vol_rel_path = path.relpath(path.dirname(sub_html), path.dirname(vol_html))+ "/" +path.basename(vol_html)
+
+ws_rel_path = Path(ws_html).relative_to(Path(sub_html).parent)
+ff_rel_path = Path(ff_html).relative_to(Path(sub_html).parent)
+vol_rel_path = Path(vol_html).relative_to(Path(sub_html).parent)
# Fill in jinja template for subject html and write it out
output = template.render(back_link="../qc_report.html",subject=subject,sample=sample,acq=acq,
@@ -31,18 +32,3 @@
with open(sub_html, 'w') as f:
f.write(output)
-# Create line to add link to subject into final qc report combining all subjects
-sub_link = f'\n\t\t{subject}-{sample}-{acq}
'
-
-# if not first sample just add the one link
-if(path.exists(total_html)):
- with open(total_html,'a') as f:
- f.write(sub_link)
-
-# if it is the first sample write out the template
-else:
- template = env.get_template("qc/resources/qc_report_temp.html")
- output = template.render()
- output+=sub_link
- with open(total_html, 'w') as f:
- f.write(output)
diff --git a/workflow/scripts/generate_volume_qc.py b/workflow/scripts/generate_volume_qc.py
index 2015dd5..1fb1398 100644
--- a/workflow/scripts/generate_volume_qc.py
+++ b/workflow/scripts/generate_volume_qc.py
@@ -10,7 +10,7 @@
import zarr
# directory containing the volume rendering files
-resource_dir = Path(snakemake.output.resources)
+resource_dir = snakemake.output.resources
# where html file should be written
html_dest = snakemake.output.html
@@ -18,8 +18,8 @@
ome_data = snakemake.input.ome
# move volume renderer into the subjects directory
-copy_tree("qc/resources/volViewer", str(resource_dir))
-shutil.move(resource_dir / "volRender.html", html_dest)
+copy_tree(snakemake.input.vol_viewer_dir, resource_dir)
+shutil.move(Path(resource_dir) / "volRender.html", html_dest)
uri = snakemake.params.uri
if is_remote(uri):
@@ -43,7 +43,7 @@
ds_z = ds_z.downsample(along_z=downsample_factor)
# Write it to a JSON for js script to read
-with open(resource_dir / "volumeData.json", 'w') as f:
+with open(Path(resource_dir) / "volumeData.json", 'w') as f:
json_data = json.dumps(ds_z.darr.compute().tolist())
f.write(json_data)
diff --git a/workflow/scripts/generate_whole_slice_qc.py b/workflow/scripts/generate_whole_slice_qc.py
index c01d555..ed64b21 100644
--- a/workflow/scripts/generate_whole_slice_qc.py
+++ b/workflow/scripts/generate_whole_slice_qc.py
@@ -12,7 +12,7 @@
# load jinja html template
file_loader = FileSystemLoader(".")
env = Environment(loader=file_loader)
-template = env.get_template("qc/resources/ws_html_temp.html")
+template = env.get_template(snakemake.input.ws_html)
# user set configurations
ws_s_step=snakemake.params.ws_s_step