From d914662af21c31316abff2451e1d0af354233c9b Mon Sep 17 00:00:00 2001 From: Spiros Maggioros Date: Tue, 13 Aug 2024 15:37:43 +0300 Subject: [PATCH 1/3] Code Review and dependencies update The workflow uses a couple of packages like spare_score and DLMUSE. The 2 packages had conflicts before, right now we will try to solve them individually(the dependencies.txt will have the same ones for both DLMUSE and spare in the future), but snakemake builds fine with python 3.12 for now. The only thing that we have to do now is to add the nnUnet model weights directly. --- README.md | 7 +++---- utils/dlmuse/util_dlmuse_apply.sh | 18 ++++++++++++++++-- utils/spare/util_spare_test.sh | 15 +++++++++++++++ workflows/w_sMRI/Snakefile | 8 ++++---- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 911a857..289f6b7 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# **NiChartWorkflows** +# **NiChartWorkflows** A toolkit that allows users to apply **NiChart** processing methods and pre-trained models on their data. **NiChartWorkflows** includes tools for [data harmonization](https://neuroimagingchart.com/components/#Harmonization) and [machine learning](https://neuroimagingchart.com/components/#Machine%20Learning), using pre-trained models and reference distributions provided by [**NiChartEngine**](https://github.com/gurayerus/NiChart_Engine). **NiChartWorkflow** utilizes [Snakemake](https://snakemake.github.io) workflows to define multi-step image processing and data analytics pipelines. @@ -13,7 +13,7 @@ Install dependencies for the **NiChartWorkflows** in a conda environm ```bash conda install -n base -c conda-forge mamba # If needed mamba init # If first time installing, then potentially source ~/.bashrc -mamba create -c conda-forge -c bioconda -n NiChart_Workflows python=3.8 +mamba create -c conda-forge -c bioconda -n NiChart_Workflows python=3.12 mamba activate NiChart_Workflows pip install NiChart-Viewer-Demo spare-scores NiChartHarmonize mamba install -c conda-forge -c bioconda snakemake @@ -58,7 +58,7 @@ If the workflow works successfully, workflow steps will be run on the data. Resu > "../../output/vTest1", -and the viewer will be launched with the combined result file as input. +and the viewer will be launched with the combined result file as input. ## Contributing: @@ -76,4 +76,3 @@ This project is licensed under the [License Name] license. Please refer to the L ## Contact: For any inquiries, please contact guray.erus@pennmedicine.upenn.edu. (Last Updated: 5/24/2024) - diff --git a/utils/dlmuse/util_dlmuse_apply.sh b/utils/dlmuse/util_dlmuse_apply.sh index 4660d49..d59730b 100644 --- a/utils/dlmuse/util_dlmuse_apply.sh +++ b/utils/dlmuse/util_dlmuse_apply.sh @@ -1,5 +1,19 @@ #! /bin/bash -x +# download NiChart DLMUSE package +echo "Downloading NiChart DLMUSE..." +# conda create --name NCP python=3.9 +# conda activate NCP +if [ -d "/NiChart_DLMUSE" ] +then + echo "Clone already exists!" +else + git clone https://github.com/CBICA/NiChart_DLMUSE.git +fi +cd NiChart_DLMUSE +pip install . +cd .. +echo "NiChart DLMUSE downloaded!" echo "About to run: $0 $@" ## Read input @@ -16,8 +30,8 @@ mkdir -pv "${tmp_dir}/nnUNet_raw_database" mkdir -pv "${tmp_dir}/nnUNet_out" ln -s `realpath ../../../../NiChart/NiChart_Models/nnUNet_model` ${tmp_dir}/nnUNet_model ln -s `realpath ${in_dir}/Images` "${tmp_dir}/nnUNet_raw_database/nnUNet_raw_data" -droi=`realpath ../../../../NiChart/NiChart_DLMUSE/shared/dicts/MUSE_mapping_derived_rois.csv` -roi=`realpath ../../../../NiChart/NiChart_DLMUSE/shared/dicts/MUSE_mapping_consecutive_indices.csv` +droi=`realpath NiChart_DLMUSE/shared/dicts/MUSE_mapping_derived_rois.csv` +roi=`realpath NiChart_DLMUSE/shared/dicts/MUSE_mapping_consecutive_indices.csv` ## Apply dlmuse test cmd="NiChart_DLMUSE --indir ${tmp_dir}/nnUNet_raw_database/nnUNet_raw_data --outdir ${tmp_dir}/nnUNet_out --pipelinetype structural --derived_ROI_mappings_file $droi --MUSE_ROI_mappings_file $roi --nnUNet_raw_data_base ${tmp_dir}/nnUNet_raw_database --nnUNet_preprocessed ${tmp_dir}/nnUNet_preprocessed --model_folder ${tmp_dir}/nnUNet_model --all_in_gpu True --mode fastest --disable_tta" diff --git a/utils/spare/util_spare_test.sh b/utils/spare/util_spare_test.sh index 0b34120..ca0f807 100755 --- a/utils/spare/util_spare_test.sh +++ b/utils/spare/util_spare_test.sh @@ -1,5 +1,20 @@ #! /bin/bash +# download spare package +echo "Downloading spare score package..." +if [ -d "/spare_score" ] +then + echo "spare score clone already exists!" +else + git clone https://github.com/CBICA/spare_score.git +fi +python setup.py bdist_wheel +cd dist +WHEEL_FILE=$(ls spare_scores*) +pip install "$WHEEL_FILE" +cd .. +echo "spare score is downloaded!" + ## Read input in_csv=$1 in_mdl=$2 diff --git a/workflows/w_sMRI/Snakefile b/workflows/w_sMRI/Snakefile index c66a30f..6752c7e 100755 --- a/workflows/w_sMRI/Snakefile +++ b/workflows/w_sMRI/Snakefile @@ -9,10 +9,10 @@ import os #report: "report/workflow.rst" ################################### -## Default config file +## Default config file ## Runs the workflow with the test dataset ## Edit this for running it on other datasets -in_config = "../../config/vTest1/config.yaml" +in_config = "../../config/vTest2/config.yaml" #in_config = "../../config/vTest2/config.yaml" configfile: f"{in_config}" ################################### @@ -192,7 +192,7 @@ rule calc_derived_ROIs: ''' Calculate derived ROIs from harmonized data - If the input data already includes derived ROIs, they will not be updated - ''' + ''' input: in_csv=f"../../{dir_output}/{{study}}/out_combat/{{study}}_{{dtype}}_COMBAT_single.csv", dict=f"../../{derived_rois}" @@ -293,7 +293,7 @@ rule launch_viewer: touch(f"../../{dir_output}/flag_VIEWED.csv"), shell: "NiChart_Viewer --data_file {input}" - + ## Delete flag for the viewer, so that it will launch the viewer next time if os.path.exists(f"../../{dir_output}/flag_VIEWED.csv"): os.remove(f"../../{dir_output}/flag_VIEWED.csv") From 441a543fb864fc81a8f9361bf092bf45c4b2ba76 Mon Sep 17 00:00:00 2001 From: Spiros Maggioros Date: Tue, 13 Aug 2024 19:21:57 +0300 Subject: [PATCH 2/3] remove unused lines of code --- utils/dlmuse/util_dlmuse_apply.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils/dlmuse/util_dlmuse_apply.sh b/utils/dlmuse/util_dlmuse_apply.sh index d59730b..855aebd 100644 --- a/utils/dlmuse/util_dlmuse_apply.sh +++ b/utils/dlmuse/util_dlmuse_apply.sh @@ -2,8 +2,6 @@ # download NiChart DLMUSE package echo "Downloading NiChart DLMUSE..." -# conda create --name NCP python=3.9 -# conda activate NCP if [ -d "/NiChart_DLMUSE" ] then echo "Clone already exists!" From 8ffa779fb79c1de44cf51036b5d81561bb57271f Mon Sep 17 00:00:00 2001 From: Spiros Maggioros Date: Tue, 13 Aug 2024 20:53:05 +0300 Subject: [PATCH 3/3] realpath bug fix --- input/vTest2/Study1/Images/Images | 1 + utils/dlmuse/util_dlmuse_apply.sh | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 120000 input/vTest2/Study1/Images/Images diff --git a/input/vTest2/Study1/Images/Images b/input/vTest2/Study1/Images/Images new file mode 120000 index 0000000..321316b --- /dev/null +++ b/input/vTest2/Study1/Images/Images @@ -0,0 +1 @@ +/Users/spirosmag/Documents/NiChart_Workflows/input/vTest2/Study1/Images \ No newline at end of file diff --git a/utils/dlmuse/util_dlmuse_apply.sh b/utils/dlmuse/util_dlmuse_apply.sh index 855aebd..9ed2059 100644 --- a/utils/dlmuse/util_dlmuse_apply.sh +++ b/utils/dlmuse/util_dlmuse_apply.sh @@ -14,9 +14,10 @@ cd .. echo "NiChart DLMUSE downloaded!" echo "About to run: $0 $@" + ## Read input in_csv=$(realpath $1) -out_csv=$(realpath $2) +out_csv=$2 ## Prep init data in_dir=$(dirname ${in_csv}) @@ -26,7 +27,7 @@ tmp_dir="${out_dir}/tmprun_dlmuse" mkdir -pv "${tmp_dir}/nnUNet_preprocessed" mkdir -pv "${tmp_dir}/nnUNet_raw_database" mkdir -pv "${tmp_dir}/nnUNet_out" -ln -s `realpath ../../../../NiChart/NiChart_Models/nnUNet_model` ${tmp_dir}/nnUNet_model +ln -s `realpath ./nnUNet_model` "${tmp_dir}/nnUNet_model" ln -s `realpath ${in_dir}/Images` "${tmp_dir}/nnUNet_raw_database/nnUNet_raw_data" droi=`realpath NiChart_DLMUSE/shared/dicts/MUSE_mapping_derived_rois.csv` roi=`realpath NiChart_DLMUSE/shared/dicts/MUSE_mapping_consecutive_indices.csv`