Added tests, remove uploading and data collection from organiser to s…

…eparate repository

Dockerfile

@@ -1,11 +1,17 @@
 FROM bitnami/python:3.10
 
+RUN mkdir /organisation-app
+
+WORKDIR /organisation-app
+
 ADD requirements.txt .
+ADD main.py .
+ADD organiser/ organiser/
+ADD tests/ tests/
 
 RUN pip install -r requirements.txt
 
-RUN mkdir /scripts
-
-COPY MiSEQ/* /scripts/
+# USER 1005
+USER 1001
 
-USER 1005
+CMD ["pytest", "tests"]

compose.yaml

@@ -2,17 +2,11 @@ version: '3.0'
 services:
   run:
     build: .
-    env_file:
-      - .env
     volumes:
-      - /home/houfek/Work/MMCI/data-catalogue-playground/muni-sc/MiSEQ:/pseudonymized
-      - /home/houfek/Work/MMCI/data-catalogue-playground/muni-sc/OrganisedRuns:/OrganisedRuns
-      - /home/houfek/Work/MMCI/data-catalogue-playground/muni-sc/Patients:/Patients
-      - /home/houfek/Work/MMCI/data-catalogue-playground/muni-sc/WSI:/WSI
-      - /home/houfek/Work/MMCI/data-catalogue-playground/muni-sc/Libraries:/Libraries
-    command: bash -c "python  /scripts/pipeline.py
-                          -r /pseudonymized/
-                          -o /OrganisedRuns/
-                          -p /Patients/
-                          -w /WSI/
-                          -d /Libraries/"
+      - /home/houfek/Work/MMCI/sequencing_pipeline/data-catalogue-playground/muni-sc/MiSEQ/:/PseudonymizedRuns
+      - /home/houfek/Work/MMCI/sequencing_pipeline/data-catalogue-playground/muni-sc/OrganisedRuns/:/OrganisedRuns
+      - /home/houfek/Work/MMCI/sequencing_pipeline/data-catalogue-playground/muni-sc/Patients/:/Patients
+    command: bash -c "python main.py
+                              -r /PseudonymizedRuns/
+                              -o /OrganisedRuns/
+                              -p /Patients/"

main.py

@@ -0,0 +1,51 @@
+import argparse
+import os
+import shutil
+import logging
+from datetime import datetime
+from organiser.organise_run import RunOrganiser
+from organiser.helpers.file_helpers import create_dictionary_if_not_exist
+
+
+def _create_important_folders_if_not_exist(organisation_folder):
+    create_dictionary_if_not_exist(os.path.join(organisation_folder, "logs"))
+    create_dictionary_if_not_exist(os.path.join(organisation_folder, "backups"))
+    create_dictionary_if_not_exist(os.path.join(organisation_folder, "errors"))
+
+
+def run(path_to_runs_for_processing, path_to_organisation_folder, path_to_patient_organisation_folder):
+    _create_important_folders_if_not_exist(path_to_organisation_folder)
+    logging.basicConfig(filename=os.path.join(path_to_organisation_folder, "logs",
+                                              datetime.now().strftime('%d_%m_%Y-%H_%M.log')),
+                        encoding='utf-8',
+                        level=logging.INFO)
+
+    for file in os.listdir(path_to_runs_for_processing):
+        logging.info(f"Organising: {file}")
+        try:
+            RunOrganiser(path_to_runs_for_processing, file,
+                         path_to_organisation_folder, path_to_patient_organisation_folder)()
+        except FileNotFoundError as e:
+            logging.error(f"Run {file} is missing some data\nError:\n{e}")
+            shutil.move(os.path.join(path_to_runs_for_processing, file),
+                        os.path.join(path_to_organisation_folder, "errors", file))
+            continue
+
+        shutil.move(os.path.join(path_to_runs_for_processing, file),
+                    os.path.join(path_to_organisation_folder, "backups", file))
+        logging.info("File moved into backups")
+
+    logging.info("Done!")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog="Organiser",
+                                     description="Organise pseudonymized runs into a specifed output folder \n." +
+                                                 " It is important to use full paths")
+
+    parser.add_argument("-r", "--runs", type=str, required=True, help="Path to pseudonymized runs")
+    parser.add_argument("-o", "--output", type=str, required=True, help="Path to the organise file")
+    parser.add_argument("-p", "--patients", type=str, required=True, help="Path to a patient folder")
+    args = parser.parse_args()
+
+    run(args.runs, args.output, args.patients)

organiser/helpers/file_helpers.py

@@ -0,0 +1,22 @@
+import os
+import shutil
+import logging
+
+
+def create_dictionary_if_not_exist(path_to_create):
+    if not os.path.exists(path_to_create):
+        os.mkdir(path_to_create)
+
+
+def copy_if_exists(orig_path, new_path):
+    if os.path.exists(orig_path):
+        shutil.copy2(orig_path, new_path)
+    else:
+        logging.warning(f"Path {orig_path} does not exist!")
+
+
+def copy_folder_if_exists(orig_path, new_path):
+    if os.path.exists(orig_path):
+        shutil.copytree(orig_path, new_path)
+    else:
+        logging.warning(f"Path {orig_path} does not exist!")