From dc2fafacd818f33d040033c7b023574030b21607 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Fri, 6 Jul 2018 14:56:01 -0400 Subject: [PATCH 01/60] Working nginx singularity container --- .gitignore | 3 + imports/docker/Dockerfile_nginx | 101 ++++++++++++++++++ imports/docker/entrypoint_nginx.sh | 5 + imports/docker/settings/auth.conf | 12 +++ imports/docker/settings/auth.htpasswd | 1 + imports/docker/settings/log/access.log | 2 + imports/docker/settings/log/error.log | 0 imports/docker/settings/log/nginx_error.log | 2 + .../docker/settings/mc_nginx_settings.json | 1 + imports/docker/settings/meteor.conf | 23 ++++ imports/docker/settings/nginx.conf | 31 ++++++ 11 files changed, 181 insertions(+) create mode 100644 imports/docker/Dockerfile_nginx create mode 100644 imports/docker/entrypoint_nginx.sh create mode 100755 imports/docker/settings/auth.conf create mode 100755 imports/docker/settings/auth.htpasswd create mode 100644 imports/docker/settings/log/access.log create mode 100644 imports/docker/settings/log/error.log create mode 100644 imports/docker/settings/log/nginx_error.log create mode 100644 imports/docker/settings/mc_nginx_settings.json create mode 100644 imports/docker/settings/meteor.conf create mode 100755 imports/docker/settings/nginx.conf diff --git a/.gitignore b/.gitignore index b42c133..353bbbe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ node_modules/ .meteor/dev_bundle +.mindcontrol/ +imports/docker/log/ +imports/docker/temp_for_nginx/ \ No newline at end of file diff --git a/imports/docker/Dockerfile_nginx b/imports/docker/Dockerfile_nginx new file mode 100644 index 0000000..39ea4a3 --- /dev/null +++ b/imports/docker/Dockerfile_nginx @@ -0,0 +1,101 @@ +FROM nginx:1.15.1 + +MAINTAINER Dylan Nielson + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + vim \ + supervisor \ + && rm -rf /var/lib/apt/lists/* + +ENV METEOR_RELEASE 1.7.0.3 +RUN curl https://install.meteor.com/ 2>/dev/null | sed 's/^RELEASE/#RELEASE/'| RELEASE=$METEOR_RELEASE sh + +RUN ln -s ~/.meteor/packages/meteor-tool/*/mt-os.linux.x86_64/dev_bundle/bin/node /usr/bin/ && \ + ln -s ~/.meteor/packages/meteor-tool/*/mt-os.linux.x86_64/dev_bundle/bin/npm /usr/bin/ && \ + rm /etc/nginx/conf.d/default.conf + + +# Installing and setting up miniconda +RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash Miniconda*.sh -b -p /usr/local/miniconda && \ + rm Miniconda*.sh + +ENV PATH=/usr/local/miniconda/bin:$PATH \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 + +# Installing precomputed python packages +RUN conda install -c conda-forge -y \ + awscli \ + boto3 \ + dipy \ + git \ + matplotlib \ + numpy \ + python=3.6 \ + scikit-image \ + scikit-learn \ + wget; \ + sync && \ + chmod +x /usr/local/miniconda/bin/* && \ + conda clean --all -y; sync && \ + python -c "from matplotlib import font_manager" && \ + sed -i 's/\(backend *: \).*$/\1Agg/g' $( python -c "import matplotlib; print(matplotlib.matplotlib_fname())" ) + +RUN npm install http-server -g + +ENV MC_DIR /home/mindcontrol +ENV LC_ALL C + + +COPY entrypoint_nginx.sh /home/entrypoint.sh +COPY ndmg_launch.sh /home/ndmg_launch.sh + +RUN useradd --create-home --home-dir ${MC_DIR} mindcontrol +RUN chown mindcontrol:mindcontrol /home/entrypoint.sh +RUN chmod +x /home/entrypoint.sh + +RUN mkdir -p ${MC_DIR}/mindcontrol &&\ + chown -R mindcontrol /home/mindcontrol &&\ + chmod -R a+rx /home/mindcontrol + +USER mindcontrol + +RUN cd ${MC_DIR}/mindcontrol &&\ + git clone https://github.com/akeshavan/mindcontrol.git ${MC_DIR}/mindcontrol + #git clone https://github.com/clowdcontrol/mindcontrol.git ${MC_DIR}/mindcontrol + + +####### Attempt at nginx security + +WORKDIR /opt +COPY ./settings/meteor.conf /etc/nginx/conf.d/meteor.conf +COPY ./settings/auth.htpasswd /etc/nginx +COPY ./settings/nginx.conf /etc/nginx/nginx.conf +USER root +RUN chmod 777 /var/log/nginx /var/cache/nginx /opt /etc +RUN mkdir -p /mc_data /mc_startup_data +RUN chown mindcontrol:mindcontrol /etc/nginx/auth.htpasswd /mc_data /mc_startup_data +USER mindcontrol +###### + +###### Load in local settings +COPY ./settings/mc_nginx_settings.json ${MC_DIR}/mindcontrol/mc_nginx_settings.json + +# Make a spot that we can put our data +RUN mkdir -p /mc_data /mc_startup_data + +WORKDIR ${MC_DIR}/mindcontrol + +ENTRYPOINT ["/home/entrypoint.sh"] + + +EXPOSE 3000 +EXPOSE 2998 +ENV PORT 3000 + + diff --git a/imports/docker/entrypoint_nginx.sh b/imports/docker/entrypoint_nginx.sh new file mode 100644 index 0000000..68a0a04 --- /dev/null +++ b/imports/docker/entrypoint_nginx.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd /home/mindcontrol/mindcontrol +nohup meteor --settings mc_nginx_settings.json --port 2998 & +nginx -g "daemon off;" diff --git a/imports/docker/settings/auth.conf b/imports/docker/settings/auth.conf new file mode 100755 index 0000000..5ad4cc0 --- /dev/null +++ b/imports/docker/settings/auth.conf @@ -0,0 +1,12 @@ +error_log /var/log/nginx/nginx_error.log info; + +server { + listen 3002 default_server; + root /data; + auth_basic "Restricted"; + auth_basic_user_file auth.htpasswd; + location / { + autoindex on; + } + + } diff --git a/imports/docker/settings/auth.htpasswd b/imports/docker/settings/auth.htpasswd new file mode 100755 index 0000000..8082836 --- /dev/null +++ b/imports/docker/settings/auth.htpasswd @@ -0,0 +1 @@ +dylan:$apr1$ieFel5ou$qOZXoFEVATdKZs9J/dKdA0 diff --git a/imports/docker/settings/log/access.log b/imports/docker/settings/log/access.log new file mode 100644 index 0000000..d3b047f --- /dev/null +++ b/imports/docker/settings/log/access.log @@ -0,0 +1,2 @@ +172.17.0.1 - - [03/Jul/2018:15:00:10 +0000] "GET / HTTP/1.1" 401 597 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36" "-" +172.17.0.1 - - [03/Jul/2018:15:00:11 +0000] "GET / HTTP/1.1" 401 597 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36" "-" diff --git a/imports/docker/settings/log/error.log b/imports/docker/settings/log/error.log new file mode 100644 index 0000000..e69de29 diff --git a/imports/docker/settings/log/nginx_error.log b/imports/docker/settings/log/nginx_error.log new file mode 100644 index 0000000..a8d79cb --- /dev/null +++ b/imports/docker/settings/log/nginx_error.log @@ -0,0 +1,2 @@ +2018/07/03 15:00:10 [info] 7#7: *1 no user/password was provided for basic authentication, client: 172.17.0.1, server: , request: "GET / HTTP/1.1", host: "localhost:3002" +2018/07/03 15:00:11 [info] 7#7: *1 no user/password was provided for basic authentication, client: 172.17.0.1, server: , request: "GET / HTTP/1.1", host: "localhost:3002" diff --git a/imports/docker/settings/mc_nginx_settings.json b/imports/docker/settings/mc_nginx_settings.json new file mode 100644 index 0000000..e2a5a7d --- /dev/null +++ b/imports/docker/settings/mc_nginx_settings.json @@ -0,0 +1 @@ +{"public": {"startup_json": "http://localhost:3003/startup.freesurfer.json", "load_if_empty": true, "use_custom": true, "needs_consent": false, "modules": [{"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "aparcaseg", "entry_type": "aparcaseg", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "custom.Freesurfer", "alpha": 0.5}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "wm", "entry_type": "wm", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Spectrum", "alpha": 0.5}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "brainmask", "entry_type": "brainmask", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Red Overlay", "alpha": 0.2, "min": 0, "max": 2000}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "ribbon", "entry_type": "ribbon", "num_overlays": 3, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Green Overlay", "alpha": 0.5, "min": 0, "max": 2000}, "2": {"name": "Blue Overlay", "alpha": 0.5, "min": 0, "max": 2000}}}]}} \ No newline at end of file diff --git a/imports/docker/settings/meteor.conf b/imports/docker/settings/meteor.conf new file mode 100644 index 0000000..6aa8373 --- /dev/null +++ b/imports/docker/settings/meteor.conf @@ -0,0 +1,23 @@ +error_log /var/log/nginx/nginx_error.log info; + +server { + listen 3003 default_server; + root /mc_startup_data; + location / { + autoindex on; + } + + } + +server { + listen 3000 default_server; + auth_basic "Restricted"; + auth_basic_user_file auth.htpasswd; + + location / { + proxy_pass http://localhost:2998/; + } + location /files/ { + alias /mc_data/; + } + } \ No newline at end of file diff --git a/imports/docker/settings/nginx.conf b/imports/docker/settings/nginx.conf new file mode 100755 index 0000000..f2845ee --- /dev/null +++ b/imports/docker/settings/nginx.conf @@ -0,0 +1,31 @@ + +worker_processes 1; +pid /var/cache/nginx/nginx.pid; +error_log /var/log/nginx/error.log warn; + + +events { + worker_connections 1024; +} + + +http { + disable_symlinks off; + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/meteor.conf; +} From 2711bfcfaef490c4fee76daeeb2832e2af6475f6 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 11 Jul 2018 17:16:40 -0400 Subject: [PATCH 02/60] Make work on HPC --- ...indcontrol.py => auto_local_mindcontrol.py | 0 auto_singularity_mindcontrol.py | 513 ++++++++++++++++++ imports/docker/Dockerfile_nginx | 24 +- imports/docker/entrypoint_nginx.sh | 12 +- .../docker/settings/mc_nginx_settings.json | 1 - start_singularity_mindcontrol.py | 270 +++++++++ 6 files changed, 807 insertions(+), 13 deletions(-) rename auto_mindcontrol.py => auto_local_mindcontrol.py (100%) create mode 100644 auto_singularity_mindcontrol.py delete mode 100644 imports/docker/settings/mc_nginx_settings.json create mode 100644 start_singularity_mindcontrol.py diff --git a/auto_mindcontrol.py b/auto_local_mindcontrol.py similarity index 100% rename from auto_mindcontrol.py rename to auto_local_mindcontrol.py diff --git a/auto_singularity_mindcontrol.py b/auto_singularity_mindcontrol.py new file mode 100644 index 0000000..40f2489 --- /dev/null +++ b/auto_singularity_mindcontrol.py @@ -0,0 +1,513 @@ +#! python +import argparse +from pathlib import Path +import json +from shutil import copyfile +import os +import getpass +from passlib.hash import bcrypt +import subprocess +import random +import sys + + +# HT password code from https://gist.github.com/eculver/1420227 + +# We need a crypt module, but Windows doesn't have one by default. Try to find +# one, and tell the user if we can't. +try: + import crypt +except ImportError: + try: + import fcrypt as crypt + except ImportError: + sys.stderr.write("Cannot find a crypt module. " + "Possibly http://carey.geek.nz/code/python-fcrypt/\n") + sys.exit(1) + + +def salt(): + """Returns a string of 2 randome letters""" + letters = 'abcdefghijklmnopqrstuvwxyz' \ + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' \ + '0123456789/.' + return random.choice(letters) + random.choice(letters) + + +class HtpasswdFile: + """A class for manipulating htpasswd files.""" + + def __init__(self, filename, create=False): + self.entries = [] + self.filename = filename + if not create: + if os.path.exists(self.filename): + self.load() + else: + raise Exception("%s does not exist" % self.filename) + + def load(self): + """Read the htpasswd file into memory.""" + lines = open(self.filename, 'r').readlines() + self.entries = [] + for line in lines: + username, pwhash = line.split(':') + entry = [username, pwhash.rstrip()] + self.entries.append(entry) + + def save(self): + """Write the htpasswd file to disk""" + open(self.filename, 'w').writelines(["%s:%s\n" % (entry[0], entry[1]) + for entry in self.entries]) + + def update(self, username, password): + """Replace the entry for the given user, or add it if new.""" + pwhash = crypt.crypt(password, salt()) + matching_entries = [entry for entry in self.entries + if entry[0] == username] + if matching_entries: + matching_entries[0][1] = pwhash + else: + self.entries.append([username, pwhash]) + + def delete(self, username): + """Remove the entry for the given user.""" + self.entries = [entry for entry in self.entries + if entry[0] != username] + + + +def write_passfile(passfile_path): + """Collects usernames and passwords and writes them to + the provided path with bcrypt encryption. + Parameters + ---------- + passfile: pathlib.Path object + The path to which to write the usernames and hashed passwords. + """ + users = set() + done = False + passfile = HtpasswdFile(passfile_path.as_posix(), create=True) + while not done: + user = "" + print("Please enter usernames and passwords for all the users you'd like to create.", flush=True) + user = input("Input user, leave blank if you are finished entering users:") + if len(users) > 0 and user == "": + print("All users entered, generating auth.htpass file", flush=True) + done= True + passfile.save() + elif len(users) == 0 and user == "": + print("Please enter at least one user", flush=True) + else: + if user in users: + print("Duplicate user, overwriting previously entered password for %s."%user, flush=True) + hs = None + while hs is None: + a = getpass.getpass(prompt="Enter Password for user: %s\n"%user) + b = getpass.getpass(prompt="Re-enter Password for user: %s\n"%user) + if a == b: + hs = "valid_pass" + passfile.update(user, a) + else: + print("Entered passwords don't match, please try again.", flush=True) + users.add(user) + + +def write_meteorconf(mcfile, startup_port=3003, nginx_port=3000, meteor_port=2998): + """Write nginx configuration file for meteor given user specified ports. + Parameters + ---------- + mcfile: pathlib.Path object + The path to which to write the config file. + startup_port: int, default is 3003 + Port number at which mindcontrol will look for startup manifest. + nginx_port: int, default is 3000 + Port number at nginx will run. This is the port you connect to reach mindcontrol. + meteor_port: int, default is 2998 + Port number at meteor will run. This is mostly under the hood, but you might + need to change it if there is a port conflict. Mongo will run on the port + one above the meteor_port. + """ + mc_string=f"""error_log /var/log/nginx/nginx_error.log info; + +server {{ + listen {startup_port} default_server; + root /mc_startup_data; + location / {{ + autoindex on; + }} + + }} + +server {{ + listen {nginx_port} default_server; + auth_basic "Restricted"; + auth_basic_user_file auth.htpasswd; + + location / {{ + proxy_pass http://localhost:{meteor_port}/; + }} + location /files/ {{ + alias /mc_data/; + }} + }}""" + mcfile.write_text(mc_string) + + +def write_nginxconf(ncfile): + """Write top level nginx configuration file. + Parameters + ---------- + ncfile: pathlib.Path object + The path to which to write the config file. + """ + nc_string=f"""worker_processes 1; +pid /var/cache/nginx/nginx.pid; +error_log /var/log/nginx/error.log warn; + + +events {{ + worker_connections 1024; +}} + + +http {{ + disable_symlinks off; + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/meteor.conf; +}} +""" + ncfile.write_text(nc_string) + + +def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port=3003, nginx_port=3000): + """ Write the mindcontrol settings json. This determines which panels mindcontrol + displays and which that information comes from. + Parameters + ---------- + mcfile: pathlib.Path object + The path to which to write the json file. + entry_types: optional, list of strings + List of names of modules you would like mindcontrol to display + freesurfer: optional, bool + True if you would like the settings generated modules for qcing aparc-aseg, wm, and ribbon + startup_port: int, default is 3003 + Port number at which mindcontrol will look for startup manifest. + nginx_port: int, default is 3000 + Port number at nginx will run. This is the port you connect to reach mindcontrol. + """ + file_server = f"http://localhost:{nginx_port}/files/" + startup_file_server = f'http://localhost:{startup_port}/' + + default_module = { + "fields": [ + { + "function_name": "get_qc_viewer", + "id": "name", + "name": "Image File" + }, + { + "function_name": "get_qc_ave_field", + "id": "average_vote", + "name": "QC vote" + }, + { + "function_name": None, + "id": "num_votes", + "name": "# votes" + }, + { + "function_name": None, + "id": "quality_check.notes_QC", + "name": "Notes" + } + ], + "metric_names": None, + "graph_type": None, + "staticURL": file_server, + "usePeerJS": False, + "logPainter": False, + "logContours": False, + "logPoints": True, + "qc_options": {"pass": 1, "fail": 1, "needs_edits": 0, "edited": 0, "assignTo": 0, "notes": 1, "confidence": 1}} + + fs_module = { + "fields": [ + { + "function_name": "get_filter_field", + "id": "subject", + "name": "Exam ID" + }, + { + "function_name": "get_qc_viewer", + "id": "name", + "name": "Freesurfer ID" + }, + { + "function_name": "get_qc_filter_field", + "id": "quality_check.QC", + "name": "QC" + }, + { + "function_name": "get_filter_field", + "id": "checkedBy", + "name": "checked by" + }, + { + "function_name": "get_filter_field", + "id": "quality_check.user_assign", + "name": "Assigned To" + }, + { + "function_name": None, + "id": "quality_check.notes_QC", + "name": "Notes" + } + ], + "metric_names": None, + "graph_type": None, + "staticURL": file_server, + "usePeerJS": False, + "logPainter": False, + "logContours": False, + "logPoints": True + } + + fs_cm_dict = {'aparcaseg': + { + "0":{"name": "Grayscale", + "alpha": 1, + "min": 0, + "max": 255 + }, + "1": { + "name": "custom.Freesurfer", + "alpha": 0.5 + } + }, + 'brainmask': + { + "0":{"name": "Grayscale", + "alpha": 1, + "min": 0, + "max": 255 + }, + "1": { + "name": "Red Overlay", + "alpha": 0.2, + "min": 0, + "max": 2000 + } + }, + 'ribbon': + { + "0":{"name": "Grayscale", + "alpha": 1, + "min": 0, + "max": 255 + }, + "1": { + "name": "Green Overlay", + "alpha": 0.5, + "min": 0, + "max": 2000 + }, + "2": { + "name": "Blue Overlay", + "alpha": 0.5, + "min":0, + "max": 2000 + } + } + } + fs_name_dict = {'brainmask': 'Brain Mask', + 'aparcaseg': 'Segmentation', + 'ribbon': 'White Matter', + } + if entry_types is None and not freesurfer: + raise Exception("You must either define entry types or have freesurfer == True") + + modules = [] + if entry_types is not None: + for et in entry_types: + et_module = default_module.copy() + et_module["name"] = et + et_module["entry_type"] = et + modules.append(et_module) + + if freesurfer: + for et,cm in fs_cm_dict.items(): + et_module = fs_module.copy() + et_module["name"] = fs_name_dict[et] + et_module["entry_type"] = et + et_module["num_overlays"] = len(cm) + et_module['colormaps'] = cm + modules.append(et_module) + + + # autogenerated settings files + pub_set = {"startup_json": startup_file_server+"startup.json", + "load_if_empty": True, + "use_custom": True, + "needs_consent": False, + "modules": modules} + settings = {"public": pub_set} + with mcsetfile.open("w") as h: + json.dump(settings,h) + +if __name__ == "__main__": + docker_build_path = Path(__file__).resolve().parent / 'imports/docker' + parser = argparse.ArgumentParser(description='Autogenerate singularity image and settings files, ' + 'for running mindcontrol in a singularity image on ' + 'another system that is hosting the data.') + parser.add_argument('--sing_out_dir', + default='.', + help='Directory to bulid singularirty image and files in.') + parser.add_argument('--custom_settings', + help='Path to custom settings json') + parser.add_argument('--freesurfer', action='store_true', + help='Generate settings for freesurfer QC in mindcontrol.') + parser.add_argument('--entry_type', action='append', + help='Name of mindcontrol module you would like to have autogenerated.' + ' This should correspond to the bids image type ' + '(specified after the final _ of the image name). ' + ' Pass this argument multiple times to add additional modules.') + parser.add_argument('--dockerfile', default=docker_build_path, + help='Path to the mindcontrol nginx dockerfile, defaults to %s'%docker_build_path) + parser.add_argument('--startup_port', + default=3003, + help='Port number at which mindcontrol will look for startup manifest.') + parser.add_argument('--nginx_port', + default=3000, + help='Port number at nginx will run. This is the port you connect to reach mindcontrol.') + parser.add_argument('--meteor_port', + default=2998, + help='Port number at meteor will run. ' + 'This is mostly under the hood, ' + 'but you might need to change it if there is a port conflict.' + 'Mongo will run on the port one above this one.') + + args = parser.parse_args() + sing_out_dir = args.sing_out_dir + if args.custom_settings is not None: + custom_settings = Path(args.custom_settings) + else: + custom_settings = None + freesurfer = args.freesurfer + entry_types = args.entry_type + startup_port = args.startup_port + nginx_port = args.nginx_port + meteor_port = args.meteor_port + + # Set up directory to be copied + basedir = Path(sing_out_dir) + setdir = basedir/"settings" + mcsetdir = setdir/"mc_settings" + manifest_dir = setdir/"mc_manifest_init" + meteor_ldir = basedir/".meteor" + + logdir = basedir/"log" + scratch_dir = logdir/"scratch" + nginx_scratch = scratch_dir/"nginx" + mc_hdir = scratch_dir/"singularity_home" + + if not basedir.exists(): + basedir.mkdir() + if not setdir.exists(): + setdir.mkdir() + if not logdir.exists(): + logdir.mkdir() + if not scratch_dir.exists(): + scratch_dir.mkdir() + if not nginx_scratch.exists(): + nginx_scratch.mkdir() + if not mcsetdir.exists(): + mcsetdir.mkdir() + if not manifest_dir.exists(): + manifest_dir.mkdir() + if not mc_hdir.exists(): + mc_hdir.mkdir() + if not meteor_ldir.exists(): + meteor_ldir.mkdir() + dockerfile = basedir/"Dockerfile_nginx" + entrypoint = basedir/"entrypoint_nginx.sh" + passfile = setdir/"auth.htpasswd" + mcfile = setdir/"meteor.conf" + ncfile = setdir/"nginx.conf" + mcsetfile = mcsetdir/"mc_nginx_settings.json" + infofile = setdir/"mc_info.json" + + # Write settings files + write_passfile(passfile) + write_nginxconf(ncfile) + write_meteorconf(mcfile, startup_port=startup_port, + nginx_port=nginx_port, meteor_port=meteor_port) + if custom_settings is not None: + copyfile(custom_settings, mcsetfile.as_posix()) + else: + write_mcsettings(mcsetfile, entry_types=entry_types, freesurfer=freesurfer, + startup_port=startup_port, nginx_port=nginx_port) + # Copy singularity run script to directory + srun_source = Path(__file__).resolve().parent / 'start_singularity_mindcontrol.py' + srun_dest = basedir / 'start_singularity_mindcontrol.py' + copyfile(srun_source.as_posix(), srun_dest.as_posix()) + + # Copy entrypoint script to directory + copyfile((docker_build_path / 'entrypoint_nginx.sh').as_posix(), entrypoint.as_posix()) + + # Copy dockerfile to base directory + copyfile((docker_build_path / 'Dockerfile_nginx').as_posix(), dockerfile.as_posix()) + + # Run docker build + subprocess.run("docker build -f Dockerfile_nginx -t auto_mc_nginx .", + cwd=basedir.as_posix(), shell=True, check=True) + + # Run docker2singularity + subprocess.run("docker run -v /var/run/docker.sock:/var/run/docker.sock " + "-v ${PWD}:/output --privileged -t --rm " + "singularityware/docker2singularity auto_mc_nginx", + cwd=basedir.as_posix(), shell=True, check=True) + + # Get name of singularity image + simg = [si for si in basedir.glob("auto_mc_nginx*.img")][0] + + info = dict(entry_types=entry_types, + freesurfer=freesurfer, + startup_port=startup_port, + nginx_port=nginx_port, + meteor_port=meteor_port, + simg=simg.parts[-1]) + + with infofile.open("w") as h: + json.dump(info, h) + + # Print next steps + print("Finished building singuliarity image and settings files.") + print(f"Copy {basedir} to machine that will be hosting the mindcontrol instance.") + print("Consider the following command: ") + print(f"rsync -avch {basedir} [host machine]:[destination path]") + print("Then, on the machine hosting the mindcontrol instance") + print(f"run the start_singularity_mindcontrol.py script included in {basedir}.") + print("Consider the following commands: ") + print(f"cd [destination path]/{basedir.parts[-1]}") + if freesurfer: + print(f"python start_singularity_mindcontrol.py --bids_dir [path to bids dir] --freesurfer_dir [path to freesurfer outputs]") + else: + print(f"python start_singularity_mindcontrol.py --bids_dir [path to bids dir]") + + diff --git a/imports/docker/Dockerfile_nginx b/imports/docker/Dockerfile_nginx index 39ea4a3..5311dbf 100644 --- a/imports/docker/Dockerfile_nginx +++ b/imports/docker/Dockerfile_nginx @@ -9,6 +9,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ vim \ supervisor \ + rsync \ + procps \ && rm -rf /var/lib/apt/lists/* ENV METEOR_RELEASE 1.7.0.3 @@ -53,20 +55,22 @@ ENV LC_ALL C COPY entrypoint_nginx.sh /home/entrypoint.sh -COPY ndmg_launch.sh /home/ndmg_launch.sh +#COPY ndmg_launch.sh /home/ndmg_launch.sh RUN useradd --create-home --home-dir ${MC_DIR} mindcontrol -RUN chown mindcontrol:mindcontrol /home/entrypoint.sh -RUN chmod +x /home/entrypoint.sh - -RUN mkdir -p ${MC_DIR}/mindcontrol &&\ +RUN chown mindcontrol:mindcontrol /home/entrypoint.sh &&\ + chmod +x /home/entrypoint.sh &&\ + mkdir -p ${MC_DIR}/mindcontrol &&\ chown -R mindcontrol /home/mindcontrol &&\ chmod -R a+rx /home/mindcontrol USER mindcontrol RUN cd ${MC_DIR}/mindcontrol &&\ - git clone https://github.com/akeshavan/mindcontrol.git ${MC_DIR}/mindcontrol + git clone https://github.com/akeshavan/mindcontrol.git ${MC_DIR}/mindcontrol &&\ + meteor update &&\ + meteor npm install --save @babel/runtime &&\ + meteor npm install --save bcrypt #git clone https://github.com/clowdcontrol/mindcontrol.git ${MC_DIR}/mindcontrol @@ -77,14 +81,14 @@ COPY ./settings/meteor.conf /etc/nginx/conf.d/meteor.conf COPY ./settings/auth.htpasswd /etc/nginx COPY ./settings/nginx.conf /etc/nginx/nginx.conf USER root -RUN chmod 777 /var/log/nginx /var/cache/nginx /opt /etc -RUN mkdir -p /mc_data /mc_startup_data -RUN chown mindcontrol:mindcontrol /etc/nginx/auth.htpasswd /mc_data /mc_startup_data +RUN chmod 777 /var/log/nginx /var/cache/nginx /opt /etc &&\ + mkdir -p /mc_data /mc_startup_data /mc_settings &&\ + chown mindcontrol:mindcontrol /etc/nginx/auth.htpasswd /mc_data /mc_startup_data /mc_settings USER mindcontrol ###### ###### Load in local settings -COPY ./settings/mc_nginx_settings.json ${MC_DIR}/mindcontrol/mc_nginx_settings.json +COPY ./settings/mc_settings/mc_nginx_settings.json /mc_settings/mc_nginx_settings.json # Make a spot that we can put our data RUN mkdir -p /mc_data /mc_startup_data diff --git a/imports/docker/entrypoint_nginx.sh b/imports/docker/entrypoint_nginx.sh index 68a0a04..4731806 100644 --- a/imports/docker/entrypoint_nginx.sh +++ b/imports/docker/entrypoint_nginx.sh @@ -1,5 +1,13 @@ #!/bin/bash -cd /home/mindcontrol/mindcontrol -nohup meteor --settings mc_nginx_settings.json --port 2998 & +cd ~ +if [ ! -x .meteor ]; then + echo "Copying meteor files into singularity_home" + rsync -ach /home/mindcontrol/mindcontrol . + rsync -ach /home/mindcontrol/.meteor . + rsync -ach /home/mindcontrol/.cordova . + ln -s /home/mindcontrol/mindcontrol/.meteor/local ~/mindcontrol/.meteor/local/ +fi +cd ~/mindcontrol +nohup meteor --settings /mc_settings/mc_nginx_settings.json --port 2998 & nginx -g "daemon off;" diff --git a/imports/docker/settings/mc_nginx_settings.json b/imports/docker/settings/mc_nginx_settings.json deleted file mode 100644 index e2a5a7d..0000000 --- a/imports/docker/settings/mc_nginx_settings.json +++ /dev/null @@ -1 +0,0 @@ -{"public": {"startup_json": "http://localhost:3003/startup.freesurfer.json", "load_if_empty": true, "use_custom": true, "needs_consent": false, "modules": [{"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "aparcaseg", "entry_type": "aparcaseg", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "custom.Freesurfer", "alpha": 0.5}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "wm", "entry_type": "wm", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Spectrum", "alpha": 0.5}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "brainmask", "entry_type": "brainmask", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Red Overlay", "alpha": 0.2, "min": 0, "max": 2000}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "ribbon", "entry_type": "ribbon", "num_overlays": 3, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Green Overlay", "alpha": 0.5, "min": 0, "max": 2000}, "2": {"name": "Blue Overlay", "alpha": 0.5, "min": 0, "max": 2000}}}]}} \ No newline at end of file diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py new file mode 100644 index 0000000..1c2efd0 --- /dev/null +++ b/start_singularity_mindcontrol.py @@ -0,0 +1,270 @@ +#! python +import argparse +from pathlib import Path +import json +from bids.grabbids import BIDSLayout +import subprocess +import os +import shutil +from nipype import MapNode, Workflow, Node +from nipype.interfaces.freesurfer import MRIConvert +from nipype.interfaces.io import DataSink +from nipype.interfaces.utility import IdentityInterface, Function +#this function finds data in the subjects_dir +def data_grabber(subjects_dir, subject, volumes): + import os + volumes_list = [os.path.join(subjects_dir, subject, 'mri', volume) for volume in volumes] + return volumes_list + +#this function parses the aseg.stats, lh.aparc.stats and rh.aparc.stats and returns a dictionary +def parse_stats(subjects_dir, subject): + from os.path import join, exists + + aseg_file = join(subjects_dir, subject, "stats", "aseg.stats") + lh_aparc = join(subjects_dir, subject, "stats", "lh.aparc.stats") + rh_aparc = join(subjects_dir, subject, "stats", "rh.aparc.stats") + + assert exists(aseg_file), "aseg file does not exists for %s" %subject + assert exists(lh_aparc), "lh aparc file does not exists for %s" %subject + assert exists(rh_aparc), "rh aparc file does not exists for %s" %subject + + def convert_stats_to_json(aseg_file, lh_aparc, rh_aparc): + import pandas as pd + import numpy as np + + def extract_other_vals_from_aseg(f): + value_labels = ["EstimatedTotalIntraCranialVol", + "Mask", + "TotalGray", + "SubCortGray", + "Cortex", + "CerebralWhiteMatter", + "CorticalWhiteMatterVol"] + value_labels = list(map(lambda x: 'Measure ' + x + ',', value_labels)) + output = pd.DataFrame() + with open(f,"r") as q: + out = q.readlines() + relevant_entries = [x for x in out if any(v in x for v in value_labels)] + for val in relevant_entries: + sname= val.split(",")[1][1:] + vol = val.split(",")[-2] + output = output.append(pd.Series({"StructName":sname,"Volume_mm3":vol}),ignore_index=True) + return output + + df = pd.DataFrame(np.genfromtxt(aseg_file,dtype=str),columns=["Index", + "SegId", + "NVoxels", + "Volume_mm3", + "StructName", + "normMean", + "normStdDev", + "normMin", + "normMax", + "normRange"]) + + df = df.append(extract_other_vals_from_aseg(aseg_file), ignore_index=True) + + aparc_columns = ["StructName", "NumVert", "SurfArea", "GrayVol", + "ThickAvg", "ThickStd", "MeanCurv", "GausCurv", + "FoldInd", "CurvInd"] + tmp_lh = pd.DataFrame(np.genfromtxt(lh_aparc,dtype=str),columns=aparc_columns) + tmp_lh["StructName"] = "lh_"+tmp_lh["StructName"] + tmp_rh = pd.DataFrame(np.genfromtxt(rh_aparc,dtype=str),columns=aparc_columns) + tmp_rh["StructName"] = "rh_"+tmp_rh["StructName"] + + aseg_melt = pd.melt(df[["StructName","Volume_mm3"]], id_vars=["StructName"]) + aseg_melt.rename(columns={"StructName": "name"},inplace=True) + aseg_melt["value"] = aseg_melt["value"].astype(float) + + lh_aparc_melt = pd.melt(tmp_lh,id_vars=["StructName"]) + lh_aparc_melt["value"] = lh_aparc_melt["value"].astype(float) + lh_aparc_melt["name"] = lh_aparc_melt["StructName"]+ "_"+lh_aparc_melt["variable"] + + rh_aparc_melt = pd.melt(tmp_rh, id_vars=["StructName"]) + rh_aparc_melt["value"] = rh_aparc_melt["value"].astype(float) + rh_aparc_melt["name"] = rh_aparc_melt["StructName"]+ "_"+rh_aparc_melt["variable"] + + output = aseg_melt[["name", + "value"]].append(lh_aparc_melt[["name", + "value"]], + ignore_index=True).append(rh_aparc_melt[["name", + "value"]], + ignore_index=True) + outdict = output.to_dict(orient="records") + final_dict = {} + for pair in outdict: + final_dict[pair["name"]] = pair["value"] + return final_dict + + output_dict = convert_stats_to_json(aseg_file, lh_aparc, rh_aparc) + return output_dict + +# This function creates valid Mindcontrol entries that are saved as .json files. # This f +# They can be loaded into the Mindcontrol database later + +def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, startup_json_path): + import os + from nipype.utils.filemanip import save_json + + cortical_wm = "CerebralWhiteMatterVol" # for later FS version + if not stats.get(cortical_wm): + cortical_wm = "CorticalWhiteMatterVol" + + metric_split = {"brainmask": ["eTIV", "CortexVol", "TotalGrayVol"], + "wm": [cortical_wm,"WM-hypointensities", + "Right-WM-hypointensities","Left-WM-hypointensities"], + "aparcaseg":[]} + + volumes = {'aparcaseg': ['T1.nii.gz', 'aparc+aseg.nii.gz'], + 'brainmask': ['T1.nii.gz', 'brainmask.nii.gz'], + 'wm': ['T1.nii.gz', 'ribbon.nii.gz', 'wm.nii.gz']} + + all_entries = [] + + for idx, entry_type in enumerate(["brainmask", "wm", "aparcaseg"]): + entry = {"entry_type":entry_type, + "subject_id": subject, + "name": subject} + volumes_list = [os.path.relpath(os.path.join(output_dir, subject, volume), mindcontrol_base_dir) + for volume in volumes[entry_type]] + entry["check_masks"] = volumes_list + entry["metrics"] = {} + for metric_name in metric_split[entry_type]: + entry["metrics"][metric_name] = stats.pop(metric_name) + if not len(metric_split[entry_type]): + entry["metrics"] = stats + all_entries.append(entry) + + output_json = os.path.abspath(os.path.join(startup_json_path)) + save_json(output_json, all_entries) + return output_json + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Start mindcontrol in a previously built' + ' singularity container and create initial manifest' + ' if needed.') + parser.add_argument('--bids_dir', help='The directory with the input dataset ' + 'formatted according to the BIDS standard.') + parser.add_argument('--freesurfer_dir', help='The directory with the freesurfer dirivatives,' + ' should be inside the bids directory') + parser.add_argument('--mc_singularity_path', default = '.', + help="Path to the directory containing the mindcontrol" + "singularity image and settings files.") + parser.add_argument('--no_server', action = 'store_true', + help="Don't start the mindcontrol server, just generate the manifest") + + args = parser.parse_args() + if args.bids_dir is not None: + bids_dir = Path(args.bids_dir) + else: + bids_dir = None + + if args.freesurfer_dir is not None: + freesurfer_dir = Path(args.freesurfer_dir) + else: + freesurfer_dir = None + + mc_singularity_path = Path(args.mc_singularity_path) + + no_server = args.no_server + + infofile = mc_singularity_path/'settings/mc_info.json' + with infofile.open('r') as h: + info = json.load(h) + manifest_dir = mc_singularity_path/'settings/mc_manifest_init/' + manifest_json = (manifest_dir/'startup.json').resolve() + + # First create the initial manifest + manifest = [] + if info['entry_types'] is not None: + entry_types = set(info['entry_types']) + unused_types = set() + for img in layout.get(extensions = ".nii.gz"): + if img.type in entry_types: + img_dict = {} + img_dict["check_masks"] = [img.filename.replace(bids_dir.as_posix(),"")] + img_dict["entry_type"] = img.type + img_dict["metrics"] = {} + img_dict["name"] = os.path.split(img.filename)[1].split('.')[0] + img_dict["subject"] = 'sub-' + img.subject + img_dict["session"] = 'ses-' + img.session + manifest.append(img_dict) + else: + unused_types.add(img.type) + if info['freesurfer']: + if freesurfer_dir is None: + #TODO: look in default location for freesurfer directory + raise Exception("Must specify the path to freesurfer files.") + + subjects = [] + for path in freesurfer_dir.glob('*'): + subject = path.parts[-1] + # check if mri dir exists, and don't add fsaverage + if os.path.exists(os.path.join(path, 'mri')) and subject != 'fsaverage': + subjects.append(subject) + + volumes = ["brainmask.mgz", "wm.mgz", "aparc+aseg.mgz", "T1.mgz", "ribbon.mgz"] + input_node = Node(IdentityInterface(fields=['subject_id',"subjects_dir", + "mindcontrol_base_dir", "output_dir", "startup_json_path"]), name='inputnode') + + input_node.iterables=("subject_id", subjects) + input_node.inputs.subjects_dir = freesurfer_dir + input_node.inputs.mindcontrol_base_dir = bids_dir.as_posix() #this is where start_static_server is running + input_node.inputs.output_dir = freesurfer_dir.as_posix() #this is in the freesurfer/ directory under the base_dir + input_node.inputs.startup_json_path = manifest_json.as_posix() + + dg_node=Node(Function(input_names=["subjects_dir", "subject", "volumes"], + output_names=["volume_paths"], + function=data_grabber), + name="datagrab") + #dg_node.inputs.subjects_dir = subjects_dir + dg_node.inputs.volumes = volumes + + mriconvert_node = MapNode(MRIConvert(out_type="niigz"), + iterfield=["in_file"], + name='convert') + + get_stats_node = Node(Function(input_names=["subjects_dir", "subject"], + output_names = ["output_dict"], + function=parse_stats), name="get_freesurfer_stats") + + write_mindcontrol_entries = Node(Function(input_names = ["mindcontrol_base_dir", + "output_dir", + "subject", + "stats", + "startup_json_path"], + output_names=["output_json"], + function=create_mindcontrol_entries), + name="get_mindcontrol_entries") + + datasink_node = Node(DataSink(), + name='datasink') + subst = [('out_file',''),('_subject_id_',''),('_out','')] + [("_convert%d" % index, "") for index in range(len(volumes))] + datasink_node.inputs.substitutions = subst + workflow_working_dir = os.path.abspath("./log/scratch") + + wf = Workflow(name="MindPrepFS") + wf.base_dir = workflow_working_dir + wf.connect(input_node,"subject_id", dg_node,"subject") + wf.connect(input_node,"subjects_dir", dg_node, "subjects_dir") + wf.connect(input_node, "subject_id", get_stats_node, "subject") + wf.connect(input_node, "subjects_dir", get_stats_node, "subjects_dir") + wf.connect(input_node, "subject_id", write_mindcontrol_entries, "subject") + wf.connect(input_node, "mindcontrol_base_dir", write_mindcontrol_entries, "mindcontrol_base_dir") + wf.connect(input_node, "output_dir", write_mindcontrol_entries, "output_dir") + wf.connect(input_node, "startup_json_path", write_mindcontrol_entries, "startup_json_path") + wf.connect(get_stats_node, "output_dict", write_mindcontrol_entries, "stats") + wf.connect(input_node, "output_dir", datasink_node, "base_directory") + wf.connect(dg_node,"volume_paths", mriconvert_node, "in_file") + wf.connect(mriconvert_node,'out_file',datasink_node,'out_file') + wf.connect(write_mindcontrol_entries, "output_json", datasink_node, "out_file.@json") + #wf.write_graph(graph2use='exec') + wf.run() + + if not args.no_server: + cmd = f"singularity run -B ${{PWD}}/log:/var/log/nginx -B {bids_dir}:/mc_data" \ + + f" -B settings:/opt/settings -B {manifest_dir}:/mc_startup_data" \ + + f" -B ${{PWD}}/log/scratch/nginx/:/var/cache/nginx -B ${{PWD}}/.meteor/:/home/mindcontrol/mindcontrol/.meteor/local" \ + + f" -B ${{PWD}}/settings/mc_settings:/mc_settings -H ${{PWD}}/log/scratch/singularity_home ${{PWD}}/{info['simg']}" + print(cmd) + subprocess.run(cmd, cwd = mc_singularity_path, shell=True, check=True) From a7c82252ffe273ec59377005bdd93d81c89e9bb1 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 12 Jul 2018 11:31:49 -0400 Subject: [PATCH 03/60] Fix start_singularity_mindcontrol.py --- start_singularity_mindcontrol.py | 166 +++++++++++++++++-------------- 1 file changed, 93 insertions(+), 73 deletions(-) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 1c2efd0..958f66f 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -10,12 +10,15 @@ from nipype.interfaces.freesurfer import MRIConvert from nipype.interfaces.io import DataSink from nipype.interfaces.utility import IdentityInterface, Function + + #this function finds data in the subjects_dir def data_grabber(subjects_dir, subject, volumes): import os volumes_list = [os.path.join(subjects_dir, subject, 'mri', volume) for volume in volumes] return volumes_list + #this function parses the aseg.stats, lh.aparc.stats and rh.aparc.stats and returns a dictionary def parse_stats(subjects_dir, subject): from os.path import join, exists @@ -24,9 +27,9 @@ def parse_stats(subjects_dir, subject): lh_aparc = join(subjects_dir, subject, "stats", "lh.aparc.stats") rh_aparc = join(subjects_dir, subject, "stats", "rh.aparc.stats") - assert exists(aseg_file), "aseg file does not exists for %s" %subject - assert exists(lh_aparc), "lh aparc file does not exists for %s" %subject - assert exists(rh_aparc), "rh aparc file does not exists for %s" %subject + assert exists(aseg_file), "aseg file does not exists for %s" % subject + assert exists(lh_aparc), "lh aparc file does not exists for %s" % subject + assert exists(rh_aparc), "rh aparc file does not exists for %s" % subject def convert_stats_to_json(aseg_file, lh_aparc, rh_aparc): import pandas as pd @@ -34,56 +37,63 @@ def convert_stats_to_json(aseg_file, lh_aparc, rh_aparc): def extract_other_vals_from_aseg(f): value_labels = ["EstimatedTotalIntraCranialVol", - "Mask", - "TotalGray", - "SubCortGray", - "Cortex", - "CerebralWhiteMatter", - "CorticalWhiteMatterVol"] + "Mask", + "TotalGray", + "SubCortGray", + "Cortex", + "CerebralWhiteMatter", + "CorticalWhiteMatterVol"] value_labels = list(map(lambda x: 'Measure ' + x + ',', value_labels)) output = pd.DataFrame() - with open(f,"r") as q: + with open(f, "r") as q: out = q.readlines() relevant_entries = [x for x in out if any(v in x for v in value_labels)] for val in relevant_entries: - sname= val.split(",")[1][1:] + sname = val.split(",")[1][1:] vol = val.split(",")[-2] - output = output.append(pd.Series({"StructName":sname,"Volume_mm3":vol}),ignore_index=True) + output = output.append(pd.Series({"StructName": sname, + "Volume_mm3": vol}), + ignore_index=True) return output - df = pd.DataFrame(np.genfromtxt(aseg_file,dtype=str),columns=["Index", - "SegId", - "NVoxels", - "Volume_mm3", - "StructName", - "normMean", - "normStdDev", - "normMin", - "normMax", - "normRange"]) + df = pd.DataFrame(np.genfromtxt(aseg_file, dtype=str), + columns=["Index", + "SegId", + "NVoxels", + "Volume_mm3", + "StructName", + "normMean", + "normStdDev", + "normMin", + "normMax", + "normRange"]) df = df.append(extract_other_vals_from_aseg(aseg_file), ignore_index=True) - + aparc_columns = ["StructName", "NumVert", "SurfArea", "GrayVol", "ThickAvg", "ThickStd", "MeanCurv", "GausCurv", "FoldInd", "CurvInd"] - tmp_lh = pd.DataFrame(np.genfromtxt(lh_aparc,dtype=str),columns=aparc_columns) + tmp_lh = pd.DataFrame(np.genfromtxt(lh_aparc, dtype=str), + columns=aparc_columns) tmp_lh["StructName"] = "lh_"+tmp_lh["StructName"] - tmp_rh = pd.DataFrame(np.genfromtxt(rh_aparc,dtype=str),columns=aparc_columns) + tmp_rh = pd.DataFrame(np.genfromtxt(rh_aparc, dtype=str), + columns=aparc_columns) tmp_rh["StructName"] = "rh_"+tmp_rh["StructName"] - aseg_melt = pd.melt(df[["StructName","Volume_mm3"]], id_vars=["StructName"]) - aseg_melt.rename(columns={"StructName": "name"},inplace=True) + aseg_melt = pd.melt(df[["StructName", "Volume_mm3"]], + id_vars=["StructName"]) + aseg_melt.rename(columns={"StructName": "name"}, + inplace=True) aseg_melt["value"] = aseg_melt["value"].astype(float) - + lh_aparc_melt = pd.melt(tmp_lh,id_vars=["StructName"]) lh_aparc_melt["value"] = lh_aparc_melt["value"].astype(float) - lh_aparc_melt["name"] = lh_aparc_melt["StructName"]+ "_"+lh_aparc_melt["variable"] - + lh_aparc_melt["name"] = lh_aparc_melt["StructName"] + "_" + lh_aparc_melt["variable"] + rh_aparc_melt = pd.melt(tmp_rh, id_vars=["StructName"]) rh_aparc_melt["value"] = rh_aparc_melt["value"].astype(float) - rh_aparc_melt["name"] = rh_aparc_melt["StructName"]+ "_"+rh_aparc_melt["variable"] - + rh_aparc_melt["name"] = rh_aparc_melt["StructName"] + "_" + rh_aparc_melt["variable"] + output = aseg_melt[["name", "value"]].append(lh_aparc_melt[["name", "value"]], @@ -95,35 +105,35 @@ def extract_other_vals_from_aseg(f): for pair in outdict: final_dict[pair["name"]] = pair["value"] return final_dict - + output_dict = convert_stats_to_json(aseg_file, lh_aparc, rh_aparc) return output_dict + # This function creates valid Mindcontrol entries that are saved as .json files. # This f # They can be loaded into the Mindcontrol database later - def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, startup_json_path): import os from nipype.utils.filemanip import save_json - + cortical_wm = "CerebralWhiteMatterVol" # for later FS version if not stats.get(cortical_wm): cortical_wm = "CorticalWhiteMatterVol" - + metric_split = {"brainmask": ["eTIV", "CortexVol", "TotalGrayVol"], - "wm": [cortical_wm,"WM-hypointensities", - "Right-WM-hypointensities","Left-WM-hypointensities"], - "aparcaseg":[]} - + "wm": [cortical_wm, "WM-hypointensities", + "Right-WM-hypointensities", "Left-WM-hypointensities"], + "aparcaseg": []} + volumes = {'aparcaseg': ['T1.nii.gz', 'aparc+aseg.nii.gz'], - 'brainmask': ['T1.nii.gz', 'brainmask.nii.gz'], - 'wm': ['T1.nii.gz', 'ribbon.nii.gz', 'wm.nii.gz']} + 'brainmask': ['T1.nii.gz', 'brainmask.nii.gz'], + 'ribbon': ['T1.nii.gz', 'ribbon.nii.gz', 'wm.nii.gz']} all_entries = [] - + for idx, entry_type in enumerate(["brainmask", "wm", "aparcaseg"]): - entry = {"entry_type":entry_type, - "subject_id": subject, + entry = {"entry_type": entry_type, + "subject_id": subject, "name": subject} volumes_list = [os.path.relpath(os.path.join(output_dir, subject, volume), mindcontrol_base_dir) for volume in volumes[entry_type]] @@ -134,7 +144,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, if not len(metric_split[entry_type]): entry["metrics"] = stats all_entries.append(entry) - + output_json = os.path.abspath(os.path.join(startup_json_path)) save_json(output_json, all_entries) return output_json @@ -147,15 +157,17 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, 'formatted according to the BIDS standard.') parser.add_argument('--freesurfer_dir', help='The directory with the freesurfer dirivatives,' ' should be inside the bids directory') - parser.add_argument('--mc_singularity_path', default = '.', - help="Path to the directory containing the mindcontrol" + parser.add_argument('--mc_singularity_path', default='.', + help="Path to the directory containing the mindcontrol" "singularity image and settings files.") - parser.add_argument('--no_server', action = 'store_true', + parser.add_argument('--no_server', action='store_true', help="Don't start the mindcontrol server, just generate the manifest") args = parser.parse_args() if args.bids_dir is not None: bids_dir = Path(args.bids_dir) + layout = BIDSLayout(bids_dir) + else: bids_dir = None @@ -174,12 +186,13 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, manifest_dir = mc_singularity_path/'settings/mc_manifest_init/' manifest_json = (manifest_dir/'startup.json').resolve() + # First create the initial manifest manifest = [] - if info['entry_types'] is not None: + if info['entry_types'] is not None and bids_dir is not None: entry_types = set(info['entry_types']) unused_types = set() - for img in layout.get(extensions = ".nii.gz"): + for img in layout.get(extensions=".nii.gz"): if img.type in entry_types: img_dict = {} img_dict["check_masks"] = [img.filename.replace(bids_dir.as_posix(),"")] @@ -204,49 +217,56 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, subjects.append(subject) volumes = ["brainmask.mgz", "wm.mgz", "aparc+aseg.mgz", "T1.mgz", "ribbon.mgz"] - input_node = Node(IdentityInterface(fields=['subject_id',"subjects_dir", - "mindcontrol_base_dir", "output_dir", "startup_json_path"]), name='inputnode') + input_node = Node(IdentityInterface(fields=['subject_id', + "subjects_dir", + "mindcontrol_base_dir", + "output_dir", + "startup_json_path"]), + name='inputnode') - input_node.iterables=("subject_id", subjects) + input_node.iterables = ("subject_id", subjects) input_node.inputs.subjects_dir = freesurfer_dir - input_node.inputs.mindcontrol_base_dir = bids_dir.as_posix() #this is where start_static_server is running - input_node.inputs.output_dir = freesurfer_dir.as_posix() #this is in the freesurfer/ directory under the base_dir - input_node.inputs.startup_json_path = manifest_json.as_posix() + input_node.inputs.mindcontrol_base_dir = bids_dir.as_posix() + input_node.inputs.output_dir = freesurfer_dir.as_posix() + input_node.inputs.startup_json_path = manifest_json.as_posix() - dg_node=Node(Function(input_names=["subjects_dir", "subject", "volumes"], - output_names=["volume_paths"], - function=data_grabber), + dg_node = Node(Function(input_names=["subjects_dir", "subject", "volumes"], + output_names=["volume_paths"], + function=data_grabber), name="datagrab") #dg_node.inputs.subjects_dir = subjects_dir dg_node.inputs.volumes = volumes - mriconvert_node = MapNode(MRIConvert(out_type="niigz"), - iterfield=["in_file"], + mriconvert_node = MapNode(MRIConvert(out_type="niigz"), + iterfield=["in_file"], name='convert') get_stats_node = Node(Function(input_names=["subjects_dir", "subject"], - output_names = ["output_dict"], + output_names=["output_dict"], function=parse_stats), name="get_freesurfer_stats") - write_mindcontrol_entries = Node(Function(input_names = ["mindcontrol_base_dir", - "output_dir", - "subject", - "stats", - "startup_json_path"], + write_mindcontrol_entries = Node(Function(input_names=["mindcontrol_base_dir", + "output_dir", + "subject", + "stats", + "startup_json_path"], output_names=["output_json"], function=create_mindcontrol_entries), name="get_mindcontrol_entries") datasink_node = Node(DataSink(), name='datasink') - subst = [('out_file',''),('_subject_id_',''),('_out','')] + [("_convert%d" % index, "") for index in range(len(volumes))] + subst = [('out_file', ''), + ('_subject_id_', ''), + ('_out', '')] + subst += [("_convert%d" % index, "") for index in range(len(volumes))] datasink_node.inputs.substitutions = subst workflow_working_dir = os.path.abspath("./log/scratch") wf = Workflow(name="MindPrepFS") wf.base_dir = workflow_working_dir - wf.connect(input_node,"subject_id", dg_node,"subject") - wf.connect(input_node,"subjects_dir", dg_node, "subjects_dir") + wf.connect(input_node, "subject_id", dg_node, "subject") + wf.connect(input_node, "subjects_dir", dg_node, "subjects_dir") wf.connect(input_node, "subject_id", get_stats_node, "subject") wf.connect(input_node, "subjects_dir", get_stats_node, "subjects_dir") wf.connect(input_node, "subject_id", write_mindcontrol_entries, "subject") @@ -256,7 +276,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, wf.connect(get_stats_node, "output_dict", write_mindcontrol_entries, "stats") wf.connect(input_node, "output_dir", datasink_node, "base_directory") wf.connect(dg_node,"volume_paths", mriconvert_node, "in_file") - wf.connect(mriconvert_node,'out_file',datasink_node,'out_file') + wf.connect(mriconvert_node,'out_file', datasink_node,'out_file') wf.connect(write_mindcontrol_entries, "output_json", datasink_node, "out_file.@json") #wf.write_graph(graph2use='exec') wf.run() @@ -267,4 +287,4 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, + f" -B ${{PWD}}/log/scratch/nginx/:/var/cache/nginx -B ${{PWD}}/.meteor/:/home/mindcontrol/mindcontrol/.meteor/local" \ + f" -B ${{PWD}}/settings/mc_settings:/mc_settings -H ${{PWD}}/log/scratch/singularity_home ${{PWD}}/{info['simg']}" print(cmd) - subprocess.run(cmd, cwd = mc_singularity_path, shell=True, check=True) + subprocess.run(cmd, cwd=mc_singularity_path, shell=True, check=True) From d2e35edd8dc3ba1c34d17e09a58dd65e2c43573e Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 12 Jul 2018 13:54:41 -0400 Subject: [PATCH 04/60] Fix ribbon display --- auto_singularity_mindcontrol.py | 2 +- imports/docker/entrypoint_nginx.sh | 2 +- start_singularity_mindcontrol.py | 18 ++++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/auto_singularity_mindcontrol.py b/auto_singularity_mindcontrol.py index 40f2489..c988afb 100644 --- a/auto_singularity_mindcontrol.py +++ b/auto_singularity_mindcontrol.py @@ -314,7 +314,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port "max": 2000 } }, - 'ribbon': + 'wm': { "0":{"name": "Grayscale", "alpha": 1, diff --git a/imports/docker/entrypoint_nginx.sh b/imports/docker/entrypoint_nginx.sh index 4731806..34585ff 100644 --- a/imports/docker/entrypoint_nginx.sh +++ b/imports/docker/entrypoint_nginx.sh @@ -9,5 +9,5 @@ if [ ! -x .meteor ]; then ln -s /home/mindcontrol/mindcontrol/.meteor/local ~/mindcontrol/.meteor/local/ fi cd ~/mindcontrol -nohup meteor --settings /mc_settings/mc_nginx_settings.json --port 2998 & +nohup meteor --settings /mc_settings/mc_nginx_settings.json --port 2998 > ~/mindcontrol.out 2>&1 & nginx -g "daemon off;" diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 958f66f..2dc3e7d 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -127,7 +127,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, volumes = {'aparcaseg': ['T1.nii.gz', 'aparc+aseg.nii.gz'], 'brainmask': ['T1.nii.gz', 'brainmask.nii.gz'], - 'ribbon': ['T1.nii.gz', 'ribbon.nii.gz', 'wm.nii.gz']} + 'wm': ['T1.nii.gz', 'ribbon.nii.gz', 'wm.nii.gz']} all_entries = [] @@ -166,7 +166,10 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, args = parser.parse_args() if args.bids_dir is not None: bids_dir = Path(args.bids_dir) - layout = BIDSLayout(bids_dir) + try: + layout = BIDSLayout(bids_dir) + except ValueError as e: + print("Invalid bids directory, skipping none freesurfer files. BIDS error:", e) else: bids_dir = None @@ -186,7 +189,6 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, manifest_dir = mc_singularity_path/'settings/mc_manifest_init/' manifest_json = (manifest_dir/'startup.json').resolve() - # First create the initial manifest manifest = [] if info['entry_types'] is not None and bids_dir is not None: @@ -195,7 +197,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, for img in layout.get(extensions=".nii.gz"): if img.type in entry_types: img_dict = {} - img_dict["check_masks"] = [img.filename.replace(bids_dir.as_posix(),"")] + img_dict["check_masks"] = [img.filename.replace(bids_dir.as_posix(), "")] img_dict["entry_type"] = img.type img_dict["metrics"] = {} img_dict["name"] = os.path.split(img.filename)[1].split('.')[0] @@ -227,13 +229,13 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, input_node.iterables = ("subject_id", subjects) input_node.inputs.subjects_dir = freesurfer_dir input_node.inputs.mindcontrol_base_dir = bids_dir.as_posix() - input_node.inputs.output_dir = freesurfer_dir.as_posix() + input_node.inputs.output_dir = freesurfer_dir.as_posix() input_node.inputs.startup_json_path = manifest_json.as_posix() dg_node = Node(Function(input_names=["subjects_dir", "subject", "volumes"], output_names=["volume_paths"], function=data_grabber), - name="datagrab") + name="datagrab") #dg_node.inputs.subjects_dir = subjects_dir dg_node.inputs.volumes = volumes @@ -251,7 +253,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, "stats", "startup_json_path"], output_names=["output_json"], - function=create_mindcontrol_entries), + function=create_mindcontrol_entries), name="get_mindcontrol_entries") datasink_node = Node(DataSink(), @@ -276,7 +278,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, wf.connect(get_stats_node, "output_dict", write_mindcontrol_entries, "stats") wf.connect(input_node, "output_dir", datasink_node, "base_directory") wf.connect(dg_node,"volume_paths", mriconvert_node, "in_file") - wf.connect(mriconvert_node,'out_file', datasink_node,'out_file') + wf.connect(mriconvert_node,'out_file', datasink_node, 'out_file') wf.connect(write_mindcontrol_entries, "output_json", datasink_node, "out_file.@json") #wf.write_graph(graph2use='exec') wf.run() From 5cadb09a8824f1625db7e771e29a516c2effb6f3 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 12 Jul 2018 14:01:38 -0400 Subject: [PATCH 05/60] Add freesurfer histograms --- auto_singularity_mindcontrol.py | 160 ++++++++++++++++---------------- 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/auto_singularity_mindcontrol.py b/auto_singularity_mindcontrol.py index c988afb..024e874 100644 --- a/auto_singularity_mindcontrol.py +++ b/auto_singularity_mindcontrol.py @@ -213,81 +213,82 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port """ file_server = f"http://localhost:{nginx_port}/files/" startup_file_server = f'http://localhost:{startup_port}/' - + default_module = { - "fields": [ - { - "function_name": "get_qc_viewer", - "id": "name", - "name": "Image File" - }, - { - "function_name": "get_qc_ave_field", - "id": "average_vote", - "name": "QC vote" - }, - { - "function_name": None, - "id": "num_votes", - "name": "# votes" - }, - { - "function_name": None, - "id": "quality_check.notes_QC", - "name": "Notes" - } - ], - "metric_names": None, - "graph_type": None, - "staticURL": file_server, - "usePeerJS": False, - "logPainter": False, - "logContours": False, - "logPoints": True, - "qc_options": {"pass": 1, "fail": 1, "needs_edits": 0, "edited": 0, "assignTo": 0, "notes": 1, "confidence": 1}} - + "fields": [ + { + "function_name": "get_qc_viewer", + "id": "name", + "name": "Image File" + }, + { + "function_name": "get_qc_ave_field", + "id": "average_vote", + "name": "QC vote" + }, + { + "function_name": None, + "id": "num_votes", + "name": "# votes" + }, + { + "function_name": None, + "id": "quality_check.notes_QC", + "name": "Notes" + } + ], + "metric_names": None, + "graph_type": None, + "staticURL": file_server, + "usePeerJS": False, + "logPainter": False, + "logContours": False, + "logPoints": True, + "qc_options": {"pass": 1, "fail": 1, "needs_edits": 0, "edited": 0, "assignTo": 0, "notes": 1, "confidence": 1} + } + fs_module = { - "fields": [ - { - "function_name": "get_filter_field", - "id": "subject", - "name": "Exam ID" - }, - { - "function_name": "get_qc_viewer", - "id": "name", - "name": "Freesurfer ID" - }, - { - "function_name": "get_qc_filter_field", - "id": "quality_check.QC", - "name": "QC" - }, - { - "function_name": "get_filter_field", - "id": "checkedBy", - "name": "checked by" - }, - { - "function_name": "get_filter_field", - "id": "quality_check.user_assign", - "name": "Assigned To" - }, - { - "function_name": None, - "id": "quality_check.notes_QC", - "name": "Notes" - } - ], - "metric_names": None, - "graph_type": None, - "staticURL": file_server, - "usePeerJS": False, - "logPainter": False, - "logContours": False, - "logPoints": True - } - + "fields": [ + { + "function_name": "get_filter_field", + "id": "subject", + "name": "Exam ID" + }, + { + "function_name": "get_qc_viewer", + "id": "name", + "name": "Freesurfer ID" + }, + { + "function_name": "get_qc_filter_field", + "id": "quality_check.QC", + "name": "QC" + }, + { + "function_name": "get_filter_field", + "id": "checkedBy", + "name": "checked by" + }, + { + "function_name": "get_filter_field", + "id": "quality_check.user_assign", + "name": "Assigned To" + }, + { + "function_name": None, + "id": "quality_check.notes_QC", + "name": "Notes" + } + ], + "metric_names": None, + "graph_type": "histogram", + "staticURL": file_server, + "usePeerJS": False, + "logPainter": False, + "logContours": False, + "logPoints": True + } + fs_cm_dict = {'aparcaseg': { "0":{"name": "Grayscale", @@ -341,7 +342,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port } if entry_types is None and not freesurfer: raise Exception("You must either define entry types or have freesurfer == True") - + modules = [] if entry_types is not None: for et in entry_types: @@ -349,9 +350,9 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port et_module["name"] = et et_module["entry_type"] = et modules.append(et_module) - + if freesurfer: - for et,cm in fs_cm_dict.items(): + for et, cm in fs_cm_dict.items(): et_module = fs_module.copy() et_module["name"] = fs_name_dict[et] et_module["entry_type"] = et @@ -359,7 +360,6 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port et_module['colormaps'] = cm modules.append(et_module) - # autogenerated settings files pub_set = {"startup_json": startup_file_server+"startup.json", "load_if_empty": True, @@ -368,7 +368,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port "modules": modules} settings = {"public": pub_set} with mcsetfile.open("w") as h: - json.dump(settings,h) + json.dump(settings, h) if __name__ == "__main__": docker_build_path = Path(__file__).resolve().parent / 'imports/docker' @@ -380,7 +380,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port help='Directory to bulid singularirty image and files in.') parser.add_argument('--custom_settings', help='Path to custom settings json') - parser.add_argument('--freesurfer', action='store_true', + parser.add_argument('--freesurfer', action='store_true', help='Generate settings for freesurfer QC in mindcontrol.') parser.add_argument('--entry_type', action='append', help='Name of mindcontrol module you would like to have autogenerated.' @@ -455,7 +455,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port # Write settings files write_passfile(passfile) write_nginxconf(ncfile) - write_meteorconf(mcfile, startup_port=startup_port, + write_meteorconf(mcfile, startup_port=startup_port, nginx_port=nginx_port, meteor_port=meteor_port) if custom_settings is not None: copyfile(custom_settings, mcsetfile.as_posix()) From d059b0343b559deeba6a42146a771249f60ce86a Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Fri, 6 Jul 2018 14:56:01 -0400 Subject: [PATCH 06/60] Working nginx singularity container --- .gitignore | 3 + imports/docker/Dockerfile_nginx | 101 ++++++++++++++++++ imports/docker/entrypoint_nginx.sh | 5 + imports/docker/settings/auth.conf | 12 +++ imports/docker/settings/auth.htpasswd | 1 + imports/docker/settings/log/access.log | 2 + imports/docker/settings/log/error.log | 0 imports/docker/settings/log/nginx_error.log | 2 + .../docker/settings/mc_nginx_settings.json | 1 + imports/docker/settings/meteor.conf | 23 ++++ imports/docker/settings/nginx.conf | 31 ++++++ 11 files changed, 181 insertions(+) create mode 100644 imports/docker/Dockerfile_nginx create mode 100644 imports/docker/entrypoint_nginx.sh create mode 100755 imports/docker/settings/auth.conf create mode 100755 imports/docker/settings/auth.htpasswd create mode 100644 imports/docker/settings/log/access.log create mode 100644 imports/docker/settings/log/error.log create mode 100644 imports/docker/settings/log/nginx_error.log create mode 100644 imports/docker/settings/mc_nginx_settings.json create mode 100644 imports/docker/settings/meteor.conf create mode 100755 imports/docker/settings/nginx.conf diff --git a/.gitignore b/.gitignore index b42c133..353bbbe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ node_modules/ .meteor/dev_bundle +.mindcontrol/ +imports/docker/log/ +imports/docker/temp_for_nginx/ \ No newline at end of file diff --git a/imports/docker/Dockerfile_nginx b/imports/docker/Dockerfile_nginx new file mode 100644 index 0000000..39ea4a3 --- /dev/null +++ b/imports/docker/Dockerfile_nginx @@ -0,0 +1,101 @@ +FROM nginx:1.15.1 + +MAINTAINER Dylan Nielson + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + vim \ + supervisor \ + && rm -rf /var/lib/apt/lists/* + +ENV METEOR_RELEASE 1.7.0.3 +RUN curl https://install.meteor.com/ 2>/dev/null | sed 's/^RELEASE/#RELEASE/'| RELEASE=$METEOR_RELEASE sh + +RUN ln -s ~/.meteor/packages/meteor-tool/*/mt-os.linux.x86_64/dev_bundle/bin/node /usr/bin/ && \ + ln -s ~/.meteor/packages/meteor-tool/*/mt-os.linux.x86_64/dev_bundle/bin/npm /usr/bin/ && \ + rm /etc/nginx/conf.d/default.conf + + +# Installing and setting up miniconda +RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash Miniconda*.sh -b -p /usr/local/miniconda && \ + rm Miniconda*.sh + +ENV PATH=/usr/local/miniconda/bin:$PATH \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 + +# Installing precomputed python packages +RUN conda install -c conda-forge -y \ + awscli \ + boto3 \ + dipy \ + git \ + matplotlib \ + numpy \ + python=3.6 \ + scikit-image \ + scikit-learn \ + wget; \ + sync && \ + chmod +x /usr/local/miniconda/bin/* && \ + conda clean --all -y; sync && \ + python -c "from matplotlib import font_manager" && \ + sed -i 's/\(backend *: \).*$/\1Agg/g' $( python -c "import matplotlib; print(matplotlib.matplotlib_fname())" ) + +RUN npm install http-server -g + +ENV MC_DIR /home/mindcontrol +ENV LC_ALL C + + +COPY entrypoint_nginx.sh /home/entrypoint.sh +COPY ndmg_launch.sh /home/ndmg_launch.sh + +RUN useradd --create-home --home-dir ${MC_DIR} mindcontrol +RUN chown mindcontrol:mindcontrol /home/entrypoint.sh +RUN chmod +x /home/entrypoint.sh + +RUN mkdir -p ${MC_DIR}/mindcontrol &&\ + chown -R mindcontrol /home/mindcontrol &&\ + chmod -R a+rx /home/mindcontrol + +USER mindcontrol + +RUN cd ${MC_DIR}/mindcontrol &&\ + git clone https://github.com/akeshavan/mindcontrol.git ${MC_DIR}/mindcontrol + #git clone https://github.com/clowdcontrol/mindcontrol.git ${MC_DIR}/mindcontrol + + +####### Attempt at nginx security + +WORKDIR /opt +COPY ./settings/meteor.conf /etc/nginx/conf.d/meteor.conf +COPY ./settings/auth.htpasswd /etc/nginx +COPY ./settings/nginx.conf /etc/nginx/nginx.conf +USER root +RUN chmod 777 /var/log/nginx /var/cache/nginx /opt /etc +RUN mkdir -p /mc_data /mc_startup_data +RUN chown mindcontrol:mindcontrol /etc/nginx/auth.htpasswd /mc_data /mc_startup_data +USER mindcontrol +###### + +###### Load in local settings +COPY ./settings/mc_nginx_settings.json ${MC_DIR}/mindcontrol/mc_nginx_settings.json + +# Make a spot that we can put our data +RUN mkdir -p /mc_data /mc_startup_data + +WORKDIR ${MC_DIR}/mindcontrol + +ENTRYPOINT ["/home/entrypoint.sh"] + + +EXPOSE 3000 +EXPOSE 2998 +ENV PORT 3000 + + diff --git a/imports/docker/entrypoint_nginx.sh b/imports/docker/entrypoint_nginx.sh new file mode 100644 index 0000000..68a0a04 --- /dev/null +++ b/imports/docker/entrypoint_nginx.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd /home/mindcontrol/mindcontrol +nohup meteor --settings mc_nginx_settings.json --port 2998 & +nginx -g "daemon off;" diff --git a/imports/docker/settings/auth.conf b/imports/docker/settings/auth.conf new file mode 100755 index 0000000..5ad4cc0 --- /dev/null +++ b/imports/docker/settings/auth.conf @@ -0,0 +1,12 @@ +error_log /var/log/nginx/nginx_error.log info; + +server { + listen 3002 default_server; + root /data; + auth_basic "Restricted"; + auth_basic_user_file auth.htpasswd; + location / { + autoindex on; + } + + } diff --git a/imports/docker/settings/auth.htpasswd b/imports/docker/settings/auth.htpasswd new file mode 100755 index 0000000..8082836 --- /dev/null +++ b/imports/docker/settings/auth.htpasswd @@ -0,0 +1 @@ +dylan:$apr1$ieFel5ou$qOZXoFEVATdKZs9J/dKdA0 diff --git a/imports/docker/settings/log/access.log b/imports/docker/settings/log/access.log new file mode 100644 index 0000000..d3b047f --- /dev/null +++ b/imports/docker/settings/log/access.log @@ -0,0 +1,2 @@ +172.17.0.1 - - [03/Jul/2018:15:00:10 +0000] "GET / HTTP/1.1" 401 597 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36" "-" +172.17.0.1 - - [03/Jul/2018:15:00:11 +0000] "GET / HTTP/1.1" 401 597 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36" "-" diff --git a/imports/docker/settings/log/error.log b/imports/docker/settings/log/error.log new file mode 100644 index 0000000..e69de29 diff --git a/imports/docker/settings/log/nginx_error.log b/imports/docker/settings/log/nginx_error.log new file mode 100644 index 0000000..a8d79cb --- /dev/null +++ b/imports/docker/settings/log/nginx_error.log @@ -0,0 +1,2 @@ +2018/07/03 15:00:10 [info] 7#7: *1 no user/password was provided for basic authentication, client: 172.17.0.1, server: , request: "GET / HTTP/1.1", host: "localhost:3002" +2018/07/03 15:00:11 [info] 7#7: *1 no user/password was provided for basic authentication, client: 172.17.0.1, server: , request: "GET / HTTP/1.1", host: "localhost:3002" diff --git a/imports/docker/settings/mc_nginx_settings.json b/imports/docker/settings/mc_nginx_settings.json new file mode 100644 index 0000000..e2a5a7d --- /dev/null +++ b/imports/docker/settings/mc_nginx_settings.json @@ -0,0 +1 @@ +{"public": {"startup_json": "http://localhost:3003/startup.freesurfer.json", "load_if_empty": true, "use_custom": true, "needs_consent": false, "modules": [{"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "aparcaseg", "entry_type": "aparcaseg", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "custom.Freesurfer", "alpha": 0.5}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "wm", "entry_type": "wm", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Spectrum", "alpha": 0.5}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "brainmask", "entry_type": "brainmask", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Red Overlay", "alpha": 0.2, "min": 0, "max": 2000}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "ribbon", "entry_type": "ribbon", "num_overlays": 3, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Green Overlay", "alpha": 0.5, "min": 0, "max": 2000}, "2": {"name": "Blue Overlay", "alpha": 0.5, "min": 0, "max": 2000}}}]}} \ No newline at end of file diff --git a/imports/docker/settings/meteor.conf b/imports/docker/settings/meteor.conf new file mode 100644 index 0000000..6aa8373 --- /dev/null +++ b/imports/docker/settings/meteor.conf @@ -0,0 +1,23 @@ +error_log /var/log/nginx/nginx_error.log info; + +server { + listen 3003 default_server; + root /mc_startup_data; + location / { + autoindex on; + } + + } + +server { + listen 3000 default_server; + auth_basic "Restricted"; + auth_basic_user_file auth.htpasswd; + + location / { + proxy_pass http://localhost:2998/; + } + location /files/ { + alias /mc_data/; + } + } \ No newline at end of file diff --git a/imports/docker/settings/nginx.conf b/imports/docker/settings/nginx.conf new file mode 100755 index 0000000..f2845ee --- /dev/null +++ b/imports/docker/settings/nginx.conf @@ -0,0 +1,31 @@ + +worker_processes 1; +pid /var/cache/nginx/nginx.pid; +error_log /var/log/nginx/error.log warn; + + +events { + worker_connections 1024; +} + + +http { + disable_symlinks off; + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/meteor.conf; +} From 07fc372935c662dd2b18d3aa3952849de44df85d Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 11 Jul 2018 17:16:40 -0400 Subject: [PATCH 07/60] Make work on HPC --- ...indcontrol.py => auto_local_mindcontrol.py | 0 auto_singularity_mindcontrol.py | 513 ++++++++++++++++++ imports/docker/Dockerfile_nginx | 24 +- imports/docker/entrypoint_nginx.sh | 12 +- .../docker/settings/mc_nginx_settings.json | 1 - start_singularity_mindcontrol.py | 270 +++++++++ 6 files changed, 807 insertions(+), 13 deletions(-) rename auto_mindcontrol.py => auto_local_mindcontrol.py (100%) create mode 100644 auto_singularity_mindcontrol.py delete mode 100644 imports/docker/settings/mc_nginx_settings.json create mode 100644 start_singularity_mindcontrol.py diff --git a/auto_mindcontrol.py b/auto_local_mindcontrol.py similarity index 100% rename from auto_mindcontrol.py rename to auto_local_mindcontrol.py diff --git a/auto_singularity_mindcontrol.py b/auto_singularity_mindcontrol.py new file mode 100644 index 0000000..40f2489 --- /dev/null +++ b/auto_singularity_mindcontrol.py @@ -0,0 +1,513 @@ +#! python +import argparse +from pathlib import Path +import json +from shutil import copyfile +import os +import getpass +from passlib.hash import bcrypt +import subprocess +import random +import sys + + +# HT password code from https://gist.github.com/eculver/1420227 + +# We need a crypt module, but Windows doesn't have one by default. Try to find +# one, and tell the user if we can't. +try: + import crypt +except ImportError: + try: + import fcrypt as crypt + except ImportError: + sys.stderr.write("Cannot find a crypt module. " + "Possibly http://carey.geek.nz/code/python-fcrypt/\n") + sys.exit(1) + + +def salt(): + """Returns a string of 2 randome letters""" + letters = 'abcdefghijklmnopqrstuvwxyz' \ + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' \ + '0123456789/.' + return random.choice(letters) + random.choice(letters) + + +class HtpasswdFile: + """A class for manipulating htpasswd files.""" + + def __init__(self, filename, create=False): + self.entries = [] + self.filename = filename + if not create: + if os.path.exists(self.filename): + self.load() + else: + raise Exception("%s does not exist" % self.filename) + + def load(self): + """Read the htpasswd file into memory.""" + lines = open(self.filename, 'r').readlines() + self.entries = [] + for line in lines: + username, pwhash = line.split(':') + entry = [username, pwhash.rstrip()] + self.entries.append(entry) + + def save(self): + """Write the htpasswd file to disk""" + open(self.filename, 'w').writelines(["%s:%s\n" % (entry[0], entry[1]) + for entry in self.entries]) + + def update(self, username, password): + """Replace the entry for the given user, or add it if new.""" + pwhash = crypt.crypt(password, salt()) + matching_entries = [entry for entry in self.entries + if entry[0] == username] + if matching_entries: + matching_entries[0][1] = pwhash + else: + self.entries.append([username, pwhash]) + + def delete(self, username): + """Remove the entry for the given user.""" + self.entries = [entry for entry in self.entries + if entry[0] != username] + + + +def write_passfile(passfile_path): + """Collects usernames and passwords and writes them to + the provided path with bcrypt encryption. + Parameters + ---------- + passfile: pathlib.Path object + The path to which to write the usernames and hashed passwords. + """ + users = set() + done = False + passfile = HtpasswdFile(passfile_path.as_posix(), create=True) + while not done: + user = "" + print("Please enter usernames and passwords for all the users you'd like to create.", flush=True) + user = input("Input user, leave blank if you are finished entering users:") + if len(users) > 0 and user == "": + print("All users entered, generating auth.htpass file", flush=True) + done= True + passfile.save() + elif len(users) == 0 and user == "": + print("Please enter at least one user", flush=True) + else: + if user in users: + print("Duplicate user, overwriting previously entered password for %s."%user, flush=True) + hs = None + while hs is None: + a = getpass.getpass(prompt="Enter Password for user: %s\n"%user) + b = getpass.getpass(prompt="Re-enter Password for user: %s\n"%user) + if a == b: + hs = "valid_pass" + passfile.update(user, a) + else: + print("Entered passwords don't match, please try again.", flush=True) + users.add(user) + + +def write_meteorconf(mcfile, startup_port=3003, nginx_port=3000, meteor_port=2998): + """Write nginx configuration file for meteor given user specified ports. + Parameters + ---------- + mcfile: pathlib.Path object + The path to which to write the config file. + startup_port: int, default is 3003 + Port number at which mindcontrol will look for startup manifest. + nginx_port: int, default is 3000 + Port number at nginx will run. This is the port you connect to reach mindcontrol. + meteor_port: int, default is 2998 + Port number at meteor will run. This is mostly under the hood, but you might + need to change it if there is a port conflict. Mongo will run on the port + one above the meteor_port. + """ + mc_string=f"""error_log /var/log/nginx/nginx_error.log info; + +server {{ + listen {startup_port} default_server; + root /mc_startup_data; + location / {{ + autoindex on; + }} + + }} + +server {{ + listen {nginx_port} default_server; + auth_basic "Restricted"; + auth_basic_user_file auth.htpasswd; + + location / {{ + proxy_pass http://localhost:{meteor_port}/; + }} + location /files/ {{ + alias /mc_data/; + }} + }}""" + mcfile.write_text(mc_string) + + +def write_nginxconf(ncfile): + """Write top level nginx configuration file. + Parameters + ---------- + ncfile: pathlib.Path object + The path to which to write the config file. + """ + nc_string=f"""worker_processes 1; +pid /var/cache/nginx/nginx.pid; +error_log /var/log/nginx/error.log warn; + + +events {{ + worker_connections 1024; +}} + + +http {{ + disable_symlinks off; + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/meteor.conf; +}} +""" + ncfile.write_text(nc_string) + + +def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port=3003, nginx_port=3000): + """ Write the mindcontrol settings json. This determines which panels mindcontrol + displays and which that information comes from. + Parameters + ---------- + mcfile: pathlib.Path object + The path to which to write the json file. + entry_types: optional, list of strings + List of names of modules you would like mindcontrol to display + freesurfer: optional, bool + True if you would like the settings generated modules for qcing aparc-aseg, wm, and ribbon + startup_port: int, default is 3003 + Port number at which mindcontrol will look for startup manifest. + nginx_port: int, default is 3000 + Port number at nginx will run. This is the port you connect to reach mindcontrol. + """ + file_server = f"http://localhost:{nginx_port}/files/" + startup_file_server = f'http://localhost:{startup_port}/' + + default_module = { + "fields": [ + { + "function_name": "get_qc_viewer", + "id": "name", + "name": "Image File" + }, + { + "function_name": "get_qc_ave_field", + "id": "average_vote", + "name": "QC vote" + }, + { + "function_name": None, + "id": "num_votes", + "name": "# votes" + }, + { + "function_name": None, + "id": "quality_check.notes_QC", + "name": "Notes" + } + ], + "metric_names": None, + "graph_type": None, + "staticURL": file_server, + "usePeerJS": False, + "logPainter": False, + "logContours": False, + "logPoints": True, + "qc_options": {"pass": 1, "fail": 1, "needs_edits": 0, "edited": 0, "assignTo": 0, "notes": 1, "confidence": 1}} + + fs_module = { + "fields": [ + { + "function_name": "get_filter_field", + "id": "subject", + "name": "Exam ID" + }, + { + "function_name": "get_qc_viewer", + "id": "name", + "name": "Freesurfer ID" + }, + { + "function_name": "get_qc_filter_field", + "id": "quality_check.QC", + "name": "QC" + }, + { + "function_name": "get_filter_field", + "id": "checkedBy", + "name": "checked by" + }, + { + "function_name": "get_filter_field", + "id": "quality_check.user_assign", + "name": "Assigned To" + }, + { + "function_name": None, + "id": "quality_check.notes_QC", + "name": "Notes" + } + ], + "metric_names": None, + "graph_type": None, + "staticURL": file_server, + "usePeerJS": False, + "logPainter": False, + "logContours": False, + "logPoints": True + } + + fs_cm_dict = {'aparcaseg': + { + "0":{"name": "Grayscale", + "alpha": 1, + "min": 0, + "max": 255 + }, + "1": { + "name": "custom.Freesurfer", + "alpha": 0.5 + } + }, + 'brainmask': + { + "0":{"name": "Grayscale", + "alpha": 1, + "min": 0, + "max": 255 + }, + "1": { + "name": "Red Overlay", + "alpha": 0.2, + "min": 0, + "max": 2000 + } + }, + 'ribbon': + { + "0":{"name": "Grayscale", + "alpha": 1, + "min": 0, + "max": 255 + }, + "1": { + "name": "Green Overlay", + "alpha": 0.5, + "min": 0, + "max": 2000 + }, + "2": { + "name": "Blue Overlay", + "alpha": 0.5, + "min":0, + "max": 2000 + } + } + } + fs_name_dict = {'brainmask': 'Brain Mask', + 'aparcaseg': 'Segmentation', + 'ribbon': 'White Matter', + } + if entry_types is None and not freesurfer: + raise Exception("You must either define entry types or have freesurfer == True") + + modules = [] + if entry_types is not None: + for et in entry_types: + et_module = default_module.copy() + et_module["name"] = et + et_module["entry_type"] = et + modules.append(et_module) + + if freesurfer: + for et,cm in fs_cm_dict.items(): + et_module = fs_module.copy() + et_module["name"] = fs_name_dict[et] + et_module["entry_type"] = et + et_module["num_overlays"] = len(cm) + et_module['colormaps'] = cm + modules.append(et_module) + + + # autogenerated settings files + pub_set = {"startup_json": startup_file_server+"startup.json", + "load_if_empty": True, + "use_custom": True, + "needs_consent": False, + "modules": modules} + settings = {"public": pub_set} + with mcsetfile.open("w") as h: + json.dump(settings,h) + +if __name__ == "__main__": + docker_build_path = Path(__file__).resolve().parent / 'imports/docker' + parser = argparse.ArgumentParser(description='Autogenerate singularity image and settings files, ' + 'for running mindcontrol in a singularity image on ' + 'another system that is hosting the data.') + parser.add_argument('--sing_out_dir', + default='.', + help='Directory to bulid singularirty image and files in.') + parser.add_argument('--custom_settings', + help='Path to custom settings json') + parser.add_argument('--freesurfer', action='store_true', + help='Generate settings for freesurfer QC in mindcontrol.') + parser.add_argument('--entry_type', action='append', + help='Name of mindcontrol module you would like to have autogenerated.' + ' This should correspond to the bids image type ' + '(specified after the final _ of the image name). ' + ' Pass this argument multiple times to add additional modules.') + parser.add_argument('--dockerfile', default=docker_build_path, + help='Path to the mindcontrol nginx dockerfile, defaults to %s'%docker_build_path) + parser.add_argument('--startup_port', + default=3003, + help='Port number at which mindcontrol will look for startup manifest.') + parser.add_argument('--nginx_port', + default=3000, + help='Port number at nginx will run. This is the port you connect to reach mindcontrol.') + parser.add_argument('--meteor_port', + default=2998, + help='Port number at meteor will run. ' + 'This is mostly under the hood, ' + 'but you might need to change it if there is a port conflict.' + 'Mongo will run on the port one above this one.') + + args = parser.parse_args() + sing_out_dir = args.sing_out_dir + if args.custom_settings is not None: + custom_settings = Path(args.custom_settings) + else: + custom_settings = None + freesurfer = args.freesurfer + entry_types = args.entry_type + startup_port = args.startup_port + nginx_port = args.nginx_port + meteor_port = args.meteor_port + + # Set up directory to be copied + basedir = Path(sing_out_dir) + setdir = basedir/"settings" + mcsetdir = setdir/"mc_settings" + manifest_dir = setdir/"mc_manifest_init" + meteor_ldir = basedir/".meteor" + + logdir = basedir/"log" + scratch_dir = logdir/"scratch" + nginx_scratch = scratch_dir/"nginx" + mc_hdir = scratch_dir/"singularity_home" + + if not basedir.exists(): + basedir.mkdir() + if not setdir.exists(): + setdir.mkdir() + if not logdir.exists(): + logdir.mkdir() + if not scratch_dir.exists(): + scratch_dir.mkdir() + if not nginx_scratch.exists(): + nginx_scratch.mkdir() + if not mcsetdir.exists(): + mcsetdir.mkdir() + if not manifest_dir.exists(): + manifest_dir.mkdir() + if not mc_hdir.exists(): + mc_hdir.mkdir() + if not meteor_ldir.exists(): + meteor_ldir.mkdir() + dockerfile = basedir/"Dockerfile_nginx" + entrypoint = basedir/"entrypoint_nginx.sh" + passfile = setdir/"auth.htpasswd" + mcfile = setdir/"meteor.conf" + ncfile = setdir/"nginx.conf" + mcsetfile = mcsetdir/"mc_nginx_settings.json" + infofile = setdir/"mc_info.json" + + # Write settings files + write_passfile(passfile) + write_nginxconf(ncfile) + write_meteorconf(mcfile, startup_port=startup_port, + nginx_port=nginx_port, meteor_port=meteor_port) + if custom_settings is not None: + copyfile(custom_settings, mcsetfile.as_posix()) + else: + write_mcsettings(mcsetfile, entry_types=entry_types, freesurfer=freesurfer, + startup_port=startup_port, nginx_port=nginx_port) + # Copy singularity run script to directory + srun_source = Path(__file__).resolve().parent / 'start_singularity_mindcontrol.py' + srun_dest = basedir / 'start_singularity_mindcontrol.py' + copyfile(srun_source.as_posix(), srun_dest.as_posix()) + + # Copy entrypoint script to directory + copyfile((docker_build_path / 'entrypoint_nginx.sh').as_posix(), entrypoint.as_posix()) + + # Copy dockerfile to base directory + copyfile((docker_build_path / 'Dockerfile_nginx').as_posix(), dockerfile.as_posix()) + + # Run docker build + subprocess.run("docker build -f Dockerfile_nginx -t auto_mc_nginx .", + cwd=basedir.as_posix(), shell=True, check=True) + + # Run docker2singularity + subprocess.run("docker run -v /var/run/docker.sock:/var/run/docker.sock " + "-v ${PWD}:/output --privileged -t --rm " + "singularityware/docker2singularity auto_mc_nginx", + cwd=basedir.as_posix(), shell=True, check=True) + + # Get name of singularity image + simg = [si for si in basedir.glob("auto_mc_nginx*.img")][0] + + info = dict(entry_types=entry_types, + freesurfer=freesurfer, + startup_port=startup_port, + nginx_port=nginx_port, + meteor_port=meteor_port, + simg=simg.parts[-1]) + + with infofile.open("w") as h: + json.dump(info, h) + + # Print next steps + print("Finished building singuliarity image and settings files.") + print(f"Copy {basedir} to machine that will be hosting the mindcontrol instance.") + print("Consider the following command: ") + print(f"rsync -avch {basedir} [host machine]:[destination path]") + print("Then, on the machine hosting the mindcontrol instance") + print(f"run the start_singularity_mindcontrol.py script included in {basedir}.") + print("Consider the following commands: ") + print(f"cd [destination path]/{basedir.parts[-1]}") + if freesurfer: + print(f"python start_singularity_mindcontrol.py --bids_dir [path to bids dir] --freesurfer_dir [path to freesurfer outputs]") + else: + print(f"python start_singularity_mindcontrol.py --bids_dir [path to bids dir]") + + diff --git a/imports/docker/Dockerfile_nginx b/imports/docker/Dockerfile_nginx index 39ea4a3..5311dbf 100644 --- a/imports/docker/Dockerfile_nginx +++ b/imports/docker/Dockerfile_nginx @@ -9,6 +9,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ vim \ supervisor \ + rsync \ + procps \ && rm -rf /var/lib/apt/lists/* ENV METEOR_RELEASE 1.7.0.3 @@ -53,20 +55,22 @@ ENV LC_ALL C COPY entrypoint_nginx.sh /home/entrypoint.sh -COPY ndmg_launch.sh /home/ndmg_launch.sh +#COPY ndmg_launch.sh /home/ndmg_launch.sh RUN useradd --create-home --home-dir ${MC_DIR} mindcontrol -RUN chown mindcontrol:mindcontrol /home/entrypoint.sh -RUN chmod +x /home/entrypoint.sh - -RUN mkdir -p ${MC_DIR}/mindcontrol &&\ +RUN chown mindcontrol:mindcontrol /home/entrypoint.sh &&\ + chmod +x /home/entrypoint.sh &&\ + mkdir -p ${MC_DIR}/mindcontrol &&\ chown -R mindcontrol /home/mindcontrol &&\ chmod -R a+rx /home/mindcontrol USER mindcontrol RUN cd ${MC_DIR}/mindcontrol &&\ - git clone https://github.com/akeshavan/mindcontrol.git ${MC_DIR}/mindcontrol + git clone https://github.com/akeshavan/mindcontrol.git ${MC_DIR}/mindcontrol &&\ + meteor update &&\ + meteor npm install --save @babel/runtime &&\ + meteor npm install --save bcrypt #git clone https://github.com/clowdcontrol/mindcontrol.git ${MC_DIR}/mindcontrol @@ -77,14 +81,14 @@ COPY ./settings/meteor.conf /etc/nginx/conf.d/meteor.conf COPY ./settings/auth.htpasswd /etc/nginx COPY ./settings/nginx.conf /etc/nginx/nginx.conf USER root -RUN chmod 777 /var/log/nginx /var/cache/nginx /opt /etc -RUN mkdir -p /mc_data /mc_startup_data -RUN chown mindcontrol:mindcontrol /etc/nginx/auth.htpasswd /mc_data /mc_startup_data +RUN chmod 777 /var/log/nginx /var/cache/nginx /opt /etc &&\ + mkdir -p /mc_data /mc_startup_data /mc_settings &&\ + chown mindcontrol:mindcontrol /etc/nginx/auth.htpasswd /mc_data /mc_startup_data /mc_settings USER mindcontrol ###### ###### Load in local settings -COPY ./settings/mc_nginx_settings.json ${MC_DIR}/mindcontrol/mc_nginx_settings.json +COPY ./settings/mc_settings/mc_nginx_settings.json /mc_settings/mc_nginx_settings.json # Make a spot that we can put our data RUN mkdir -p /mc_data /mc_startup_data diff --git a/imports/docker/entrypoint_nginx.sh b/imports/docker/entrypoint_nginx.sh index 68a0a04..4731806 100644 --- a/imports/docker/entrypoint_nginx.sh +++ b/imports/docker/entrypoint_nginx.sh @@ -1,5 +1,13 @@ #!/bin/bash -cd /home/mindcontrol/mindcontrol -nohup meteor --settings mc_nginx_settings.json --port 2998 & +cd ~ +if [ ! -x .meteor ]; then + echo "Copying meteor files into singularity_home" + rsync -ach /home/mindcontrol/mindcontrol . + rsync -ach /home/mindcontrol/.meteor . + rsync -ach /home/mindcontrol/.cordova . + ln -s /home/mindcontrol/mindcontrol/.meteor/local ~/mindcontrol/.meteor/local/ +fi +cd ~/mindcontrol +nohup meteor --settings /mc_settings/mc_nginx_settings.json --port 2998 & nginx -g "daemon off;" diff --git a/imports/docker/settings/mc_nginx_settings.json b/imports/docker/settings/mc_nginx_settings.json deleted file mode 100644 index e2a5a7d..0000000 --- a/imports/docker/settings/mc_nginx_settings.json +++ /dev/null @@ -1 +0,0 @@ -{"public": {"startup_json": "http://localhost:3003/startup.freesurfer.json", "load_if_empty": true, "use_custom": true, "needs_consent": false, "modules": [{"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "aparcaseg", "entry_type": "aparcaseg", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "custom.Freesurfer", "alpha": 0.5}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "wm", "entry_type": "wm", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Spectrum", "alpha": 0.5}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "brainmask", "entry_type": "brainmask", "num_overlays": 2, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Red Overlay", "alpha": 0.2, "min": 0, "max": 2000}}}, {"fields": [{"function_name": "get_filter_field", "id": "subject", "name": "Exam ID"}, {"function_name": "get_qc_viewer", "id": "name", "name": "Freesurfer ID"}, {"function_name": "get_qc_filter_field", "id": "quality_check.QC", "name": "QC"}, {"function_name": "get_filter_field", "id": "checkedBy", "name": "checked by"}, {"function_name": "get_filter_field", "id": "quality_check.user_assign", "name": "Assigned To"}, {"function_name": null, "id": "quality_check.notes_QC", "name": "Notes"}], "metric_names": null, "graph_type": null, "staticURL": "http://localhost:3000/files/", "usePeerJS": false, "logPainter": false, "logContours": false, "logPoints": true, "name": "ribbon", "entry_type": "ribbon", "num_overlays": 3, "colormaps": {"0": {"name": "Grayscale", "alpha": 1, "min": 0, "max": 255}, "1": {"name": "Green Overlay", "alpha": 0.5, "min": 0, "max": 2000}, "2": {"name": "Blue Overlay", "alpha": 0.5, "min": 0, "max": 2000}}}]}} \ No newline at end of file diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py new file mode 100644 index 0000000..1c2efd0 --- /dev/null +++ b/start_singularity_mindcontrol.py @@ -0,0 +1,270 @@ +#! python +import argparse +from pathlib import Path +import json +from bids.grabbids import BIDSLayout +import subprocess +import os +import shutil +from nipype import MapNode, Workflow, Node +from nipype.interfaces.freesurfer import MRIConvert +from nipype.interfaces.io import DataSink +from nipype.interfaces.utility import IdentityInterface, Function +#this function finds data in the subjects_dir +def data_grabber(subjects_dir, subject, volumes): + import os + volumes_list = [os.path.join(subjects_dir, subject, 'mri', volume) for volume in volumes] + return volumes_list + +#this function parses the aseg.stats, lh.aparc.stats and rh.aparc.stats and returns a dictionary +def parse_stats(subjects_dir, subject): + from os.path import join, exists + + aseg_file = join(subjects_dir, subject, "stats", "aseg.stats") + lh_aparc = join(subjects_dir, subject, "stats", "lh.aparc.stats") + rh_aparc = join(subjects_dir, subject, "stats", "rh.aparc.stats") + + assert exists(aseg_file), "aseg file does not exists for %s" %subject + assert exists(lh_aparc), "lh aparc file does not exists for %s" %subject + assert exists(rh_aparc), "rh aparc file does not exists for %s" %subject + + def convert_stats_to_json(aseg_file, lh_aparc, rh_aparc): + import pandas as pd + import numpy as np + + def extract_other_vals_from_aseg(f): + value_labels = ["EstimatedTotalIntraCranialVol", + "Mask", + "TotalGray", + "SubCortGray", + "Cortex", + "CerebralWhiteMatter", + "CorticalWhiteMatterVol"] + value_labels = list(map(lambda x: 'Measure ' + x + ',', value_labels)) + output = pd.DataFrame() + with open(f,"r") as q: + out = q.readlines() + relevant_entries = [x for x in out if any(v in x for v in value_labels)] + for val in relevant_entries: + sname= val.split(",")[1][1:] + vol = val.split(",")[-2] + output = output.append(pd.Series({"StructName":sname,"Volume_mm3":vol}),ignore_index=True) + return output + + df = pd.DataFrame(np.genfromtxt(aseg_file,dtype=str),columns=["Index", + "SegId", + "NVoxels", + "Volume_mm3", + "StructName", + "normMean", + "normStdDev", + "normMin", + "normMax", + "normRange"]) + + df = df.append(extract_other_vals_from_aseg(aseg_file), ignore_index=True) + + aparc_columns = ["StructName", "NumVert", "SurfArea", "GrayVol", + "ThickAvg", "ThickStd", "MeanCurv", "GausCurv", + "FoldInd", "CurvInd"] + tmp_lh = pd.DataFrame(np.genfromtxt(lh_aparc,dtype=str),columns=aparc_columns) + tmp_lh["StructName"] = "lh_"+tmp_lh["StructName"] + tmp_rh = pd.DataFrame(np.genfromtxt(rh_aparc,dtype=str),columns=aparc_columns) + tmp_rh["StructName"] = "rh_"+tmp_rh["StructName"] + + aseg_melt = pd.melt(df[["StructName","Volume_mm3"]], id_vars=["StructName"]) + aseg_melt.rename(columns={"StructName": "name"},inplace=True) + aseg_melt["value"] = aseg_melt["value"].astype(float) + + lh_aparc_melt = pd.melt(tmp_lh,id_vars=["StructName"]) + lh_aparc_melt["value"] = lh_aparc_melt["value"].astype(float) + lh_aparc_melt["name"] = lh_aparc_melt["StructName"]+ "_"+lh_aparc_melt["variable"] + + rh_aparc_melt = pd.melt(tmp_rh, id_vars=["StructName"]) + rh_aparc_melt["value"] = rh_aparc_melt["value"].astype(float) + rh_aparc_melt["name"] = rh_aparc_melt["StructName"]+ "_"+rh_aparc_melt["variable"] + + output = aseg_melt[["name", + "value"]].append(lh_aparc_melt[["name", + "value"]], + ignore_index=True).append(rh_aparc_melt[["name", + "value"]], + ignore_index=True) + outdict = output.to_dict(orient="records") + final_dict = {} + for pair in outdict: + final_dict[pair["name"]] = pair["value"] + return final_dict + + output_dict = convert_stats_to_json(aseg_file, lh_aparc, rh_aparc) + return output_dict + +# This function creates valid Mindcontrol entries that are saved as .json files. # This f +# They can be loaded into the Mindcontrol database later + +def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, startup_json_path): + import os + from nipype.utils.filemanip import save_json + + cortical_wm = "CerebralWhiteMatterVol" # for later FS version + if not stats.get(cortical_wm): + cortical_wm = "CorticalWhiteMatterVol" + + metric_split = {"brainmask": ["eTIV", "CortexVol", "TotalGrayVol"], + "wm": [cortical_wm,"WM-hypointensities", + "Right-WM-hypointensities","Left-WM-hypointensities"], + "aparcaseg":[]} + + volumes = {'aparcaseg': ['T1.nii.gz', 'aparc+aseg.nii.gz'], + 'brainmask': ['T1.nii.gz', 'brainmask.nii.gz'], + 'wm': ['T1.nii.gz', 'ribbon.nii.gz', 'wm.nii.gz']} + + all_entries = [] + + for idx, entry_type in enumerate(["brainmask", "wm", "aparcaseg"]): + entry = {"entry_type":entry_type, + "subject_id": subject, + "name": subject} + volumes_list = [os.path.relpath(os.path.join(output_dir, subject, volume), mindcontrol_base_dir) + for volume in volumes[entry_type]] + entry["check_masks"] = volumes_list + entry["metrics"] = {} + for metric_name in metric_split[entry_type]: + entry["metrics"][metric_name] = stats.pop(metric_name) + if not len(metric_split[entry_type]): + entry["metrics"] = stats + all_entries.append(entry) + + output_json = os.path.abspath(os.path.join(startup_json_path)) + save_json(output_json, all_entries) + return output_json + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Start mindcontrol in a previously built' + ' singularity container and create initial manifest' + ' if needed.') + parser.add_argument('--bids_dir', help='The directory with the input dataset ' + 'formatted according to the BIDS standard.') + parser.add_argument('--freesurfer_dir', help='The directory with the freesurfer dirivatives,' + ' should be inside the bids directory') + parser.add_argument('--mc_singularity_path', default = '.', + help="Path to the directory containing the mindcontrol" + "singularity image and settings files.") + parser.add_argument('--no_server', action = 'store_true', + help="Don't start the mindcontrol server, just generate the manifest") + + args = parser.parse_args() + if args.bids_dir is not None: + bids_dir = Path(args.bids_dir) + else: + bids_dir = None + + if args.freesurfer_dir is not None: + freesurfer_dir = Path(args.freesurfer_dir) + else: + freesurfer_dir = None + + mc_singularity_path = Path(args.mc_singularity_path) + + no_server = args.no_server + + infofile = mc_singularity_path/'settings/mc_info.json' + with infofile.open('r') as h: + info = json.load(h) + manifest_dir = mc_singularity_path/'settings/mc_manifest_init/' + manifest_json = (manifest_dir/'startup.json').resolve() + + # First create the initial manifest + manifest = [] + if info['entry_types'] is not None: + entry_types = set(info['entry_types']) + unused_types = set() + for img in layout.get(extensions = ".nii.gz"): + if img.type in entry_types: + img_dict = {} + img_dict["check_masks"] = [img.filename.replace(bids_dir.as_posix(),"")] + img_dict["entry_type"] = img.type + img_dict["metrics"] = {} + img_dict["name"] = os.path.split(img.filename)[1].split('.')[0] + img_dict["subject"] = 'sub-' + img.subject + img_dict["session"] = 'ses-' + img.session + manifest.append(img_dict) + else: + unused_types.add(img.type) + if info['freesurfer']: + if freesurfer_dir is None: + #TODO: look in default location for freesurfer directory + raise Exception("Must specify the path to freesurfer files.") + + subjects = [] + for path in freesurfer_dir.glob('*'): + subject = path.parts[-1] + # check if mri dir exists, and don't add fsaverage + if os.path.exists(os.path.join(path, 'mri')) and subject != 'fsaverage': + subjects.append(subject) + + volumes = ["brainmask.mgz", "wm.mgz", "aparc+aseg.mgz", "T1.mgz", "ribbon.mgz"] + input_node = Node(IdentityInterface(fields=['subject_id',"subjects_dir", + "mindcontrol_base_dir", "output_dir", "startup_json_path"]), name='inputnode') + + input_node.iterables=("subject_id", subjects) + input_node.inputs.subjects_dir = freesurfer_dir + input_node.inputs.mindcontrol_base_dir = bids_dir.as_posix() #this is where start_static_server is running + input_node.inputs.output_dir = freesurfer_dir.as_posix() #this is in the freesurfer/ directory under the base_dir + input_node.inputs.startup_json_path = manifest_json.as_posix() + + dg_node=Node(Function(input_names=["subjects_dir", "subject", "volumes"], + output_names=["volume_paths"], + function=data_grabber), + name="datagrab") + #dg_node.inputs.subjects_dir = subjects_dir + dg_node.inputs.volumes = volumes + + mriconvert_node = MapNode(MRIConvert(out_type="niigz"), + iterfield=["in_file"], + name='convert') + + get_stats_node = Node(Function(input_names=["subjects_dir", "subject"], + output_names = ["output_dict"], + function=parse_stats), name="get_freesurfer_stats") + + write_mindcontrol_entries = Node(Function(input_names = ["mindcontrol_base_dir", + "output_dir", + "subject", + "stats", + "startup_json_path"], + output_names=["output_json"], + function=create_mindcontrol_entries), + name="get_mindcontrol_entries") + + datasink_node = Node(DataSink(), + name='datasink') + subst = [('out_file',''),('_subject_id_',''),('_out','')] + [("_convert%d" % index, "") for index in range(len(volumes))] + datasink_node.inputs.substitutions = subst + workflow_working_dir = os.path.abspath("./log/scratch") + + wf = Workflow(name="MindPrepFS") + wf.base_dir = workflow_working_dir + wf.connect(input_node,"subject_id", dg_node,"subject") + wf.connect(input_node,"subjects_dir", dg_node, "subjects_dir") + wf.connect(input_node, "subject_id", get_stats_node, "subject") + wf.connect(input_node, "subjects_dir", get_stats_node, "subjects_dir") + wf.connect(input_node, "subject_id", write_mindcontrol_entries, "subject") + wf.connect(input_node, "mindcontrol_base_dir", write_mindcontrol_entries, "mindcontrol_base_dir") + wf.connect(input_node, "output_dir", write_mindcontrol_entries, "output_dir") + wf.connect(input_node, "startup_json_path", write_mindcontrol_entries, "startup_json_path") + wf.connect(get_stats_node, "output_dict", write_mindcontrol_entries, "stats") + wf.connect(input_node, "output_dir", datasink_node, "base_directory") + wf.connect(dg_node,"volume_paths", mriconvert_node, "in_file") + wf.connect(mriconvert_node,'out_file',datasink_node,'out_file') + wf.connect(write_mindcontrol_entries, "output_json", datasink_node, "out_file.@json") + #wf.write_graph(graph2use='exec') + wf.run() + + if not args.no_server: + cmd = f"singularity run -B ${{PWD}}/log:/var/log/nginx -B {bids_dir}:/mc_data" \ + + f" -B settings:/opt/settings -B {manifest_dir}:/mc_startup_data" \ + + f" -B ${{PWD}}/log/scratch/nginx/:/var/cache/nginx -B ${{PWD}}/.meteor/:/home/mindcontrol/mindcontrol/.meteor/local" \ + + f" -B ${{PWD}}/settings/mc_settings:/mc_settings -H ${{PWD}}/log/scratch/singularity_home ${{PWD}}/{info['simg']}" + print(cmd) + subprocess.run(cmd, cwd = mc_singularity_path, shell=True, check=True) From 23f2844a068539435ca1d04b91e53b3dcdc837a6 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 12 Jul 2018 11:31:49 -0400 Subject: [PATCH 08/60] Fix start_singularity_mindcontrol.py --- start_singularity_mindcontrol.py | 166 +++++++++++++++++-------------- 1 file changed, 93 insertions(+), 73 deletions(-) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 1c2efd0..958f66f 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -10,12 +10,15 @@ from nipype.interfaces.freesurfer import MRIConvert from nipype.interfaces.io import DataSink from nipype.interfaces.utility import IdentityInterface, Function + + #this function finds data in the subjects_dir def data_grabber(subjects_dir, subject, volumes): import os volumes_list = [os.path.join(subjects_dir, subject, 'mri', volume) for volume in volumes] return volumes_list + #this function parses the aseg.stats, lh.aparc.stats and rh.aparc.stats and returns a dictionary def parse_stats(subjects_dir, subject): from os.path import join, exists @@ -24,9 +27,9 @@ def parse_stats(subjects_dir, subject): lh_aparc = join(subjects_dir, subject, "stats", "lh.aparc.stats") rh_aparc = join(subjects_dir, subject, "stats", "rh.aparc.stats") - assert exists(aseg_file), "aseg file does not exists for %s" %subject - assert exists(lh_aparc), "lh aparc file does not exists for %s" %subject - assert exists(rh_aparc), "rh aparc file does not exists for %s" %subject + assert exists(aseg_file), "aseg file does not exists for %s" % subject + assert exists(lh_aparc), "lh aparc file does not exists for %s" % subject + assert exists(rh_aparc), "rh aparc file does not exists for %s" % subject def convert_stats_to_json(aseg_file, lh_aparc, rh_aparc): import pandas as pd @@ -34,56 +37,63 @@ def convert_stats_to_json(aseg_file, lh_aparc, rh_aparc): def extract_other_vals_from_aseg(f): value_labels = ["EstimatedTotalIntraCranialVol", - "Mask", - "TotalGray", - "SubCortGray", - "Cortex", - "CerebralWhiteMatter", - "CorticalWhiteMatterVol"] + "Mask", + "TotalGray", + "SubCortGray", + "Cortex", + "CerebralWhiteMatter", + "CorticalWhiteMatterVol"] value_labels = list(map(lambda x: 'Measure ' + x + ',', value_labels)) output = pd.DataFrame() - with open(f,"r") as q: + with open(f, "r") as q: out = q.readlines() relevant_entries = [x for x in out if any(v in x for v in value_labels)] for val in relevant_entries: - sname= val.split(",")[1][1:] + sname = val.split(",")[1][1:] vol = val.split(",")[-2] - output = output.append(pd.Series({"StructName":sname,"Volume_mm3":vol}),ignore_index=True) + output = output.append(pd.Series({"StructName": sname, + "Volume_mm3": vol}), + ignore_index=True) return output - df = pd.DataFrame(np.genfromtxt(aseg_file,dtype=str),columns=["Index", - "SegId", - "NVoxels", - "Volume_mm3", - "StructName", - "normMean", - "normStdDev", - "normMin", - "normMax", - "normRange"]) + df = pd.DataFrame(np.genfromtxt(aseg_file, dtype=str), + columns=["Index", + "SegId", + "NVoxels", + "Volume_mm3", + "StructName", + "normMean", + "normStdDev", + "normMin", + "normMax", + "normRange"]) df = df.append(extract_other_vals_from_aseg(aseg_file), ignore_index=True) - + aparc_columns = ["StructName", "NumVert", "SurfArea", "GrayVol", "ThickAvg", "ThickStd", "MeanCurv", "GausCurv", "FoldInd", "CurvInd"] - tmp_lh = pd.DataFrame(np.genfromtxt(lh_aparc,dtype=str),columns=aparc_columns) + tmp_lh = pd.DataFrame(np.genfromtxt(lh_aparc, dtype=str), + columns=aparc_columns) tmp_lh["StructName"] = "lh_"+tmp_lh["StructName"] - tmp_rh = pd.DataFrame(np.genfromtxt(rh_aparc,dtype=str),columns=aparc_columns) + tmp_rh = pd.DataFrame(np.genfromtxt(rh_aparc, dtype=str), + columns=aparc_columns) tmp_rh["StructName"] = "rh_"+tmp_rh["StructName"] - aseg_melt = pd.melt(df[["StructName","Volume_mm3"]], id_vars=["StructName"]) - aseg_melt.rename(columns={"StructName": "name"},inplace=True) + aseg_melt = pd.melt(df[["StructName", "Volume_mm3"]], + id_vars=["StructName"]) + aseg_melt.rename(columns={"StructName": "name"}, + inplace=True) aseg_melt["value"] = aseg_melt["value"].astype(float) - + lh_aparc_melt = pd.melt(tmp_lh,id_vars=["StructName"]) lh_aparc_melt["value"] = lh_aparc_melt["value"].astype(float) - lh_aparc_melt["name"] = lh_aparc_melt["StructName"]+ "_"+lh_aparc_melt["variable"] - + lh_aparc_melt["name"] = lh_aparc_melt["StructName"] + "_" + lh_aparc_melt["variable"] + rh_aparc_melt = pd.melt(tmp_rh, id_vars=["StructName"]) rh_aparc_melt["value"] = rh_aparc_melt["value"].astype(float) - rh_aparc_melt["name"] = rh_aparc_melt["StructName"]+ "_"+rh_aparc_melt["variable"] - + rh_aparc_melt["name"] = rh_aparc_melt["StructName"] + "_" + rh_aparc_melt["variable"] + output = aseg_melt[["name", "value"]].append(lh_aparc_melt[["name", "value"]], @@ -95,35 +105,35 @@ def extract_other_vals_from_aseg(f): for pair in outdict: final_dict[pair["name"]] = pair["value"] return final_dict - + output_dict = convert_stats_to_json(aseg_file, lh_aparc, rh_aparc) return output_dict + # This function creates valid Mindcontrol entries that are saved as .json files. # This f # They can be loaded into the Mindcontrol database later - def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, startup_json_path): import os from nipype.utils.filemanip import save_json - + cortical_wm = "CerebralWhiteMatterVol" # for later FS version if not stats.get(cortical_wm): cortical_wm = "CorticalWhiteMatterVol" - + metric_split = {"brainmask": ["eTIV", "CortexVol", "TotalGrayVol"], - "wm": [cortical_wm,"WM-hypointensities", - "Right-WM-hypointensities","Left-WM-hypointensities"], - "aparcaseg":[]} - + "wm": [cortical_wm, "WM-hypointensities", + "Right-WM-hypointensities", "Left-WM-hypointensities"], + "aparcaseg": []} + volumes = {'aparcaseg': ['T1.nii.gz', 'aparc+aseg.nii.gz'], - 'brainmask': ['T1.nii.gz', 'brainmask.nii.gz'], - 'wm': ['T1.nii.gz', 'ribbon.nii.gz', 'wm.nii.gz']} + 'brainmask': ['T1.nii.gz', 'brainmask.nii.gz'], + 'ribbon': ['T1.nii.gz', 'ribbon.nii.gz', 'wm.nii.gz']} all_entries = [] - + for idx, entry_type in enumerate(["brainmask", "wm", "aparcaseg"]): - entry = {"entry_type":entry_type, - "subject_id": subject, + entry = {"entry_type": entry_type, + "subject_id": subject, "name": subject} volumes_list = [os.path.relpath(os.path.join(output_dir, subject, volume), mindcontrol_base_dir) for volume in volumes[entry_type]] @@ -134,7 +144,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, if not len(metric_split[entry_type]): entry["metrics"] = stats all_entries.append(entry) - + output_json = os.path.abspath(os.path.join(startup_json_path)) save_json(output_json, all_entries) return output_json @@ -147,15 +157,17 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, 'formatted according to the BIDS standard.') parser.add_argument('--freesurfer_dir', help='The directory with the freesurfer dirivatives,' ' should be inside the bids directory') - parser.add_argument('--mc_singularity_path', default = '.', - help="Path to the directory containing the mindcontrol" + parser.add_argument('--mc_singularity_path', default='.', + help="Path to the directory containing the mindcontrol" "singularity image and settings files.") - parser.add_argument('--no_server', action = 'store_true', + parser.add_argument('--no_server', action='store_true', help="Don't start the mindcontrol server, just generate the manifest") args = parser.parse_args() if args.bids_dir is not None: bids_dir = Path(args.bids_dir) + layout = BIDSLayout(bids_dir) + else: bids_dir = None @@ -174,12 +186,13 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, manifest_dir = mc_singularity_path/'settings/mc_manifest_init/' manifest_json = (manifest_dir/'startup.json').resolve() + # First create the initial manifest manifest = [] - if info['entry_types'] is not None: + if info['entry_types'] is not None and bids_dir is not None: entry_types = set(info['entry_types']) unused_types = set() - for img in layout.get(extensions = ".nii.gz"): + for img in layout.get(extensions=".nii.gz"): if img.type in entry_types: img_dict = {} img_dict["check_masks"] = [img.filename.replace(bids_dir.as_posix(),"")] @@ -204,49 +217,56 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, subjects.append(subject) volumes = ["brainmask.mgz", "wm.mgz", "aparc+aseg.mgz", "T1.mgz", "ribbon.mgz"] - input_node = Node(IdentityInterface(fields=['subject_id',"subjects_dir", - "mindcontrol_base_dir", "output_dir", "startup_json_path"]), name='inputnode') + input_node = Node(IdentityInterface(fields=['subject_id', + "subjects_dir", + "mindcontrol_base_dir", + "output_dir", + "startup_json_path"]), + name='inputnode') - input_node.iterables=("subject_id", subjects) + input_node.iterables = ("subject_id", subjects) input_node.inputs.subjects_dir = freesurfer_dir - input_node.inputs.mindcontrol_base_dir = bids_dir.as_posix() #this is where start_static_server is running - input_node.inputs.output_dir = freesurfer_dir.as_posix() #this is in the freesurfer/ directory under the base_dir - input_node.inputs.startup_json_path = manifest_json.as_posix() + input_node.inputs.mindcontrol_base_dir = bids_dir.as_posix() + input_node.inputs.output_dir = freesurfer_dir.as_posix() + input_node.inputs.startup_json_path = manifest_json.as_posix() - dg_node=Node(Function(input_names=["subjects_dir", "subject", "volumes"], - output_names=["volume_paths"], - function=data_grabber), + dg_node = Node(Function(input_names=["subjects_dir", "subject", "volumes"], + output_names=["volume_paths"], + function=data_grabber), name="datagrab") #dg_node.inputs.subjects_dir = subjects_dir dg_node.inputs.volumes = volumes - mriconvert_node = MapNode(MRIConvert(out_type="niigz"), - iterfield=["in_file"], + mriconvert_node = MapNode(MRIConvert(out_type="niigz"), + iterfield=["in_file"], name='convert') get_stats_node = Node(Function(input_names=["subjects_dir", "subject"], - output_names = ["output_dict"], + output_names=["output_dict"], function=parse_stats), name="get_freesurfer_stats") - write_mindcontrol_entries = Node(Function(input_names = ["mindcontrol_base_dir", - "output_dir", - "subject", - "stats", - "startup_json_path"], + write_mindcontrol_entries = Node(Function(input_names=["mindcontrol_base_dir", + "output_dir", + "subject", + "stats", + "startup_json_path"], output_names=["output_json"], function=create_mindcontrol_entries), name="get_mindcontrol_entries") datasink_node = Node(DataSink(), name='datasink') - subst = [('out_file',''),('_subject_id_',''),('_out','')] + [("_convert%d" % index, "") for index in range(len(volumes))] + subst = [('out_file', ''), + ('_subject_id_', ''), + ('_out', '')] + subst += [("_convert%d" % index, "") for index in range(len(volumes))] datasink_node.inputs.substitutions = subst workflow_working_dir = os.path.abspath("./log/scratch") wf = Workflow(name="MindPrepFS") wf.base_dir = workflow_working_dir - wf.connect(input_node,"subject_id", dg_node,"subject") - wf.connect(input_node,"subjects_dir", dg_node, "subjects_dir") + wf.connect(input_node, "subject_id", dg_node, "subject") + wf.connect(input_node, "subjects_dir", dg_node, "subjects_dir") wf.connect(input_node, "subject_id", get_stats_node, "subject") wf.connect(input_node, "subjects_dir", get_stats_node, "subjects_dir") wf.connect(input_node, "subject_id", write_mindcontrol_entries, "subject") @@ -256,7 +276,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, wf.connect(get_stats_node, "output_dict", write_mindcontrol_entries, "stats") wf.connect(input_node, "output_dir", datasink_node, "base_directory") wf.connect(dg_node,"volume_paths", mriconvert_node, "in_file") - wf.connect(mriconvert_node,'out_file',datasink_node,'out_file') + wf.connect(mriconvert_node,'out_file', datasink_node,'out_file') wf.connect(write_mindcontrol_entries, "output_json", datasink_node, "out_file.@json") #wf.write_graph(graph2use='exec') wf.run() @@ -267,4 +287,4 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, + f" -B ${{PWD}}/log/scratch/nginx/:/var/cache/nginx -B ${{PWD}}/.meteor/:/home/mindcontrol/mindcontrol/.meteor/local" \ + f" -B ${{PWD}}/settings/mc_settings:/mc_settings -H ${{PWD}}/log/scratch/singularity_home ${{PWD}}/{info['simg']}" print(cmd) - subprocess.run(cmd, cwd = mc_singularity_path, shell=True, check=True) + subprocess.run(cmd, cwd=mc_singularity_path, shell=True, check=True) From e5c06b6701a0b634df4c0e2b128d91b060759ade Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 12 Jul 2018 13:54:41 -0400 Subject: [PATCH 09/60] Fix ribbon display --- auto_singularity_mindcontrol.py | 2 +- imports/docker/entrypoint_nginx.sh | 2 +- start_singularity_mindcontrol.py | 18 ++++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/auto_singularity_mindcontrol.py b/auto_singularity_mindcontrol.py index 40f2489..c988afb 100644 --- a/auto_singularity_mindcontrol.py +++ b/auto_singularity_mindcontrol.py @@ -314,7 +314,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port "max": 2000 } }, - 'ribbon': + 'wm': { "0":{"name": "Grayscale", "alpha": 1, diff --git a/imports/docker/entrypoint_nginx.sh b/imports/docker/entrypoint_nginx.sh index 4731806..34585ff 100644 --- a/imports/docker/entrypoint_nginx.sh +++ b/imports/docker/entrypoint_nginx.sh @@ -9,5 +9,5 @@ if [ ! -x .meteor ]; then ln -s /home/mindcontrol/mindcontrol/.meteor/local ~/mindcontrol/.meteor/local/ fi cd ~/mindcontrol -nohup meteor --settings /mc_settings/mc_nginx_settings.json --port 2998 & +nohup meteor --settings /mc_settings/mc_nginx_settings.json --port 2998 > ~/mindcontrol.out 2>&1 & nginx -g "daemon off;" diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 958f66f..2dc3e7d 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -127,7 +127,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, volumes = {'aparcaseg': ['T1.nii.gz', 'aparc+aseg.nii.gz'], 'brainmask': ['T1.nii.gz', 'brainmask.nii.gz'], - 'ribbon': ['T1.nii.gz', 'ribbon.nii.gz', 'wm.nii.gz']} + 'wm': ['T1.nii.gz', 'ribbon.nii.gz', 'wm.nii.gz']} all_entries = [] @@ -166,7 +166,10 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, args = parser.parse_args() if args.bids_dir is not None: bids_dir = Path(args.bids_dir) - layout = BIDSLayout(bids_dir) + try: + layout = BIDSLayout(bids_dir) + except ValueError as e: + print("Invalid bids directory, skipping none freesurfer files. BIDS error:", e) else: bids_dir = None @@ -186,7 +189,6 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, manifest_dir = mc_singularity_path/'settings/mc_manifest_init/' manifest_json = (manifest_dir/'startup.json').resolve() - # First create the initial manifest manifest = [] if info['entry_types'] is not None and bids_dir is not None: @@ -195,7 +197,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, for img in layout.get(extensions=".nii.gz"): if img.type in entry_types: img_dict = {} - img_dict["check_masks"] = [img.filename.replace(bids_dir.as_posix(),"")] + img_dict["check_masks"] = [img.filename.replace(bids_dir.as_posix(), "")] img_dict["entry_type"] = img.type img_dict["metrics"] = {} img_dict["name"] = os.path.split(img.filename)[1].split('.')[0] @@ -227,13 +229,13 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, input_node.iterables = ("subject_id", subjects) input_node.inputs.subjects_dir = freesurfer_dir input_node.inputs.mindcontrol_base_dir = bids_dir.as_posix() - input_node.inputs.output_dir = freesurfer_dir.as_posix() + input_node.inputs.output_dir = freesurfer_dir.as_posix() input_node.inputs.startup_json_path = manifest_json.as_posix() dg_node = Node(Function(input_names=["subjects_dir", "subject", "volumes"], output_names=["volume_paths"], function=data_grabber), - name="datagrab") + name="datagrab") #dg_node.inputs.subjects_dir = subjects_dir dg_node.inputs.volumes = volumes @@ -251,7 +253,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, "stats", "startup_json_path"], output_names=["output_json"], - function=create_mindcontrol_entries), + function=create_mindcontrol_entries), name="get_mindcontrol_entries") datasink_node = Node(DataSink(), @@ -276,7 +278,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, wf.connect(get_stats_node, "output_dict", write_mindcontrol_entries, "stats") wf.connect(input_node, "output_dir", datasink_node, "base_directory") wf.connect(dg_node,"volume_paths", mriconvert_node, "in_file") - wf.connect(mriconvert_node,'out_file', datasink_node,'out_file') + wf.connect(mriconvert_node,'out_file', datasink_node, 'out_file') wf.connect(write_mindcontrol_entries, "output_json", datasink_node, "out_file.@json") #wf.write_graph(graph2use='exec') wf.run() From 119afc09ab5eba79501fc0c5aa434c5e3bf95947 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 12 Jul 2018 14:01:38 -0400 Subject: [PATCH 10/60] Add freesurfer histograms --- auto_singularity_mindcontrol.py | 160 ++++++++++++++++---------------- 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/auto_singularity_mindcontrol.py b/auto_singularity_mindcontrol.py index c988afb..024e874 100644 --- a/auto_singularity_mindcontrol.py +++ b/auto_singularity_mindcontrol.py @@ -213,81 +213,82 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port """ file_server = f"http://localhost:{nginx_port}/files/" startup_file_server = f'http://localhost:{startup_port}/' - + default_module = { - "fields": [ - { - "function_name": "get_qc_viewer", - "id": "name", - "name": "Image File" - }, - { - "function_name": "get_qc_ave_field", - "id": "average_vote", - "name": "QC vote" - }, - { - "function_name": None, - "id": "num_votes", - "name": "# votes" - }, - { - "function_name": None, - "id": "quality_check.notes_QC", - "name": "Notes" - } - ], - "metric_names": None, - "graph_type": None, - "staticURL": file_server, - "usePeerJS": False, - "logPainter": False, - "logContours": False, - "logPoints": True, - "qc_options": {"pass": 1, "fail": 1, "needs_edits": 0, "edited": 0, "assignTo": 0, "notes": 1, "confidence": 1}} - + "fields": [ + { + "function_name": "get_qc_viewer", + "id": "name", + "name": "Image File" + }, + { + "function_name": "get_qc_ave_field", + "id": "average_vote", + "name": "QC vote" + }, + { + "function_name": None, + "id": "num_votes", + "name": "# votes" + }, + { + "function_name": None, + "id": "quality_check.notes_QC", + "name": "Notes" + } + ], + "metric_names": None, + "graph_type": None, + "staticURL": file_server, + "usePeerJS": False, + "logPainter": False, + "logContours": False, + "logPoints": True, + "qc_options": {"pass": 1, "fail": 1, "needs_edits": 0, "edited": 0, "assignTo": 0, "notes": 1, "confidence": 1} + } + fs_module = { - "fields": [ - { - "function_name": "get_filter_field", - "id": "subject", - "name": "Exam ID" - }, - { - "function_name": "get_qc_viewer", - "id": "name", - "name": "Freesurfer ID" - }, - { - "function_name": "get_qc_filter_field", - "id": "quality_check.QC", - "name": "QC" - }, - { - "function_name": "get_filter_field", - "id": "checkedBy", - "name": "checked by" - }, - { - "function_name": "get_filter_field", - "id": "quality_check.user_assign", - "name": "Assigned To" - }, - { - "function_name": None, - "id": "quality_check.notes_QC", - "name": "Notes" - } - ], - "metric_names": None, - "graph_type": None, - "staticURL": file_server, - "usePeerJS": False, - "logPainter": False, - "logContours": False, - "logPoints": True - } - + "fields": [ + { + "function_name": "get_filter_field", + "id": "subject", + "name": "Exam ID" + }, + { + "function_name": "get_qc_viewer", + "id": "name", + "name": "Freesurfer ID" + }, + { + "function_name": "get_qc_filter_field", + "id": "quality_check.QC", + "name": "QC" + }, + { + "function_name": "get_filter_field", + "id": "checkedBy", + "name": "checked by" + }, + { + "function_name": "get_filter_field", + "id": "quality_check.user_assign", + "name": "Assigned To" + }, + { + "function_name": None, + "id": "quality_check.notes_QC", + "name": "Notes" + } + ], + "metric_names": None, + "graph_type": "histogram", + "staticURL": file_server, + "usePeerJS": False, + "logPainter": False, + "logContours": False, + "logPoints": True + } + fs_cm_dict = {'aparcaseg': { "0":{"name": "Grayscale", @@ -341,7 +342,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port } if entry_types is None and not freesurfer: raise Exception("You must either define entry types or have freesurfer == True") - + modules = [] if entry_types is not None: for et in entry_types: @@ -349,9 +350,9 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port et_module["name"] = et et_module["entry_type"] = et modules.append(et_module) - + if freesurfer: - for et,cm in fs_cm_dict.items(): + for et, cm in fs_cm_dict.items(): et_module = fs_module.copy() et_module["name"] = fs_name_dict[et] et_module["entry_type"] = et @@ -359,7 +360,6 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port et_module['colormaps'] = cm modules.append(et_module) - # autogenerated settings files pub_set = {"startup_json": startup_file_server+"startup.json", "load_if_empty": True, @@ -368,7 +368,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port "modules": modules} settings = {"public": pub_set} with mcsetfile.open("w") as h: - json.dump(settings,h) + json.dump(settings, h) if __name__ == "__main__": docker_build_path = Path(__file__).resolve().parent / 'imports/docker' @@ -380,7 +380,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port help='Directory to bulid singularirty image and files in.') parser.add_argument('--custom_settings', help='Path to custom settings json') - parser.add_argument('--freesurfer', action='store_true', + parser.add_argument('--freesurfer', action='store_true', help='Generate settings for freesurfer QC in mindcontrol.') parser.add_argument('--entry_type', action='append', help='Name of mindcontrol module you would like to have autogenerated.' @@ -455,7 +455,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port # Write settings files write_passfile(passfile) write_nginxconf(ncfile) - write_meteorconf(mcfile, startup_port=startup_port, + write_meteorconf(mcfile, startup_port=startup_port, nginx_port=nginx_port, meteor_port=meteor_port) if custom_settings is not None: copyfile(custom_settings, mcsetfile.as_posix()) From 14af080bba224570249bc1bc523ced1fbad361c3 Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Thu, 12 Jul 2018 14:20:07 -0400 Subject: [PATCH 11/60] Update README.md --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index ab83f6c..d0f912d 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,34 @@ Create a database json file similar to [http://dxugxjm290185.cloudfront.net/hbn/ * Host your database json file on a server and copy/paste its url into the "startup_json" value on `settings.dev.json` * Define each module in `settings.dev.json` to point to your `entry_type`, and define the module's `staticURL` +## Bulid and deploy with singularity +First you'll build a singularity container on a system where you've got docker installed. + +Clone this repository + +``` +git clone https://github.com/akeshavan/mindcontrol +``` + +Run auto_singularity_mindcontrol.py. You can see its documentation with `python auto_singularity_mindcontrol.py -h`. +``` +python3 mindcontrol/auto_singularity_mindcontrol.py --sing_out_dir ~/mc_sing --freesurfer +``` + +Copy the directory produced to the system you want to run mindcontrol on. For example, a node on the cluster where your data is already located. + +``` +rsync -ach ~/mc_sing server:/data/project/ +``` + +Connect to that server with a few ports forwarded and run start_singularity_mindcontrol from that directory. You can see its documentation with `python start_singularity_mindcontrol.py -h`. You'll need freesurfer, singularity, python3 and nipype on that system. + +``` +ssh -L 3000:localhost:3000 -L 3003:localhost:3003 server +cd /data/project/mc_sing +start_singularity_mindcontrol.py --bids_dir /data/project/bids_directory --freesurfer_dir /data/project/bids_directory/derivatives/freesurfer +``` + ## Demo Check out the [demo](http://mindcontrol.herokuapp.com/). [This data is from the 1000 Functional Connectomes Project](http://fcon_1000.projects.nitrc.org/fcpClassic/FcpTable.html) From 7c92a37f07e5d203304ee238daa1bb48d1b96b65 Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Thu, 12 Jul 2018 14:24:58 -0400 Subject: [PATCH 12/60] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d0f912d..ac755de 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ First you'll build a singularity container on a system where you've got docker i Clone this repository ``` -git clone https://github.com/akeshavan/mindcontrol +git clone https://github.com/Shotgunosine/mindcontrol +git checkout hpc_compat ``` Run auto_singularity_mindcontrol.py. You can see its documentation with `python auto_singularity_mindcontrol.py -h`. From d170e5b2459985d7d48741d2d73871e01146b79d Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Thu, 12 Jul 2018 14:44:44 -0400 Subject: [PATCH 13/60] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ac755de..8587d61 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,13 @@ Clone this repository ``` git clone https://github.com/Shotgunosine/mindcontrol +cd mindcontrol git checkout hpc_compat ``` Run auto_singularity_mindcontrol.py. You can see its documentation with `python auto_singularity_mindcontrol.py -h`. ``` -python3 mindcontrol/auto_singularity_mindcontrol.py --sing_out_dir ~/mc_sing --freesurfer +python3 auto_singularity_mindcontrol.py --sing_out_dir ~/mc_sing --freesurfer ``` Copy the directory produced to the system you want to run mindcontrol on. For example, a node on the cluster where your data is already located. From 312742accee620a87c7a3b7a97cb4d327647c35c Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 12 Jul 2018 14:45:41 -0400 Subject: [PATCH 14/60] fix: call it wm not ribbon --- auto_singularity_mindcontrol.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/auto_singularity_mindcontrol.py b/auto_singularity_mindcontrol.py index 024e874..d0fa1fa 100644 --- a/auto_singularity_mindcontrol.py +++ b/auto_singularity_mindcontrol.py @@ -5,7 +5,6 @@ from shutil import copyfile import os import getpass -from passlib.hash import bcrypt import subprocess import random import sys @@ -79,7 +78,7 @@ def delete(self, username): def write_passfile(passfile_path): """Collects usernames and passwords and writes them to - the provided path with bcrypt encryption. + the provided path with encryption. Parameters ---------- passfile: pathlib.Path object @@ -338,7 +337,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port } fs_name_dict = {'brainmask': 'Brain Mask', 'aparcaseg': 'Segmentation', - 'ribbon': 'White Matter', + 'wm': 'White Matter', } if entry_types is None and not freesurfer: raise Exception("You must either define entry types or have freesurfer == True") From db7f9fe1a3f663f04ed4a40bc7d2122504859553 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 12 Jul 2018 15:25:24 -0400 Subject: [PATCH 15/60] add nipype plugin option --- start_singularity_mindcontrol.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 2dc3e7d..6a93eb4 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -162,6 +162,9 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, "singularity image and settings files.") parser.add_argument('--no_server', action='store_true', help="Don't start the mindcontrol server, just generate the manifest") + parser.add_argument('--nipype_plugin', + help="Run the mgz to nii.gz conversion with the specified nipype plugin." + "see https://nipype.readthedocs.io/en/latest/users/plugins.html") args = parser.parse_args() if args.bids_dir is not None: @@ -182,6 +185,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, mc_singularity_path = Path(args.mc_singularity_path) no_server = args.no_server + nipype_plugin = args.nipype_plugin infofile = mc_singularity_path/'settings/mc_info.json' with infofile.open('r') as h: @@ -281,7 +285,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, wf.connect(mriconvert_node,'out_file', datasink_node, 'out_file') wf.connect(write_mindcontrol_entries, "output_json", datasink_node, "out_file.@json") #wf.write_graph(graph2use='exec') - wf.run() + wf.run(plugin=nipype_plugin) if not args.no_server: cmd = f"singularity run -B ${{PWD}}/log:/var/log/nginx -B {bids_dir}:/mc_data" \ From 6e033c5383f3bd58c35287a146d2cf844035367b Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 12 Jul 2018 16:16:32 -0400 Subject: [PATCH 16/60] add nipype plugin args option --- start_singularity_mindcontrol.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 6a93eb4..1bebad1 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -42,6 +42,7 @@ def extract_other_vals_from_aseg(f): "SubCortGray", "Cortex", "CerebralWhiteMatter", + "CorticalWhiteMatter", "CorticalWhiteMatterVol"] value_labels = list(map(lambda x: 'Measure ' + x + ',', value_labels)) output = pd.DataFrame() @@ -119,6 +120,8 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, cortical_wm = "CerebralWhiteMatterVol" # for later FS version if not stats.get(cortical_wm): cortical_wm = "CorticalWhiteMatterVol" + if not stats.get(cortical_wm): + cortical_wm = "CorticalWhiteMatter" metric_split = {"brainmask": ["eTIV", "CortexVol", "TotalGrayVol"], "wm": [cortical_wm, "WM-hypointensities", @@ -165,6 +168,8 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, parser.add_argument('--nipype_plugin', help="Run the mgz to nii.gz conversion with the specified nipype plugin." "see https://nipype.readthedocs.io/en/latest/users/plugins.html") + parser.add_argument('--nipype_plugin_args', + help='json formatted string of keyword arguments for nipype_plugin') args = parser.parse_args() if args.bids_dir is not None: @@ -186,6 +191,10 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, no_server = args.no_server nipype_plugin = args.nipype_plugin + if args.nipype_plugin_args is not None: + nipype_plugin_args = json.loads(args.nipype_plugin_args) + else: + nipype_plugin_args = {} infofile = mc_singularity_path/'settings/mc_info.json' with infofile.open('r') as h: @@ -219,7 +228,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, for path in freesurfer_dir.glob('*'): subject = path.parts[-1] # check if mri dir exists, and don't add fsaverage - if os.path.exists(os.path.join(path, 'mri')) and subject != 'fsaverage': + if os.path.exists(os.path.join(path, 'mri')) and 'average' not in subject: subjects.append(subject) volumes = ["brainmask.mgz", "wm.mgz", "aparc+aseg.mgz", "T1.mgz", "ribbon.mgz"] @@ -285,7 +294,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, wf.connect(mriconvert_node,'out_file', datasink_node, 'out_file') wf.connect(write_mindcontrol_entries, "output_json", datasink_node, "out_file.@json") #wf.write_graph(graph2use='exec') - wf.run(plugin=nipype_plugin) + wf.run(plugin=nipype_plugin, plugin_args=nipype_plugin_args) if not args.no_server: cmd = f"singularity run -B ${{PWD}}/log:/var/log/nginx -B {bids_dir}:/mc_data" \ From bac9079703584096abef7947c932b1f1cf1f2786 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Tue, 17 Jul 2018 17:23:07 -0400 Subject: [PATCH 17/60] use singularity pull and instance.start --- imports/docker/Dockerfile_nginx | 6 +- imports/docker/Singularity | 26 ++ start_singularity_mindcontrol.py | 550 +++++++++++++++++++++++++++++-- 3 files changed, 548 insertions(+), 34 deletions(-) create mode 100644 imports/docker/Singularity diff --git a/imports/docker/Dockerfile_nginx b/imports/docker/Dockerfile_nginx index 5311dbf..a22772c 100644 --- a/imports/docker/Dockerfile_nginx +++ b/imports/docker/Dockerfile_nginx @@ -70,7 +70,7 @@ RUN cd ${MC_DIR}/mindcontrol &&\ git clone https://github.com/akeshavan/mindcontrol.git ${MC_DIR}/mindcontrol &&\ meteor update &&\ meteor npm install --save @babel/runtime &&\ - meteor npm install --save bcrypt + meteor npm install --save bcrypt &&\ #git clone https://github.com/clowdcontrol/mindcontrol.git ${MC_DIR}/mindcontrol @@ -82,8 +82,8 @@ COPY ./settings/auth.htpasswd /etc/nginx COPY ./settings/nginx.conf /etc/nginx/nginx.conf USER root RUN chmod 777 /var/log/nginx /var/cache/nginx /opt /etc &&\ - mkdir -p /mc_data /mc_startup_data /mc_settings &&\ - chown mindcontrol:mindcontrol /etc/nginx/auth.htpasswd /mc_data /mc_startup_data /mc_settings + mkdir -p /mc_data /mc_startup_data /mc_settings /mc_fs &&\ + chown mindcontrol:mindcontrol /etc/nginx/auth.htpasswd /mc_data /mc_startup_data /mc_settings /mc_fs USER mindcontrol ###### diff --git a/imports/docker/Singularity b/imports/docker/Singularity new file mode 100644 index 0000000..303ad1a --- /dev/null +++ b/imports/docker/Singularity @@ -0,0 +1,26 @@ +Bootstrap: docker +Registry: http://localhost:5000 +Namespace: +From: mc_services:latest + +%labels + Mainteiner Dylan Nielson\ + +%startscript + export HOME=/home/`whoami` + if [ ! -d $HOME/mindcontrol ] || [ ! -d $HOME/.meteor ] || [ ! -d $HOME/.cordova ] ; then + echo "Copying meteor files into singularity_home" > /output/out + rsync -ach /home/mindcontrol/mindcontrol $HOME > /output/rsync 2>&1 + echo "/home/mindcontrol/mindcontrol copied to $HOME" >> /output/out + rsync -ach /home/mindcontrol/.meteor $HOME >> /output/rsync 2>&1 + echo "/home/mindcontrol/.meteor copied to $HOME" >> /output/out + rsync -ach /home/mindcontrol/.cordova $HOME >> /output/rsync 2>&1 + echo "home/mindcontrol/.cordova copied to $HOME" >> /output/out + fi + ln -s /home/mindcontrol/mindcontrol/.meteor/local $HOME/mindcontrol/.meteor/local/ + echo "Starting mindcontrol and nginx" >> /output/out + cd $HOME/mindcontrol + # grab proper meteor port from settings file + MC_PORT=$(cat /mc_settings/mc_port) + nohup meteor --settings /mc_settings/mc_nginx_settings.json --port $MC_PORT > /output/mindcontrol.out 2>&1 & + nginx -g "daemon off;" \ No newline at end of file diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 1bebad1..e2d70db 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -5,14 +5,380 @@ from bids.grabbids import BIDSLayout import subprocess import os -import shutil +from shutil import copyfile +import getpass +import random +import sys + from nipype import MapNode, Workflow, Node from nipype.interfaces.freesurfer import MRIConvert from nipype.interfaces.io import DataSink from nipype.interfaces.utility import IdentityInterface, Function -#this function finds data in the subjects_dir +# HT password code from https://gist.github.com/eculver/1420227 + +# We need a crypt module, but Windows doesn't have one by default. Try to find +# one, and tell the user if we can't. +try: + import crypt +except ImportError: + try: + import fcrypt as crypt + except ImportError: + sys.stderr.write("Cannot find a crypt module. " + "Possibly http://carey.geek.nz/code/python-fcrypt/\n") + sys.exit(1) + + +def salt(): + """Returns a string of 2 randome letters""" + letters = 'abcdefghijklmnopqrstuvwxyz' \ + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' \ + '0123456789/.' + return random.choice(letters) + random.choice(letters) + + +class HtpasswdFile: + """A class for manipulating htpasswd files.""" + + def __init__(self, filename, create=False): + self.entries = [] + self.filename = filename + if not create: + if os.path.exists(self.filename): + self.load() + else: + raise Exception("%s does not exist" % self.filename) + + def load(self): + """Read the htpasswd file into memory.""" + lines = open(self.filename, 'r').readlines() + self.entries = [] + for line in lines: + username, pwhash = line.split(':') + entry = [username, pwhash.rstrip()] + self.entries.append(entry) + + def save(self): + """Write the htpasswd file to disk""" + open(self.filename, 'w').writelines(["%s:%s\n" % (entry[0], entry[1]) + for entry in self.entries]) + + def update(self, username, password): + """Replace the entry for the given user, or add it if new.""" + pwhash = crypt.crypt(password, salt()) + matching_entries = [entry for entry in self.entries + if entry[0] == username] + if matching_entries: + matching_entries[0][1] = pwhash + else: + self.entries.append([username, pwhash]) + + def delete(self, username): + """Remove the entry for the given user.""" + self.entries = [entry for entry in self.entries + if entry[0] != username] + + +def write_passfile(passfile_path): + """Collects usernames and passwords and writes them to + the provided path with encryption. + Parameters + ---------- + passfile: pathlib.Path object + The path to which to write the usernames and hashed passwords. + """ + users = set() + done = False + passfile = HtpasswdFile(passfile_path.as_posix(), create=True) + while not done: + user = "" + print("Please enter usernames and passwords for all the users you'd like to create.", flush=True) + user = input("Input user, leave blank if you are finished entering users:") + if len(users) > 0 and user == "": + print("All users entered, generating auth.htpass file", flush=True) + done= True + passfile.save() + elif len(users) == 0 and user == "": + print("Please enter at least one user", flush=True) + else: + if user in users: + print("Duplicate user, overwriting previously entered password for %s."%user, flush=True) + hs = None + while hs is None: + a = getpass.getpass(prompt="Enter Password for user: %s\n"%user) + b = getpass.getpass(prompt="Re-enter Password for user: %s\n"%user) + if a == b: + hs = "valid_pass" + passfile.update(user, a) + else: + print("Entered passwords don't match, please try again.", flush=True) + users.add(user) + + +def write_meteorconf(mcfile, startup_port=3003, nginx_port=3000, meteor_port=2998): + """Write nginx configuration file for meteor given user specified ports. + Parameters + ---------- + mcfile: pathlib.Path object + The path to which to write the config file. + startup_port: int, default is 3003 + Port number at which mindcontrol will look for startup manifest. + nginx_port: int, default is 3000 + Port number at nginx will run. This is the port you connect to reach mindcontrol. + meteor_port: int, default is 2998 + Port number at meteor will run. This is mostly under the hood, but you might + need to change it if there is a port conflict. Mongo will run on the port + one above the meteor_port. + """ + mc_string = f"""error_log /var/log/nginx/nginx_error.log info; + +server {{ + listen {startup_port} default_server; + root /mc_startup_data; + location / {{ + autoindex on; + }} + }} + +server {{ + listen {nginx_port} default_server; + auth_basic "Restricted"; + auth_basic_user_file auth.htpasswd; + + location / {{ + proxy_pass http://localhost:{meteor_port}/; + }} + location /files/ {{ + alias /mc_data/; + }} + location /fs/ {{ + alias /mc_fs/; + }} + }}""" + mcfile.write_text(mc_string) + + +def write_nginxconf(ncfile): + """Write top level nginx configuration file. + Parameters + ---------- + ncfile: pathlib.Path object + The path to which to write the config file. + """ + nc_string = f"""worker_processes 1; +pid /var/cache/nginx/nginx.pid; +error_log /var/log/nginx/error.log warn; + + +events {{ + worker_connections 1024; +}} + + +http {{ + disable_symlinks off; + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/meteor.conf; +}} +""" + ncfile.write_text(nc_string) + + +def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port=3003, nginx_port=3000): + """ Write the mindcontrol settings json. This determines which panels mindcontrol + displays and which that information comes from. + Parameters + ---------- + mcfile: pathlib.Path object + The path to which to write the json file. + entry_types: optional, list of strings + List of names of modules you would like mindcontrol to display + freesurfer: optional, bool + True if you would like the settings generated modules for qcing aparc-aseg, wm, and ribbon + startup_port: int, default is 3003 + Port number at which mindcontrol will look for startup manifest. + nginx_port: int, default is 3000 + Port number at nginx will run. This is the port you connect to reach mindcontrol. + """ + file_server = f"http://localhost:{nginx_port}/files/" + fs_server = f"http://localhost:{nginx_port}/fs/" + startup_file_server = f'http://localhost:{startup_port}/' + + default_module = { + "fields": [ + { + "function_name": "get_qc_viewer", + "id": "name", + "name": "Image File" + }, + { + "function_name": "get_qc_ave_field", + "id": "average_vote", + "name": "QC vote" + }, + { + "function_name": None, + "id": "num_votes", + "name": "# votes" + }, + { + "function_name": None, + "id": "quality_check.notes_QC", + "name": "Notes" + } + ], + "metric_names": None, + "graph_type": None, + "staticURL": file_server, + "usePeerJS": False, + "logPainter": False, + "logContours": False, + "logPoints": True, + "qc_options": {"pass": 1, "fail": 1, "needs_edits": 0, "edited": 0, "assignTo": 0, "notes": 1, "confidence": 1} + } + + fs_module = { + "fields": [ + { + "function_name": "get_filter_field", + "id": "subject", + "name": "Exam ID" + }, + { + "function_name": "get_qc_viewer", + "id": "name", + "name": "Freesurfer ID" + }, + { + "function_name": "get_qc_filter_field", + "id": "quality_check.QC", + "name": "QC" + }, + { + "function_name": "get_filter_field", + "id": "checkedBy", + "name": "checked by" + }, + { + "function_name": "get_filter_field", + "id": "quality_check.user_assign", + "name": "Assigned To" + }, + { + "function_name": None, + "id": "quality_check.notes_QC", + "name": "Notes" + } + ], + "metric_names": None, + "graph_type": "histogram", + "staticURL": fs_server, + "usePeerJS": False, + "logPainter": False, + "logContours": False, + "logPoints": True + } + + fs_cm_dict = {'aparcaseg': + { + "0":{"name": "Grayscale", + "alpha": 1, + "min": 0, + "max": 255 + }, + "1": { + "name": "custom.Freesurfer", + "alpha": 0.5 + } + }, + 'brainmask': + { + "0":{"name": "Grayscale", + "alpha": 1, + "min": 0, + "max": 255 + }, + "1": { + "name": "Red Overlay", + "alpha": 0.2, + "min": 0, + "max": 2000 + } + }, + 'wm': + { + "0":{"name": "Grayscale", + "alpha": 1, + "min": 0, + "max": 255 + }, + "1": { + "name": "Green Overlay", + "alpha": 0.5, + "min": 0, + "max": 2000 + }, + "2": { + "name": "Blue Overlay", + "alpha": 0.5, + "min":0, + "max": 2000 + } + } + } + fs_name_dict = {'brainmask': 'Brain Mask', + 'aparcaseg': 'Segmentation', + 'wm': 'White Matter', + } + if entry_types is None and not freesurfer: + raise Exception("You must either define entry types or have freesurfer == True") + + modules = [] + if entry_types is not None: + for et in entry_types: + et_module = default_module.copy() + et_module["name"] = et + et_module["entry_type"] = et + modules.append(et_module) + + if freesurfer: + for et, cm in fs_cm_dict.items(): + et_module = fs_module.copy() + et_module["name"] = fs_name_dict[et] + et_module["entry_type"] = et + et_module["num_overlays"] = len(cm) + et_module['colormaps'] = cm + modules.append(et_module) + + # autogenerated settings files + pub_set = {"startup_json": startup_file_server+"startup.json", + "load_if_empty": True, + "use_custom": True, + "needs_consent": False, + "modules": modules} + settings = {"public": pub_set} + with mcsetfile.open("w") as h: + json.dump(settings, h) + + +#this function finds data in the subjects_dir def data_grabber(subjects_dir, subject, volumes): import os volumes_list = [os.path.join(subjects_dir, subject, 'mri', volume) for volume in volumes] @@ -87,7 +453,7 @@ def extract_other_vals_from_aseg(f): inplace=True) aseg_melt["value"] = aseg_melt["value"].astype(float) - lh_aparc_melt = pd.melt(tmp_lh,id_vars=["StructName"]) + lh_aparc_melt = pd.melt(tmp_lh, id_vars=["StructName"]) lh_aparc_melt["value"] = lh_aparc_melt["value"].astype(float) lh_aparc_melt["name"] = lh_aparc_melt["StructName"] + "_" + lh_aparc_melt["variable"] @@ -113,7 +479,7 @@ def extract_other_vals_from_aseg(f): # This function creates valid Mindcontrol entries that are saved as .json files. # This f # They can be loaded into the Mindcontrol database later -def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, startup_json_path): +def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats): import os from nipype.utils.filemanip import save_json @@ -138,7 +504,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, entry = {"entry_type": entry_type, "subject_id": subject, "name": subject} - volumes_list = [os.path.relpath(os.path.join(output_dir, subject, volume), mindcontrol_base_dir) + volumes_list = [os.path.join(subject, 'mri', volume) for volume in volumes[entry_type]] entry["check_masks"] = volumes_list entry["metrics"] = {} @@ -148,7 +514,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, entry["metrics"] = stats all_entries.append(entry) - output_json = os.path.abspath(os.path.join(startup_json_path)) + output_json = os.path.abspath("mindcontrol_entries.json") save_json(output_json, all_entries) return output_json @@ -156,15 +522,38 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, parser = argparse.ArgumentParser(description='Start mindcontrol in a previously built' ' singularity container and create initial manifest' ' if needed.') + parser.add_argument('--sing_out_dir', + default='.', + help='Directory to bulid singularirty image and files in.') + parser.add_argument('--custom_settings', + help='Path to custom settings json') + parser.add_argument('--freesurfer', action='store_true', + help='Generate settings for freesurfer QC in mindcontrol.') + parser.add_argument('--entry_type', action='append', + help='Name of mindcontrol module you would like to have autogenerated.' + ' This should correspond to the bids image type ' + '(specified after the final _ of the image name). ' + ' Pass this argument multiple times to add additional modules.') + parser.add_argument('--startup_port', + default=3003, + help='Port number at which mindcontrol will look for startup manifest.') + parser.add_argument('--nginx_port', + default=3000, + help='Port number at nginx will run. This is the port you connect to reach mindcontrol.') + parser.add_argument('--meteor_port', + default=2998, + help='Port number at meteor will run. ' + 'This is mostly under the hood, ' + 'but you might need to change it if there is a port conflict.' + 'Mongo will run on the port one above this one.') parser.add_argument('--bids_dir', help='The directory with the input dataset ' 'formatted according to the BIDS standard.') parser.add_argument('--freesurfer_dir', help='The directory with the freesurfer dirivatives,' ' should be inside the bids directory') - parser.add_argument('--mc_singularity_path', default='.', - help="Path to the directory containing the mindcontrol" - "singularity image and settings files.") + parser.add_argument('--no_mriconvert', action='store_true', + help="Don't convert mgzs to nifti, just assume the images are present.") parser.add_argument('--no_server', action='store_true', - help="Don't start the mindcontrol server, just generate the manifest") + help="Don't start the mindcontrol server, just generate the manifest.") parser.add_argument('--nipype_plugin', help="Run the mgz to nii.gz conversion with the specified nipype plugin." "see https://nipype.readthedocs.io/en/latest/users/plugins.html") @@ -172,6 +561,20 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, help='json formatted string of keyword arguments for nipype_plugin') args = parser.parse_args() + sing_out_dir = args.sing_out_dir + if args.custom_settings is not None: + custom_settings = Path(args.custom_settings) + else: + custom_settings = None + freesurfer = args.freesurfer + if args.entry_type is not None: + entry_types = set(args.entry_type) + else: + entry_types = set([]) + startup_port = args.startup_port + nginx_port = args.nginx_port + meteor_port = args.meteor_port + if args.bids_dir is not None: bids_dir = Path(args.bids_dir) try: @@ -187,25 +590,84 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, else: freesurfer_dir = None - mc_singularity_path = Path(args.mc_singularity_path) - no_server = args.no_server + no_mriconvert = args.no_mriconvert nipype_plugin = args.nipype_plugin if args.nipype_plugin_args is not None: nipype_plugin_args = json.loads(args.nipype_plugin_args) else: nipype_plugin_args = {} - infofile = mc_singularity_path/'settings/mc_info.json' - with infofile.open('r') as h: - info = json.load(h) - manifest_dir = mc_singularity_path/'settings/mc_manifest_init/' + # Set up directory to be copied + basedir = Path(sing_out_dir).resolve() + setdir = basedir/"settings" + mcsetdir = setdir/"mc_settings" + manifest_dir = setdir/"mc_manifest_init" + meteor_ldir = basedir/".meteor" + simg_path = basedir/"mc_service.simg" + + logdir = basedir/"log" + simg_out = logdir/"simg_out" + scratch_dir = basedir/"scratch" + nginx_scratch = scratch_dir/"nginx" + mc_hdir = scratch_dir/"singularity_home" + + if not basedir.exists(): + basedir.mkdir() + if not setdir.exists(): + setdir.mkdir() + if not logdir.exists(): + logdir.mkdir() + if not simg_out.exists(): + simg_out.mkdir() + if not scratch_dir.exists(): + scratch_dir.mkdir() + if not nginx_scratch.exists(): + nginx_scratch.mkdir() + if not mcsetdir.exists(): + mcsetdir.mkdir() + if not manifest_dir.exists(): + manifest_dir.mkdir() + if not mc_hdir.exists(): + mc_hdir.mkdir() + if not meteor_ldir.exists(): + meteor_ldir.mkdir() + dockerfile = basedir/"Dockerfile_nginx" + entrypoint = basedir/"entrypoint_nginx.sh" + passfile = setdir/"auth.htpasswd" + mcfile = setdir/"meteor.conf" + ncfile = setdir/"nginx.conf" + mcsetfile = mcsetdir/"mc_nginx_settings.json" + mcportfile = mcsetdir/"mc_port" + infofile = setdir/"mc_info.json" + + + # Write settings files + write_passfile(passfile) + write_nginxconf(ncfile) + write_meteorconf(mcfile, startup_port=startup_port, + nginx_port=nginx_port, meteor_port=meteor_port) + # write the meteor port to a file so we can load it in the Singularity start script + mcportfile.write_text(str(meteor_port)) + if custom_settings is not None: + copyfile(custom_settings, mcsetfile.as_posix()) + else: + write_mcsettings(mcsetfile, entry_types=entry_types, freesurfer=freesurfer, + startup_port=startup_port, nginx_port=nginx_port) + # Copy singularity run script to directory + srun_source = Path(__file__) + srun_dest = basedir / 'start_singularity_mindcontrol.py' + copyfile(srun_source.as_posix(), srun_dest.as_posix()) + + # infofile = mc_singularity_path/'settings/mc_info.json' + # with infofile.open('r') as h: + # info = json.load(h) + manifest_dir = basedir/'settings/mc_manifest_init/' manifest_json = (manifest_dir/'startup.json').resolve() # First create the initial manifest manifest = [] - if info['entry_types'] is not None and bids_dir is not None: - entry_types = set(info['entry_types']) + if len(entry_types) != 0 and bids_dir is not None: unused_types = set() for img in layout.get(extensions=".nii.gz"): if img.type in entry_types: @@ -219,9 +681,9 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, manifest.append(img_dict) else: unused_types.add(img.type) - if info['freesurfer']: + if freesurfer: if freesurfer_dir is None: - #TODO: look in default location for freesurfer directory + # TODO: look in default location for freesurfer directory raise Exception("Must specify the path to freesurfer files.") subjects = [] @@ -243,7 +705,6 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, input_node.inputs.subjects_dir = freesurfer_dir input_node.inputs.mindcontrol_base_dir = bids_dir.as_posix() input_node.inputs.output_dir = freesurfer_dir.as_posix() - input_node.inputs.startup_json_path = manifest_json.as_posix() dg_node = Node(Function(input_names=["subjects_dir", "subject", "volumes"], output_names=["volume_paths"], @@ -274,9 +735,9 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, subst = [('out_file', ''), ('_subject_id_', ''), ('_out', '')] - subst += [("_convert%d" % index, "") for index in range(len(volumes))] + subst += [("_convert%d" % index, "mri") for index in range(len(volumes))] datasink_node.inputs.substitutions = subst - workflow_working_dir = os.path.abspath("./log/scratch") + workflow_working_dir = scratch_dir.absolute() wf = Workflow(name="MindPrepFS") wf.base_dir = workflow_working_dir @@ -287,19 +748,46 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats, wf.connect(input_node, "subject_id", write_mindcontrol_entries, "subject") wf.connect(input_node, "mindcontrol_base_dir", write_mindcontrol_entries, "mindcontrol_base_dir") wf.connect(input_node, "output_dir", write_mindcontrol_entries, "output_dir") - wf.connect(input_node, "startup_json_path", write_mindcontrol_entries, "startup_json_path") wf.connect(get_stats_node, "output_dict", write_mindcontrol_entries, "stats") wf.connect(input_node, "output_dir", datasink_node, "base_directory") - wf.connect(dg_node,"volume_paths", mriconvert_node, "in_file") - wf.connect(mriconvert_node,'out_file', datasink_node, 'out_file') + if not no_mriconvert: + wf.connect(dg_node, "volume_paths", mriconvert_node, "in_file") + wf.connect(mriconvert_node, 'out_file', datasink_node, 'out_file') wf.connect(write_mindcontrol_entries, "output_json", datasink_node, "out_file.@json") #wf.write_graph(graph2use='exec') wf.run(plugin=nipype_plugin, plugin_args=nipype_plugin_args) + #load all the freesurfer jsons into the manifest + for path in freesurfer_dir.glob('*'): + subject = path.parts[-1] + # check if mri dir exists, and don't add fsaverage + if os.path.exists(os.path.join(path, 'mri')) and 'average' not in subject: + subj_json = path / 'mindcontrol_entries.json' + with subj_json.open('r') as h: + manifest.extend(json.load(h)) + + with manifest_json.open('w') as h: + json.dump(manifest, h) + + build_command = f"singularity build {simg_path.absolute()} shub://shotgunosine/mindcontrol" + + cmd = f"singularity instance.start -B {logdir.absolute()}:/var/log/nginx" \ + + f" -B {bids_dir.absolute()}:/mc_data" \ + + f" -B {freesurfer_dir.absolute()}:/mc_fs" \ + + f" -B {setdir.absolute()}:/opt/settings" \ + + f" -B {manifest_dir.absolute()}:/mc_startup_data" \ + + f" -B {nginx_scratch.absolute()}:/var/cache/nginx" \ + + f" -B {meteor_ldir.absolute()}:/home/mindcontrol/mindcontrol/.meteor/local" \ + + f" -B {simg_out.absolute()}:/output" \ + + f" -B {mcsetdir.absolute()}:/mc_settings" \ + + f" -H {mc_hdir.absolute()}:/home/{getpass.getuser()} {simg_path.absolute()}" \ + + " mindcontrol" if not args.no_server: - cmd = f"singularity run -B ${{PWD}}/log:/var/log/nginx -B {bids_dir}:/mc_data" \ - + f" -B settings:/opt/settings -B {manifest_dir}:/mc_startup_data" \ - + f" -B ${{PWD}}/log/scratch/nginx/:/var/cache/nginx -B ${{PWD}}/.meteor/:/home/mindcontrol/mindcontrol/.meteor/local" \ - + f" -B ${{PWD}}/settings/mc_settings:/mc_settings -H ${{PWD}}/log/scratch/singularity_home ${{PWD}}/{info['simg']}" + print(build_command) + subprocess.run(build_command, cwd=basedir, shell=True, check=True) + print(cmd) + subprocess.run(cmd, cwd=basedir, shell=True, check=True) + else: + print("Not starting server, but here's the command you would use if you wanted to:") + print(build_command) print(cmd) - subprocess.run(cmd, cwd=mc_singularity_path, shell=True, check=True) From 23dd007dba47e46994c81cab2f81bbb05f396846 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 18 Jul 2018 17:12:02 -0400 Subject: [PATCH 18/60] add circleci config --- .circleci/config.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..7959044 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,10 @@ +version: 2 +jobs: + build: + machine: true + steps: + - checkout + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker build -t shotgunosine/mindcontrol:$CIRCLE_BUILD_NUM-${CIRCLE_SHA1:0:6} -f Dockerfile_services imports/docker/ + - run: docker tag shotgunosine/mindcontrol:$CIRCLE_BUILD_NUM-${CIRCLE_SHA1:0:6} shotgunosine/mindcontrol:latest + - run: docker push shotgunosine/mindcontrol \ No newline at end of file From dcdf746ddbfa444ed4e134406f1d3de29e6d3bed Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 18 Jul 2018 17:17:49 -0400 Subject: [PATCH 19/60] fix dockerpath --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7959044..37c880c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,6 +5,6 @@ jobs: steps: - checkout - run: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: docker build -t shotgunosine/mindcontrol:$CIRCLE_BUILD_NUM-${CIRCLE_SHA1:0:6} -f Dockerfile_services imports/docker/ + - run: docker build -t shotgunosine/mindcontrol:$CIRCLE_BUILD_NUM-${CIRCLE_SHA1:0:6} -f imports/docker/Dockerfile_services - run: docker tag shotgunosine/mindcontrol:$CIRCLE_BUILD_NUM-${CIRCLE_SHA1:0:6} shotgunosine/mindcontrol:latest - run: docker push shotgunosine/mindcontrol \ No newline at end of file From cf8e946bede10a6602261a403e1e779e6121145f Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 18 Jul 2018 17:26:55 -0400 Subject: [PATCH 20/60] add dockerfile_services --- imports/docker/Dockerfile_services | 85 ++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 imports/docker/Dockerfile_services diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services new file mode 100644 index 0000000..72c0afb --- /dev/null +++ b/imports/docker/Dockerfile_services @@ -0,0 +1,85 @@ +FROM nginx:1.15.1 + +MAINTAINER Dylan Nielson + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + vim \ + supervisor \ + rsync \ + procps \ + && rm -rf /var/lib/apt/lists/* + +ENV METEOR_RELEASE 1.7.0.3 +RUN curl https://install.meteor.com/ 2>/dev/null | sed 's/^RELEASE/#RELEASE/'| RELEASE=$METEOR_RELEASE sh + +RUN ln -s ~/.meteor/packages/meteor-tool/*/mt-os.linux.x86_64/dev_bundle/bin/node /usr/bin/ && \ + ln -s ~/.meteor/packages/meteor-tool/*/mt-os.linux.x86_64/dev_bundle/bin/npm /usr/bin/ && \ + rm /etc/nginx/conf.d/default.conf + + +# Installing and setting up miniconda +RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash Miniconda*.sh -b -p /usr/local/miniconda && \ + rm Miniconda*.sh + +ENV PATH=/usr/local/miniconda/bin:$PATH \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 + +# Installing precomputed python packages +RUN conda install -c conda-forge -y \ + awscli \ + boto3 \ + dipy \ + git \ + matplotlib \ + numpy \ + python=3.6 \ + scikit-image \ + scikit-learn \ + wget; \ + sync && \ + chmod +x /usr/local/miniconda/bin/* && \ + conda clean --all -y; sync && \ + python -c "from matplotlib import font_manager" && \ + sed -i 's/\(backend *: \).*$/\1Agg/g' $( python -c "import matplotlib; print(matplotlib.matplotlib_fname())" ) + +RUN npm install http-server -g + +ENV MC_DIR /home/mindcontrol +ENV LC_ALL C + + +COPY entrypoint_nginx.sh /home/entrypoint.sh +#COPY ndmg_launch.sh /home/ndmg_launch.sh + +RUN useradd --create-home --home-dir ${MC_DIR} mindcontrol +RUN chown mindcontrol:mindcontrol /home/entrypoint.sh &&\ + chmod +x /home/entrypoint.sh &&\ + mkdir -p ${MC_DIR}/mindcontrol &&\ + chown -R mindcontrol:mindcontrol /home/mindcontrol &&\ + chmod -R a+rx /home/mindcontrol &&\ + chmod 777 /var/log/nginx /var/cache/nginx /opt /etc &&\ + mkdir -p /mc_data /mc_startup_data /mc_settings /mc_fs &&\ + chown mindcontrol:mindcontrol /mc_data /mc_startup_data /mc_settings /mc_fs &&\ + rm -f /etc/nginx/nginx.conf &&\ + ln -s /opt/settings/auth.htpasswd /etc/nginx/auth.htpasswd &&\ + ln -s /opt/settings/nginx.conf /etc/nginx/nginx.conf &&\ + ln -s /opt/settings/meteor.conf /etc/nginx/conf.d/meteor.conf + +USER mindcontrol + +RUN cd ${MC_DIR}/mindcontrol &&\ + git clone https://github.com/akeshavan/mindcontrol.git ${MC_DIR}/mindcontrol &&\ + meteor update &&\ + meteor npm install --save @babel/runtime &&\ + meteor npm install --save bcrypt &&\ + meteor reset + #git clone https://github.com/clowdcontrol/mindcontrol.git ${MC_DIR}/mindcontrol + + + From 6bf146feb3760d720eebbce04362923651d68e6f Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 18 Jul 2018 17:31:21 -0400 Subject: [PATCH 21/60] fix circlci config --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 37c880c..59c4a5a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,6 +5,6 @@ jobs: steps: - checkout - run: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: docker build -t shotgunosine/mindcontrol:$CIRCLE_BUILD_NUM-${CIRCLE_SHA1:0:6} -f imports/docker/Dockerfile_services + - run: cd imports/docker && docker build -t shotgunosine/mindcontrol:$CIRCLE_BUILD_NUM-${CIRCLE_SHA1:0:6} -f Dockerfile_services . - run: docker tag shotgunosine/mindcontrol:$CIRCLE_BUILD_NUM-${CIRCLE_SHA1:0:6} shotgunosine/mindcontrol:latest - run: docker push shotgunosine/mindcontrol \ No newline at end of file From 39eec6dc4c9a9759708d18ee86b8d60e59d013c0 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 18 Jul 2018 17:54:29 -0400 Subject: [PATCH 22/60] Add dockerhub pull to Singularity file --- imports/docker/Singularity | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index 303ad1a..960e122 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -1,7 +1,6 @@ Bootstrap: docker -Registry: http://localhost:5000 -Namespace: -From: mc_services:latest +Namespace:shotgunosine +From: mindcontrol:latest %labels Mainteiner Dylan Nielson\ From 3cf4129476cbc3201f535da9088570b518ad02c6 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 19 Jul 2018 10:27:59 -0400 Subject: [PATCH 23/60] Fix capitalization of singularity hub repo --- start_singularity_mindcontrol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index e2d70db..bed313a 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -769,7 +769,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats) with manifest_json.open('w') as h: json.dump(manifest, h) - build_command = f"singularity build {simg_path.absolute()} shub://shotgunosine/mindcontrol" + build_command = f"singularity build {simg_path.absolute()} shub://Shotgunosine/mindcontrol" cmd = f"singularity instance.start -B {logdir.absolute()}:/var/log/nginx" \ + f" -B {bids_dir.absolute()}:/mc_data" \ From c587963a735ba0f410ade7897ccf4e4da506b421 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 19 Jul 2018 16:34:56 -0400 Subject: [PATCH 24/60] pass string not path object to pybids --- start_singularity_mindcontrol.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index bed313a..a9627dc 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -578,7 +578,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats) if args.bids_dir is not None: bids_dir = Path(args.bids_dir) try: - layout = BIDSLayout(bids_dir) + layout = BIDSLayout(bids_dir.as_posix()) except ValueError as e: print("Invalid bids directory, skipping none freesurfer files. BIDS error:", e) @@ -783,11 +783,11 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats) + f" -H {mc_hdir.absolute()}:/home/{getpass.getuser()} {simg_path.absolute()}" \ + " mindcontrol" if not args.no_server: - print(build_command) + print(build_command, flush=True) subprocess.run(build_command, cwd=basedir, shell=True, check=True) - print(cmd) + print(cmd, flush=True) subprocess.run(cmd, cwd=basedir, shell=True, check=True) else: print("Not starting server, but here's the command you would use if you wanted to:") - print(build_command) - print(cmd) + print(build_command, flush=True) + print(cmd, flush=True) From ec7c40832ccd0d587693f31aacb775a7f55c5190 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 19 Jul 2018 16:40:52 -0400 Subject: [PATCH 25/60] pullout mindcontrol base dir --- start_singularity_mindcontrol.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index a9627dc..1bb20a7 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -479,7 +479,7 @@ def extract_other_vals_from_aseg(f): # This function creates valid Mindcontrol entries that are saved as .json files. # This f # They can be loaded into the Mindcontrol database later -def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats): +def create_mindcontrol_entries(output_dir, subject, stats): import os from nipype.utils.filemanip import save_json @@ -696,14 +696,12 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats) volumes = ["brainmask.mgz", "wm.mgz", "aparc+aseg.mgz", "T1.mgz", "ribbon.mgz"] input_node = Node(IdentityInterface(fields=['subject_id', "subjects_dir", - "mindcontrol_base_dir", "output_dir", "startup_json_path"]), name='inputnode') input_node.iterables = ("subject_id", subjects) input_node.inputs.subjects_dir = freesurfer_dir - input_node.inputs.mindcontrol_base_dir = bids_dir.as_posix() input_node.inputs.output_dir = freesurfer_dir.as_posix() dg_node = Node(Function(input_names=["subjects_dir", "subject", "volumes"], @@ -721,8 +719,7 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats) output_names=["output_dict"], function=parse_stats), name="get_freesurfer_stats") - write_mindcontrol_entries = Node(Function(input_names=["mindcontrol_base_dir", - "output_dir", + write_mindcontrol_entries = Node(Function(input_names=["output_dir", "subject", "stats", "startup_json_path"], @@ -746,7 +743,6 @@ def create_mindcontrol_entries(mindcontrol_base_dir, output_dir, subject, stats) wf.connect(input_node, "subject_id", get_stats_node, "subject") wf.connect(input_node, "subjects_dir", get_stats_node, "subjects_dir") wf.connect(input_node, "subject_id", write_mindcontrol_entries, "subject") - wf.connect(input_node, "mindcontrol_base_dir", write_mindcontrol_entries, "mindcontrol_base_dir") wf.connect(input_node, "output_dir", write_mindcontrol_entries, "output_dir") wf.connect(get_stats_node, "output_dict", write_mindcontrol_entries, "stats") wf.connect(input_node, "output_dir", datasink_node, "base_directory") From d39c9e75cbf16b692e12dbd8f311d74efd807c9c Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 19 Jul 2018 16:44:07 -0400 Subject: [PATCH 26/60] allow bids_dir to be optional --- start_singularity_mindcontrol.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 1bb20a7..a4f4744 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -766,7 +766,8 @@ def create_mindcontrol_entries(output_dir, subject, stats): json.dump(manifest, h) build_command = f"singularity build {simg_path.absolute()} shub://Shotgunosine/mindcontrol" - + if bids_dir is None: + bids_dir = freesurfer_dir cmd = f"singularity instance.start -B {logdir.absolute()}:/var/log/nginx" \ + f" -B {bids_dir.absolute()}:/mc_data" \ + f" -B {freesurfer_dir.absolute()}:/mc_fs" \ From 1e972cb7cd285af2351a0bc473a0f68f592cae00 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 19 Jul 2018 19:00:25 -0400 Subject: [PATCH 27/60] more general way to find home in startscript --- imports/docker/Singularity | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index 960e122..450fc16 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -6,7 +6,7 @@ From: mindcontrol:latest Mainteiner Dylan Nielson\ %startscript - export HOME=/home/`whoami` + export HOME=$(find /home/ -maxdepth 1 -writable) if [ ! -d $HOME/mindcontrol ] || [ ! -d $HOME/.meteor ] || [ ! -d $HOME/.cordova ] ; then echo "Copying meteor files into singularity_home" > /output/out rsync -ach /home/mindcontrol/mindcontrol $HOME > /output/rsync 2>&1 From abb811203dc37f21e94621b459bc89c2894496fb Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 19 Jul 2018 20:59:07 -0400 Subject: [PATCH 28/60] akeshavan fixing histograms and use_custom --- imports/docker/Dockerfile_services | 2 ++ start_singularity_mindcontrol.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services index 72c0afb..1cb6bbe 100644 --- a/imports/docker/Dockerfile_services +++ b/imports/docker/Dockerfile_services @@ -78,6 +78,8 @@ RUN cd ${MC_DIR}/mindcontrol &&\ meteor update &&\ meteor npm install --save @babel/runtime &&\ meteor npm install --save bcrypt &&\ + meteor remove meteorhacks:aggregate &&\ + meteor add sakulstra:aggregate &&\ meteor reset #git clone https://github.com/clowdcontrol/mindcontrol.git ${MC_DIR}/mindcontrol diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index a4f4744..830f2ef 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -370,7 +370,7 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port # autogenerated settings files pub_set = {"startup_json": startup_file_server+"startup.json", "load_if_empty": True, - "use_custom": True, + "use_custom": False, "needs_consent": False, "modules": modules} settings = {"public": pub_set} From 65f866a5b0e1a03a1437af8e4e3fda194fc51da3 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Tue, 24 Jul 2018 14:05:19 -0400 Subject: [PATCH 29/60] drop symlink for meteor local and add info if pid namespaces aren't allowed --- imports/docker/Singularity | 1 - start_singularity_mindcontrol.py | 33 +++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index 450fc16..de8d5df 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -16,7 +16,6 @@ From: mindcontrol:latest rsync -ach /home/mindcontrol/.cordova $HOME >> /output/rsync 2>&1 echo "home/mindcontrol/.cordova copied to $HOME" >> /output/out fi - ln -s /home/mindcontrol/mindcontrol/.meteor/local $HOME/mindcontrol/.meteor/local/ echo "Starting mindcontrol and nginx" >> /output/out cd $HOME/mindcontrol # grab proper meteor port from settings file diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 830f2ef..6f30fd0 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -603,7 +603,6 @@ def create_mindcontrol_entries(output_dir, subject, stats): setdir = basedir/"settings" mcsetdir = setdir/"mc_settings" manifest_dir = setdir/"mc_manifest_init" - meteor_ldir = basedir/".meteor" simg_path = basedir/"mc_service.simg" logdir = basedir/"log" @@ -630,8 +629,7 @@ def create_mindcontrol_entries(output_dir, subject, stats): manifest_dir.mkdir() if not mc_hdir.exists(): mc_hdir.mkdir() - if not meteor_ldir.exists(): - meteor_ldir.mkdir() + dockerfile = basedir/"Dockerfile_nginx" entrypoint = basedir/"entrypoint_nginx.sh" passfile = setdir/"auth.htpasswd" @@ -641,7 +639,6 @@ def create_mindcontrol_entries(output_dir, subject, stats): mcportfile = mcsetdir/"mc_port" infofile = setdir/"mc_info.json" - # Write settings files write_passfile(passfile) write_nginxconf(ncfile) @@ -765,6 +762,32 @@ def create_mindcontrol_entries(output_dir, subject, stats): with manifest_json.open('w') as h: json.dump(manifest, h) + # Find out if singularity settings allow for pid namespaces + singularity_prefix = (subprocess.check_output("grep '^prefix' $(which singularity)", shell=True) + .decode() + .split('"')[1]) + sysconfdir = (subprocess.check_output("grep '^sysconfdir' $(which singularity)", shell=True) + .decode() + .split('"')[1] + .split('}')[1]) + conf_path = os.path.join(singularity_prefix, sysconfdir[1:], 'singularity/singularity.conf') + allow_pid = (subprocess.check_output(f"grep '^allow pid ns' {conf_path}", shell=True) + .decode() + .split('=')[1] + .strip()) == "yes" + if not allow_pid: + print("Host is not configured to allow pid namespaces!", flush=True) + print("You won't see the instance listed when you run ", flush=True) + print("'singularity instance.list'", flush=True) + print("To stop the mindcontrol server you'll need to ", flush=True) + print("find the process group id for the startscript ", flush=True) + print("with the following command: ", flush=True) + print("'ps -u $(whoami) -o pid,ppid,pgid,sess,cmd --forest'", flush=True) + print("then run:", flush=True) + print("'pkill -9 -g [the PGID for the startscript process]", flush=True) + print("Then you'll need to delete the mongo socket file with: ", flush=True) + print(f"rm /tmp/mongodb-{meteor_port + 1}.sock", flush=True) + build_command = f"singularity build {simg_path.absolute()} shub://Shotgunosine/mindcontrol" if bids_dir is None: bids_dir = freesurfer_dir @@ -774,11 +797,11 @@ def create_mindcontrol_entries(output_dir, subject, stats): + f" -B {setdir.absolute()}:/opt/settings" \ + f" -B {manifest_dir.absolute()}:/mc_startup_data" \ + f" -B {nginx_scratch.absolute()}:/var/cache/nginx" \ - + f" -B {meteor_ldir.absolute()}:/home/mindcontrol/mindcontrol/.meteor/local" \ + f" -B {simg_out.absolute()}:/output" \ + f" -B {mcsetdir.absolute()}:/mc_settings" \ + f" -H {mc_hdir.absolute()}:/home/{getpass.getuser()} {simg_path.absolute()}" \ + " mindcontrol" + if not args.no_server: print(build_command, flush=True) subprocess.run(build_command, cwd=basedir, shell=True, check=True) From 0b2f8fe1eb265bb98aabd036ced204e3945bc210 Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Tue, 24 Jul 2018 14:16:52 -0400 Subject: [PATCH 30/60] Update README.md --- README.md | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8587d61..66bb8ed 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![CircleCI](https://circleci.com/gh/Shotgunosine/mindcontrol.svg?style=svg)](https://circleci.com/gh/Shotgunosine/mindcontrol) + # mindcontrol MindControl is an app for quality control of neuroimaging pipeline outputs. @@ -35,34 +37,34 @@ Create a database json file similar to [http://dxugxjm290185.cloudfront.net/hbn/ * Define each module in `settings.dev.json` to point to your `entry_type`, and define the module's `staticURL` ## Bulid and deploy with singularity -First you'll build a singularity container on a system where you've got docker installed. - -Clone this repository - +Connect to the system where you'd like to host mindcontrol with a port forwarded: +``` +ssh -L 3000:localhost:3000 server +``` +Clone mindcontrol to a directory where you've got ~ 10 GB of free space. ``` git clone https://github.com/Shotgunosine/mindcontrol cd mindcontrol -git checkout hpc_compat ``` +You'll need to have a python 3 environment with access to nipype, pybids, freesurfer, and singularity. +Run start_singularity_mindcontrol.py. You can see its documentation with `python start_singularity_mindcontrol.py -h`. -Run auto_singularity_mindcontrol.py. You can see its documentation with `python auto_singularity_mindcontrol.py -h`. ``` -python3 auto_singularity_mindcontrol.py --sing_out_dir ~/mc_sing --freesurfer +python start_singularity_mindcontrol.py —freesurfer_dir [path to directory containing subdirectories for all the subjects] --sing_out_dir [path where mindcontrol can output files] --freesurfer ``` -Copy the directory produced to the system you want to run mindcontrol on. For example, a node on the cluster where your data is already located. +This command does a number of things: +1) Prompt you to create users and passwords. I would just create one user for your lab and a password that you don’t mind sharing with others in the lab.creates folders with all the settings files that need to be loaded. +2) Runs mriconvert to convert .mgz to .nii.gz for T1, aparc+aseg, ribbion, and wm. Right now I’m just dropping those converted niftis in the directory beside the .mgzs, let me know if that’s a problem though. +3) Pulls the singularity image. +4) Starts an instance of the singularity image running with everything mounted appropriately. -``` -rsync -ach ~/mc_sing server:/data/project/ -``` - -Connect to that server with a few ports forwarded and run start_singularity_mindcontrol from that directory. You can see its documentation with `python start_singularity_mindcontrol.py -h`. You'll need freesurfer, singularity, python3 and nipype on that system. +Inside the image, there’s a fair bit of file copying that has to get done. It takes a while. +You can check the progress with `cat log/simg_out/out`. +Once that says `Starting mindcontrol and nginx`, you can `cat log/simg_out/mindcontrol.out` to see what mindcontrol is doing. +Once that says `App running at: http://localhost:2998/`, mindcontrol is all set up and running (but ignore that port number, it’s running on port 3000). -``` -ssh -L 3000:localhost:3000 -L 3003:localhost:3003 server -cd /data/project/mc_sing -start_singularity_mindcontrol.py --bids_dir /data/project/bids_directory --freesurfer_dir /data/project/bids_directory/derivatives/freesurfer -``` +Anyone who wants to see mindcontrol can then `ssh -L 3000:localhost:3000` to the server and browse to http://localhost:3000 in their browser. They’ll be prompted to login with the username and password you created way back in step 1. ## Demo From 09fc5eff3564ff83fda4e964084a78670740d17d Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Tue, 24 Jul 2018 14:40:12 -0400 Subject: [PATCH 31/60] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 66bb8ed..75f70a5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![CircleCI](https://circleci.com/gh/Shotgunosine/mindcontrol.svg?style=svg)](https://circleci.com/gh/Shotgunosine/mindcontrol) +[![https://www.singularity-hub.org/static/img/hosted-singularity--hub-%23e32929.svg](https://www.singularity-hub.org/static/img/hosted-singularity--hub-%23e32929.svg)](https://singularity-hub.org/collections/1293) # mindcontrol MindControl is an app for quality control of neuroimaging pipeline outputs. From af0c2039e1bea48a7013f3980b6632d388a85472 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 26 Jul 2018 15:41:05 -0400 Subject: [PATCH 32/60] Create one homedir per user --- imports/docker/Singularity | 2 +- start_singularity_mindcontrol.py | 48 +++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index de8d5df..eb818f1 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -6,7 +6,7 @@ From: mindcontrol:latest Mainteiner Dylan Nielson\ %startscript - export HOME=$(find /home/ -maxdepth 1 -writable) + export HOME=$(find /home/ -maxdepth 1 -writable -not -name "singularity_home") if [ ! -d $HOME/mindcontrol ] || [ ! -d $HOME/.meteor ] || [ ! -d $HOME/.cordova ] ; then echo "Copying meteor files into singularity_home" > /output/out rsync -ach /home/mindcontrol/mindcontrol $HOME > /output/rsync 2>&1 diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 6f30fd0..7b88af9 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -378,6 +378,22 @@ def write_mcsettings(mcsetfile, entry_types=None, freesurfer=False, startup_port json.dump(settings, h) +def write_startfile(startfile, workdir, cmd): + + script = f"""#! /bin/bash +cd {workdir.absolute()} +if [ ! -d scratch/singularity_home_${{USER}} ]; then + mkdir scratch/singularity_home_${{USER}} + cd scratch/singularity_home_${{USER}} + ln -s ../singularity_home/.cordova + ln -s ../singularity_home/.meteor + ln -s ../singularity_home/mindcontrol +fi +{cmd} +""" + startfile.write_text(script) + + #this function finds data in the subjects_dir def data_grabber(subjects_dir, subject, volumes): import os @@ -536,12 +552,15 @@ def create_mindcontrol_entries(output_dir, subject, stats): ' Pass this argument multiple times to add additional modules.') parser.add_argument('--startup_port', default=3003, + type=int, help='Port number at which mindcontrol will look for startup manifest.') parser.add_argument('--nginx_port', default=3000, + type=int, help='Port number at nginx will run. This is the port you connect to reach mindcontrol.') parser.add_argument('--meteor_port', default=2998, + type=int, help='Port number at meteor will run. ' 'This is mostly under the hood, ' 'but you might need to change it if there is a port conflict.' @@ -638,6 +657,7 @@ def create_mindcontrol_entries(output_dir, subject, stats): mcsetfile = mcsetdir/"mc_nginx_settings.json" mcportfile = mcsetdir/"mc_port" infofile = setdir/"mc_info.json" + startfile = basedir/"start_mindcontrol.sh" # Write settings files write_passfile(passfile) @@ -771,11 +791,11 @@ def create_mindcontrol_entries(output_dir, subject, stats): .split('"')[1] .split('}')[1]) conf_path = os.path.join(singularity_prefix, sysconfdir[1:], 'singularity/singularity.conf') - allow_pid = (subprocess.check_output(f"grep '^allow pid ns' {conf_path}", shell=True) + allow_pidns = (subprocess.check_output(f"grep '^allow pid ns' {conf_path}", shell=True) .decode() .split('=')[1] .strip()) == "yes" - if not allow_pid: + if not allow_pidns: print("Host is not configured to allow pid namespaces!", flush=True) print("You won't see the instance listed when you run ", flush=True) print("'singularity instance.list'", flush=True) @@ -791,17 +811,19 @@ def create_mindcontrol_entries(output_dir, subject, stats): build_command = f"singularity build {simg_path.absolute()} shub://Shotgunosine/mindcontrol" if bids_dir is None: bids_dir = freesurfer_dir - cmd = f"singularity instance.start -B {logdir.absolute()}:/var/log/nginx" \ - + f" -B {bids_dir.absolute()}:/mc_data" \ - + f" -B {freesurfer_dir.absolute()}:/mc_fs" \ - + f" -B {setdir.absolute()}:/opt/settings" \ - + f" -B {manifest_dir.absolute()}:/mc_startup_data" \ - + f" -B {nginx_scratch.absolute()}:/var/cache/nginx" \ - + f" -B {simg_out.absolute()}:/output" \ - + f" -B {mcsetdir.absolute()}:/mc_settings" \ - + f" -H {mc_hdir.absolute()}:/home/{getpass.getuser()} {simg_path.absolute()}" \ - + " mindcontrol" - + startcmd = f"singularity instance.start -B {logdir.absolute()}:/var/log/nginx" \ + + f" -B {bids_dir.absolute()}:/mc_data" \ + + f" -B {freesurfer_dir.absolute()}:/mc_fs" \ + + f" -B {setdir.absolute()}:/opt/settings" \ + + f" -B {manifest_dir.absolute()}:/mc_startup_data" \ + + f" -B {nginx_scratch.absolute()}:/var/cache/nginx" \ + + f" -B {simg_out.absolute()}:/output" \ + + f" -B {mcsetdir.absolute()}:/mc_settings" \ + + f" -B {mc_hdir.absolute()}:/home/singularity_home" \ + + f" -H {mc_hdir.absolute() + '_' + getpass.getuser()}:/home/{getpass.getuser()} {simg_path.absolute()}" \ + + " mindcontrol" + write_startfile(startfile, basedir, startcmd) + cmd = f"/bin/bash {startfile.absolute()}" if not args.no_server: print(build_command, flush=True) subprocess.run(build_command, cwd=basedir, shell=True, check=True) From e70f5da09b283ed32bfc5134ae42c1cf4b8e02a4 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 26 Jul 2018 17:05:00 -0400 Subject: [PATCH 33/60] Fix start script --- start_singularity_mindcontrol.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 7b88af9..929316f 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -788,13 +788,17 @@ def create_mindcontrol_entries(output_dir, subject, stats): .split('"')[1]) sysconfdir = (subprocess.check_output("grep '^sysconfdir' $(which singularity)", shell=True) .decode() - .split('"')[1] - .split('}')[1]) - conf_path = os.path.join(singularity_prefix, sysconfdir[1:], 'singularity/singularity.conf') + .split('"')[1]) + try: + sysconfdir = sysconfdir.split('}')[1] + conf_path = os.path.join(singularity_prefix, sysconfdir[1:], 'singularity/singularity.conf') + except IndexError: + conf_path = os.path.join(sysconfdir, 'singularity/singularity.conf') + allow_pidns = (subprocess.check_output(f"grep '^allow pid ns' {conf_path}", shell=True) - .decode() - .split('=')[1] - .strip()) == "yes" + .decode() + .split('=')[1] + .strip()) == "yes" if not allow_pidns: print("Host is not configured to allow pid namespaces!", flush=True) print("You won't see the instance listed when you run ", flush=True) @@ -820,7 +824,7 @@ def create_mindcontrol_entries(output_dir, subject, stats): + f" -B {simg_out.absolute()}:/output" \ + f" -B {mcsetdir.absolute()}:/mc_settings" \ + f" -B {mc_hdir.absolute()}:/home/singularity_home" \ - + f" -H {mc_hdir.absolute() + '_' + getpass.getuser()}:/home/{getpass.getuser()} {simg_path.absolute()}" \ + + f" -H {mc_hdir.absolute().as_posix() + '_' + getpass.getuser()}:/home/{getpass.getuser()} {simg_path.absolute()}" \ + " mindcontrol" write_startfile(startfile, basedir, startcmd) cmd = f"/bin/bash {startfile.absolute()}" From 02d055122ac5a536b5a7435d35d2e9a429fb5940 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 26 Jul 2018 17:35:09 -0400 Subject: [PATCH 34/60] add my_readme creation --- start_singularity_mindcontrol.py | 42 +++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 929316f..a4d9be8 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -658,6 +658,8 @@ def create_mindcontrol_entries(output_dir, subject, stats): mcportfile = mcsetdir/"mc_port" infofile = setdir/"mc_info.json" startfile = basedir/"start_mindcontrol.sh" + readme = basedir/"my_readme.md" + readme_str = "# Welcome to your mindcontrol instance \n" # Write settings files write_passfile(passfile) @@ -800,17 +802,22 @@ def create_mindcontrol_entries(output_dir, subject, stats): .split('=')[1] .strip()) == "yes" if not allow_pidns: - print("Host is not configured to allow pid namespaces!", flush=True) - print("You won't see the instance listed when you run ", flush=True) - print("'singularity instance.list'", flush=True) - print("To stop the mindcontrol server you'll need to ", flush=True) - print("find the process group id for the startscript ", flush=True) - print("with the following command: ", flush=True) - print("'ps -u $(whoami) -o pid,ppid,pgid,sess,cmd --forest'", flush=True) - print("then run:", flush=True) - print("'pkill -9 -g [the PGID for the startscript process]", flush=True) - print("Then you'll need to delete the mongo socket file with: ", flush=True) - print(f"rm /tmp/mongodb-{meteor_port + 1}.sock", flush=True) + stop_str = '\n'.join(["Host is not configured to allow pid namespaces!", + "You won't see the instance listed when you run ", + "'singularity instance.list'", + "To stop the mindcontrol server you'll need to ", + "find the process group id for the startscript ", + "with the following command: ", + "`ps -u $(whoami) -o pid,ppid,pgid,sess,cmd --forest`", + "then run:", + "`pkill -9 -g [the PGID for the startscript process]`", + "Then you'll need to delete the mongo socket file with: ", + f"rm /tmp/mongodb-{meteor_port + 1}.sock" + ]) + else: + print("To stop the mindcontrol server run:", flush=True) + stop_str += "`singularity instance.stop mindcontrol`" + print(stop_str, flush=True) build_command = f"singularity build {simg_path.absolute()} shub://Shotgunosine/mindcontrol" if bids_dir is None: @@ -824,16 +831,27 @@ def create_mindcontrol_entries(output_dir, subject, stats): + f" -B {simg_out.absolute()}:/output" \ + f" -B {mcsetdir.absolute()}:/mc_settings" \ + f" -B {mc_hdir.absolute()}:/home/singularity_home" \ - + f" -H {mc_hdir.absolute().as_posix() + '_' + getpass.getuser()}:/home/{getpass.getuser()} {simg_path.absolute()}" \ + + f" -H {mc_hdir.absolute().as_posix() + '_'}${{USER}}:/home/${{USER}} {simg_path.absolute()}" \ + " mindcontrol" write_startfile(startfile, basedir, startcmd) cmd = f"/bin/bash {startfile.absolute()}" if not args.no_server: + readme_str += "## Sinularity image was built with this comand \n" print(build_command, flush=True) subprocess.run(build_command, cwd=basedir, shell=True, check=True) print(cmd, flush=True) subprocess.run(cmd, cwd=basedir, shell=True, check=True) else: + readme_str += "## To build the singularity image \n" print("Not starting server, but here's the command you would use if you wanted to:") print(build_command, flush=True) print(cmd, flush=True) + readme_str += f"`{build_command}` \n" + readme_str += "## Check to see if the singularity instance is running \n" + readme_str += "`singularity instance.list mindcontrol` \n" + readme_str += "## Start a singularity mindcontrol instance is running \n" + readme_str += f"`{cmd}` \n" + readme_str += "## Stop a singularity mindcontrol instance \n" + readme_str += stop_str + readme.write_text(readme_str) + From 16992fd6f65901df99798f351625d073b30dbbde Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Fri, 27 Jul 2018 11:48:10 -0400 Subject: [PATCH 35/60] add qroup as a required parameter --- imports/docker/Singularity | 10 +++++----- start_singularity_mindcontrol.py | 25 ++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index eb818f1..8b5850f 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -7,14 +7,14 @@ From: mindcontrol:latest %startscript export HOME=$(find /home/ -maxdepth 1 -writable -not -name "singularity_home") - if [ ! -d $HOME/mindcontrol ] || [ ! -d $HOME/.meteor ] || [ ! -d $HOME/.cordova ] ; then + if [ ! -d /home/singularity_home/mindcontrol ] || [ ! -d /home/singularity_home/.meteor ] || [ ! -d /home/singularity_home/.cordova ] ; then echo "Copying meteor files into singularity_home" > /output/out - rsync -ach /home/mindcontrol/mindcontrol $HOME > /output/rsync 2>&1 + rsync -ach /home/mindcontrol/mindcontrol /home/singularity_home/ > /output/rsync 2>&1 echo "/home/mindcontrol/mindcontrol copied to $HOME" >> /output/out - rsync -ach /home/mindcontrol/.meteor $HOME >> /output/rsync 2>&1 + rsync -ach /home/mindcontrol/.meteor /home/singularity_home/ >> /output/rsync 2>&1 echo "/home/mindcontrol/.meteor copied to $HOME" >> /output/out - rsync -ach /home/mindcontrol/.cordova $HOME >> /output/rsync 2>&1 - echo "home/mindcontrol/.cordova copied to $HOME" >> /output/out + rsync -ach /home/mindcontrol/.cordova /home/singularity_home/ >> /output/rsync 2>&1 + echo "/home/mindcontrol/.cordova copied to $HOME" >> /output/out fi echo "Starting mindcontrol and nginx" >> /output/out cd $HOME/mindcontrol diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index a4d9be8..ed1c2d2 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -9,6 +9,7 @@ import getpass import random import sys +import grp from nipype import MapNode, Workflow, Node from nipype.interfaces.freesurfer import MRIConvert @@ -538,6 +539,8 @@ def create_mindcontrol_entries(output_dir, subject, stats): parser = argparse.ArgumentParser(description='Start mindcontrol in a previously built' ' singularity container and create initial manifest' ' if needed.') + parser.add_argument('group', + help='Name of the group under which mindcontrol directories should be created') parser.add_argument('--sing_out_dir', default='.', help='Directory to bulid singularirty image and files in.') @@ -580,7 +583,25 @@ def create_mindcontrol_entries(output_dir, subject, stats): help='json formatted string of keyword arguments for nipype_plugin') args = parser.parse_args() + mc_gnam = args.group + mc_gid = grp.getgrnam(mc_gnam)[2] + # Check if username and gnam are the same and print a warning + if mc_gnam == getpass.getuser(): + print("WARNING: You've set the group to your user group, no one else " + "will be able to start this mindcontrol instance.") + # Check if user is in group and if not throw an error + if mc_gid not in os.getgroups(): + raise ValueError("You must be a member of the group specified for" + " mindcontrol.") + # Check current umask, if it's not 002, throw an error + current_umask = os.umask(0) + os.umask(current_umask) + if current_umask > 2: + raise Exception("This command must be run with a umask of 2, run" + " 'umask 002' to set the umask then try" + " this command again") sing_out_dir = args.sing_out_dir + if args.custom_settings is not None: custom_settings = Path(args.custom_settings) else: @@ -632,6 +653,8 @@ def create_mindcontrol_entries(output_dir, subject, stats): if not basedir.exists(): basedir.mkdir() + chmod_cmd = f'chgrp -R {mc_gnam} {basedir} && chmod -R g+s {basedir} && chmod -R 770 {basedir}' + _chmod_res = subprocess.check_output(chmod_cmd, shell=True) if not setdir.exists(): setdir.mkdir() if not logdir.exists(): @@ -816,7 +839,7 @@ def create_mindcontrol_entries(output_dir, subject, stats): ]) else: print("To stop the mindcontrol server run:", flush=True) - stop_str += "`singularity instance.stop mindcontrol`" + stop_str = "`singularity instance.stop mindcontrol`" print(stop_str, flush=True) build_command = f"singularity build {simg_path.absolute()} shub://Shotgunosine/mindcontrol" From b28689ac435c9856d0b0584714227c21c51d4521 Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Fri, 27 Jul 2018 11:56:37 -0400 Subject: [PATCH 36/60] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 75f70a5..d626481 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![CircleCI](https://circleci.com/gh/Shotgunosine/mindcontrol.svg?style=svg)](https://circleci.com/gh/Shotgunosine/mindcontrol) +[![CircleCI](https://circleci.com/gh/Shotgunosine/mindcontrol/tree/master.svg?style=svg)](https://circleci.com/gh/Shotgunosine/mindcontrol/tree/master) [![https://www.singularity-hub.org/static/img/hosted-singularity--hub-%23e32929.svg](https://www.singularity-hub.org/static/img/hosted-singularity--hub-%23e32929.svg)](https://singularity-hub.org/collections/1293) # mindcontrol From 71c3d2bc13bba7f38070ea566fb92f4ea35e47b0 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Fri, 27 Jul 2018 20:24:24 -0400 Subject: [PATCH 37/60] change rsync command --- imports/docker/Singularity | 9 +++++--- start_singularity_mindcontrol.py | 37 +++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index 8b5850f..5095f70 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -9,11 +9,14 @@ From: mindcontrol:latest export HOME=$(find /home/ -maxdepth 1 -writable -not -name "singularity_home") if [ ! -d /home/singularity_home/mindcontrol ] || [ ! -d /home/singularity_home/.meteor ] || [ ! -d /home/singularity_home/.cordova ] ; then echo "Copying meteor files into singularity_home" > /output/out - rsync -ach /home/mindcontrol/mindcontrol /home/singularity_home/ > /output/rsync 2>&1 + rsync -rlD /home/mindcontrol/mindcontrol /home/singularity_home/ > /output/rsync 2>&1 + chmod -R 770 /home/singularity_home/mindcontrol echo "/home/mindcontrol/mindcontrol copied to $HOME" >> /output/out - rsync -ach /home/mindcontrol/.meteor /home/singularity_home/ >> /output/rsync 2>&1 + rsync -rlD /home/mindcontrol/.meteor /home/singularity_home/ >> /output/rsync 2>&1 + chmod -R 770 /home/singularity_home/.meteor echo "/home/mindcontrol/.meteor copied to $HOME" >> /output/out - rsync -ach /home/mindcontrol/.cordova /home/singularity_home/ >> /output/rsync 2>&1 + rsync -rlD /home/mindcontrol/.cordova /home/singularity_home/ >> /output/rsync 2>&1 + chmod -R 770 /home/singularity_home/.cordova echo "/home/mindcontrol/.cordova copied to $HOME" >> /output/out fi echo "Starting mindcontrol and nginx" >> /output/out diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index ed1c2d2..5d195ab 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -10,6 +10,7 @@ import random import sys import grp +import socket from nipype import MapNode, Workflow, Node from nipype.interfaces.freesurfer import MRIConvert @@ -395,6 +396,21 @@ def write_startfile(startfile, workdir, cmd): startfile.write_text(script) +def write_stopfile(stopfile, workdir, group, cmd, run_stop=True): + #find scratch/singularity_home ! -group {group} -exec chmod 770 {{}} \; -exec chown :{group} {{}} \; + + if run_stop: + script = f"""#! /bin/bash +cd {workdir.absolute()} +{cmd} +chown -R :{group} scratch/singularity_home/mindcontrol/.meteor/local +chmod -R 770 scratch/singularity_home/mindcontrol/.meteor/local +rm -rf scratch/singularity_home/mindcontrol/.meteor/local/db/mongod.lock +""" + else: + raise NotImplementedError + stopfile.write_text(script) + #this function finds data in the subjects_dir def data_grabber(subjects_dir, subject, volumes): import os @@ -681,6 +697,7 @@ def create_mindcontrol_entries(output_dir, subject, stats): mcportfile = mcsetdir/"mc_port" infofile = setdir/"mc_info.json" startfile = basedir/"start_mindcontrol.sh" + stopfile = basedir/"stop_mindcontrol.sh" readme = basedir/"my_readme.md" readme_str = "# Welcome to your mindcontrol instance \n" @@ -696,10 +713,6 @@ def create_mindcontrol_entries(output_dir, subject, stats): else: write_mcsettings(mcsetfile, entry_types=entry_types, freesurfer=freesurfer, startup_port=startup_port, nginx_port=nginx_port) - # Copy singularity run script to directory - srun_source = Path(__file__) - srun_dest = basedir / 'start_singularity_mindcontrol.py' - copyfile(srun_source.as_posix(), srun_dest.as_posix()) # infofile = mc_singularity_path/'settings/mc_info.json' # with infofile.open('r') as h: @@ -825,7 +838,7 @@ def create_mindcontrol_entries(output_dir, subject, stats): .split('=')[1] .strip()) == "yes" if not allow_pidns: - stop_str = '\n'.join(["Host is not configured to allow pid namespaces!", + stop_cmd = '\n'.join(["Host is not configured to allow pid namespaces!", "You won't see the instance listed when you run ", "'singularity instance.list'", "To stop the mindcontrol server you'll need to ", @@ -838,9 +851,7 @@ def create_mindcontrol_entries(output_dir, subject, stats): f"rm /tmp/mongodb-{meteor_port + 1}.sock" ]) else: - print("To stop the mindcontrol server run:", flush=True) - stop_str = "`singularity instance.stop mindcontrol`" - print(stop_str, flush=True) + stop_cmd = "singularity instance.stop mindcontrol" build_command = f"singularity build {simg_path.absolute()} shub://Shotgunosine/mindcontrol" if bids_dir is None: @@ -857,6 +868,7 @@ def create_mindcontrol_entries(output_dir, subject, stats): + f" -H {mc_hdir.absolute().as_posix() + '_'}${{USER}}:/home/${{USER}} {simg_path.absolute()}" \ + " mindcontrol" write_startfile(startfile, basedir, startcmd) + write_stopfile(stopfile, basedir, mc_gnam, stop_cmd, allow_pidns) cmd = f"/bin/bash {startfile.absolute()}" if not args.no_server: readme_str += "## Sinularity image was built with this comand \n" @@ -869,12 +881,17 @@ def create_mindcontrol_entries(output_dir, subject, stats): print("Not starting server, but here's the command you would use if you wanted to:") print(build_command, flush=True) print(cmd, flush=True) + print("To stop the mindcontrol server run:", flush=True) + print(f'/bin/bash {stopfile.absolute()}', flush=True) readme_str += f"`{build_command}` \n" readme_str += "## Check to see if the singularity instance is running \n" readme_str += "`singularity instance.list mindcontrol` \n" - readme_str += "## Start a singularity mindcontrol instance is running \n" + readme_str += "## Start a singularity mindcontrol instance \n" readme_str += f"`{cmd}` \n" + readme_str += "## Connect to this instance \n" + readme_str += f"`ssh -L {nginx_port}:localhost:{nginx_port} {socket.gethostname()}` \n" + readme_str += f"then browse to http:\\localhost:{nginx_port} on the machine you connected from. \n" readme_str += "## Stop a singularity mindcontrol instance \n" - readme_str += stop_str + readme_str += f'/bin/bash {stopfile.absolute()}' readme.write_text(readme_str) From b4a7d3c1c7e932de4489e570c6c3107f844e6a3a Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Mon, 30 Jul 2018 16:02:51 -0400 Subject: [PATCH 38/60] add mongo utilities --- imports/docker/Dockerfile_services | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services index 1cb6bbe..e637901 100644 --- a/imports/docker/Dockerfile_services +++ b/imports/docker/Dockerfile_services @@ -2,7 +2,9 @@ FROM nginx:1.15.1 MAINTAINER Dylan Nielson -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 \ + echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/4.0 main" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list \ + apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ curl \ @@ -11,6 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ supervisor \ rsync \ procps \ + mongodb-org \ && rm -rf /var/lib/apt/lists/* ENV METEOR_RELEASE 1.7.0.3 From b88dbdddeec6be282275b806dd31e5c9a6589df8 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Mon, 30 Jul 2018 16:18:35 -0400 Subject: [PATCH 39/60] fix mongo install commands --- imports/docker/Dockerfile_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services index e637901..bdef2aa 100644 --- a/imports/docker/Dockerfile_services +++ b/imports/docker/Dockerfile_services @@ -3,7 +3,7 @@ FROM nginx:1.15.1 MAINTAINER Dylan Nielson RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 \ - echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/4.0 main" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list \ + echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/4.0 main" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list \ apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ From 0a5aa6b5215131c544c6e35137765db81ff8f43e Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Mon, 30 Jul 2018 16:21:18 -0400 Subject: [PATCH 40/60] fix mongo install commands --- imports/docker/Dockerfile_services | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services index bdef2aa..8dfee7c 100644 --- a/imports/docker/Dockerfile_services +++ b/imports/docker/Dockerfile_services @@ -3,8 +3,8 @@ FROM nginx:1.15.1 MAINTAINER Dylan Nielson RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 \ - echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/4.0 main" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list \ - apt-get install -y --no-install-recommends \ + && echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/4.0 main" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list \ + && apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ curl \ From 7c04a77af46c1b324947b0f41427a31ed82d61e2 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Mon, 30 Jul 2018 16:22:54 -0400 Subject: [PATCH 41/60] fix mongo install commands --- imports/docker/Dockerfile_services | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services index 8dfee7c..2ae5195 100644 --- a/imports/docker/Dockerfile_services +++ b/imports/docker/Dockerfile_services @@ -2,7 +2,8 @@ FROM nginx:1.15.1 MAINTAINER Dylan Nielson -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 \ +RUN apt-get update && apt-get install -my wget gnupg \ + && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 \ && echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/4.0 main" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list \ && apt-get install -y --no-install-recommends \ build-essential \ From 8669503910df6bf2344f0a55b3974f597deb4796 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Mon, 30 Jul 2018 16:27:03 -0400 Subject: [PATCH 42/60] fix mongo install commands --- imports/docker/Dockerfile_services | 1 + 1 file changed, 1 insertion(+) diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services index 2ae5195..32572f1 100644 --- a/imports/docker/Dockerfile_services +++ b/imports/docker/Dockerfile_services @@ -5,6 +5,7 @@ MAINTAINER Dylan Nielson RUN apt-get update && apt-get install -my wget gnupg \ && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 \ && echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/4.0 main" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list \ + && apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ From c97bfc90c278bbc08e0b8708c1a03231f06cb4bd Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Tue, 31 Jul 2018 23:16:23 -0400 Subject: [PATCH 43/60] database dump and restore --- imports/docker/Singularity | 24 ++++++++++++++++++++++-- start_singularity_mindcontrol.py | 16 +++++++++++++--- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index 5095f70..d380092 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -19,9 +19,29 @@ From: mindcontrol:latest chmod -R 770 /home/singularity_home/.cordova echo "/home/mindcontrol/.cordova copied to $HOME" >> /output/out fi - echo "Starting mindcontrol and nginx" >> /output/out cd $HOME/mindcontrol # grab proper meteor port from settings file MC_PORT=$(cat /mc_settings/mc_port) + MONGO_PORT=$(expr ${MC_PORT} + 1) + + # Check if we need to reset meteor + DB_OWNER=$(stat -c %U /home/singularity_home/mindcontrol/.meteor/local/db) + + if [ ${DB_OWNER} != $(stat -c %U $HOME) ] ; then + if [ ! -d /output/mindcontrol_database ] ; then + echo "Someone else owns the mongo database and there's no database dump to load from at /output/mindcontrol_database. Something's gone wrong. Exiting so we don't destroy data." >> /output/out + exit 64 + fi + echo "Someone else owns the db and a database dump was found, resetting Meteor" >> /output/out + meteor reset + RESTORE_DB=Yes + fi + echo "Starting mindcontrol and nginx" >> /output/out + nginx nohup meteor --settings /mc_settings/mc_nginx_settings.json --port $MC_PORT > /output/mindcontrol.out 2>&1 & - nginx -g "daemon off;" \ No newline at end of file + if [ ${RESTORE_DB} == Yes ] ; then + + # from https://superuser.com/a/809159 + grep -m 1 "=> Started your app." <(tail -f /output/mindcontrol.out) + mongorestore --port=${MONGO_PORT} --drop --preserveUUID --gzip /output/mindcontrol_database/ + fi diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 5d195ab..d30e30e 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -396,16 +396,26 @@ def write_startfile(startfile, workdir, cmd): startfile.write_text(script) -def write_stopfile(stopfile, workdir, group, cmd, run_stop=True): +def write_stopfile(stopfile, workdir, group, cmd, meteor_port, run_stop=True): #find scratch/singularity_home ! -group {group} -exec chmod 770 {{}} \; -exec chown :{group} {{}} \; if run_stop: script = f"""#! /bin/bash cd {workdir.absolute()} +singularity exec instance://mindcontrol mongodump --out=/output/mindcontrol_database --port={meteor_port+1} --gzip +if [ -d log/simg_out/mindcontrol_database ] ; then + DATE=$(date +"%Y%m%d%H%M%S") + echo "Saving previous database dump to log/simg_out/mindcontrol_database_${{DATE}}.tar.gz" + tar -czf log/simg_out/mindcontrol_database_${{DATE}}.tar.gz log/simg_out/mindcontrol_database/ +fi + +singularity exec instance://mindcontrol mongod --dbpath=/home/${{USER}}/mindcontrol/.meteor/local/db --shutdown {cmd} +echo "Waiting 30 seconds for everything to finish writing" +sleep 30 chown -R :{group} scratch/singularity_home/mindcontrol/.meteor/local chmod -R 770 scratch/singularity_home/mindcontrol/.meteor/local -rm -rf scratch/singularity_home/mindcontrol/.meteor/local/db/mongod.lock +chmod -R 770 log """ else: raise NotImplementedError @@ -868,7 +878,7 @@ def create_mindcontrol_entries(output_dir, subject, stats): + f" -H {mc_hdir.absolute().as_posix() + '_'}${{USER}}:/home/${{USER}} {simg_path.absolute()}" \ + " mindcontrol" write_startfile(startfile, basedir, startcmd) - write_stopfile(stopfile, basedir, mc_gnam, stop_cmd, allow_pidns) + write_stopfile(stopfile, basedir, mc_gnam, stop_cmd, meteor_port, allow_pidns) cmd = f"/bin/bash {startfile.absolute()}" if not args.no_server: readme_str += "## Sinularity image was built with this comand \n" From ec78b06d26bbf4e4704306e74a83102c2ec78bba Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 1 Aug 2018 01:04:21 -0400 Subject: [PATCH 44/60] Fix bash specific syntax in startscript --- imports/docker/Singularity | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index d380092..fc81528 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -42,6 +42,7 @@ From: mindcontrol:latest if [ ${RESTORE_DB} == Yes ] ; then # from https://superuser.com/a/809159 - grep -m 1 "=> Started your app." <(tail -f /output/mindcontrol.out) + export DELAY_STRING="=> Started your app." + /bin/bash -c "grep -m 1 $DELAY_STRING <(tail -f /output/mindcontrol.out)" mongorestore --port=${MONGO_PORT} --drop --preserveUUID --gzip /output/mindcontrol_database/ fi From 7767f0551c147276d9acdc9fd181a311cb22f704 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 1 Aug 2018 02:10:03 -0400 Subject: [PATCH 45/60] Fix sh bugs --- imports/docker/Singularity | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index fc81528..c28e488 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -34,12 +34,12 @@ From: mindcontrol:latest fi echo "Someone else owns the db and a database dump was found, resetting Meteor" >> /output/out meteor reset - RESTORE_DB=Yes + RESTORE_DB="Yes" fi echo "Starting mindcontrol and nginx" >> /output/out nginx nohup meteor --settings /mc_settings/mc_nginx_settings.json --port $MC_PORT > /output/mindcontrol.out 2>&1 & - if [ ${RESTORE_DB} == Yes ] ; then + if [ "${RESTORE_DB}"" == "Yes" ] ; then # from https://superuser.com/a/809159 export DELAY_STRING="=> Started your app." From 0b8c1521d004307574d5add8ca2dc02bbea81687 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 1 Aug 2018 04:43:54 -0400 Subject: [PATCH 46/60] For real fix sh bugs --- imports/docker/Singularity | 18 ++++++++++-------- start_singularity_mindcontrol.py | 3 +-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index c28e488..62b22ec 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -25,8 +25,11 @@ From: mindcontrol:latest MONGO_PORT=$(expr ${MC_PORT} + 1) # Check if we need to reset meteor - DB_OWNER=$(stat -c %U /home/singularity_home/mindcontrol/.meteor/local/db) - + DB_OWNER="NONE" + if [ -d /home/singularity_home/mindcontrol/.meteor/local/db ] ; then + DB_OWNER=$(stat -c %U /home/singularity_home/mindcontrol/.meteor/local/db) + fi + RESTORE_DB=0 if [ ${DB_OWNER} != $(stat -c %U $HOME) ] ; then if [ ! -d /output/mindcontrol_database ] ; then echo "Someone else owns the mongo database and there's no database dump to load from at /output/mindcontrol_database. Something's gone wrong. Exiting so we don't destroy data." >> /output/out @@ -34,15 +37,14 @@ From: mindcontrol:latest fi echo "Someone else owns the db and a database dump was found, resetting Meteor" >> /output/out meteor reset - RESTORE_DB="Yes" + RESTORE_DB=1 fi echo "Starting mindcontrol and nginx" >> /output/out nginx nohup meteor --settings /mc_settings/mc_nginx_settings.json --port $MC_PORT > /output/mindcontrol.out 2>&1 & - if [ "${RESTORE_DB}"" == "Yes" ] ; then - - # from https://superuser.com/a/809159 - export DELAY_STRING="=> Started your app." - /bin/bash -c "grep -m 1 $DELAY_STRING <(tail -f /output/mindcontrol.out)" + if [ ${RESTORE_DB} -eq 1 ] ; then + echo "a" + bash -c 'grep -m 1 "=> Started your app." <( tail -f /output/mindcontrol.out)' + echo "b" mongorestore --port=${MONGO_PORT} --drop --preserveUUID --gzip /output/mindcontrol_database/ fi diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index d30e30e..528e870 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -402,13 +402,12 @@ def write_stopfile(stopfile, workdir, group, cmd, meteor_port, run_stop=True): if run_stop: script = f"""#! /bin/bash cd {workdir.absolute()} -singularity exec instance://mindcontrol mongodump --out=/output/mindcontrol_database --port={meteor_port+1} --gzip if [ -d log/simg_out/mindcontrol_database ] ; then DATE=$(date +"%Y%m%d%H%M%S") echo "Saving previous database dump to log/simg_out/mindcontrol_database_${{DATE}}.tar.gz" tar -czf log/simg_out/mindcontrol_database_${{DATE}}.tar.gz log/simg_out/mindcontrol_database/ fi - +singularity exec instance://mindcontrol mongodump --out=/output/mindcontrol_database --port={meteor_port+1} --gzip singularity exec instance://mindcontrol mongod --dbpath=/home/${{USER}}/mindcontrol/.meteor/local/db --shutdown {cmd} echo "Waiting 30 seconds for everything to finish writing" From db21d656fde4424339905595c709ed8056b8cbe4 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 1 Aug 2018 17:11:59 -0400 Subject: [PATCH 47/60] catch startup case in startscript --- imports/docker/Singularity | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index 62b22ec..d440f31 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -30,7 +30,7 @@ From: mindcontrol:latest DB_OWNER=$(stat -c %U /home/singularity_home/mindcontrol/.meteor/local/db) fi RESTORE_DB=0 - if [ ${DB_OWNER} != $(stat -c %U $HOME) ] ; then + if [ ${DB_OWNER} != $(stat -c %U $HOME) ] && [ ${DB_OWNER} != "NONE" ] ; then if [ ! -d /output/mindcontrol_database ] ; then echo "Someone else owns the mongo database and there's no database dump to load from at /output/mindcontrol_database. Something's gone wrong. Exiting so we don't destroy data." >> /output/out exit 64 From 14a25bfa4bc18d220fb9a9ed08ef867ab7028a97 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 2 Aug 2018 10:44:02 -0400 Subject: [PATCH 48/60] fix nginx directory permissions --- start_singularity_mindcontrol.py | 1 + 1 file changed, 1 insertion(+) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 528e870..1dc3b05 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -415,6 +415,7 @@ def write_stopfile(stopfile, workdir, group, cmd, meteor_port, run_stop=True): chown -R :{group} scratch/singularity_home/mindcontrol/.meteor/local chmod -R 770 scratch/singularity_home/mindcontrol/.meteor/local chmod -R 770 log +chmod -R 770 scratch/nginx """ else: raise NotImplementedError From 2fda51670f3f71a50f4784d9dc7ede86c1b7aaba Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Thu, 2 Aug 2018 10:57:06 -0400 Subject: [PATCH 49/60] Update README.md --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d626481..49647e0 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,10 @@ git clone https://github.com/Shotgunosine/mindcontrol cd mindcontrol ``` You'll need to have a python 3 environment with access to nipype, pybids, freesurfer, and singularity. -Run start_singularity_mindcontrol.py. You can see its documentation with `python start_singularity_mindcontrol.py -h`. +Run start_singularity_mindcontrol.py. You can see its documentation with `python start_singularity_mindcontrol.py -h`. The script is now set up to take the name of a linux group as it's first argument. Any member of this group will be able to run the mindcontrol instance and access the mindcontrol files. ``` -python start_singularity_mindcontrol.py —freesurfer_dir [path to directory containing subdirectories for all the subjects] --sing_out_dir [path where mindcontrol can output files] --freesurfer +python start_singularity_mindcontrol.py [name of the group you want to won mindcontrol files] —freesurfer_dir [path to directory containing subdirectories for all the subjects] --sing_out_dir [path where mindcontrol can output files] --freesurfer ``` This command does a number of things: @@ -60,13 +60,19 @@ This command does a number of things: 3) Pulls the singularity image. 4) Starts an instance of the singularity image running with everything mounted appropriately. -Inside the image, there’s a fair bit of file copying that has to get done. It takes a while. +Inside the image, there’s a fair bit of file copying that has to get done. It takes a while, potentially quite a while if you are on a system with a slow filesystem. You can check the progress with `cat log/simg_out/out`. Once that says `Starting mindcontrol and nginx`, you can `cat log/simg_out/mindcontrol.out` to see what mindcontrol is doing. Once that says `App running at: http://localhost:2998/`, mindcontrol is all set up and running (but ignore that port number, it’s running on port 3000). Anyone who wants to see mindcontrol can then `ssh -L 3000:localhost:3000` to the server and browse to http://localhost:3000 in their browser. They’ll be prompted to login with the username and password you created way back in step 1. +Once you're done running mindcontrol you can stop it with `/bin/bash stop_mindcontrol.sh`. This command dumps the databse to log/simg_out/mindcontrol.out and compresses any previous database dump if finds there. It also fixes permissions on all of the files in the mindcontrol directory, so it may take a while to run. Killing the script before it finishes may prevent another user from being able to run your mindcontrol instance. + +The next time a user in the appropriate group would like to start it, they should just run `/bin/bash start_mindcontrol.sh`. This will start the instance and restore the database from the previously dumped data. + +Don't use the `singularity instance.start` and `singularity instance.stop` commands, as the additional permissions fixing and database restoring or dumping won't run. + ## Demo Check out the [demo](http://mindcontrol.herokuapp.com/). [This data is from the 1000 Functional Connectomes Project](http://fcon_1000.projects.nitrc.org/fcpClassic/FcpTable.html) From afbff7e63c4624730ecc10c011e756e5521fd77a Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Thu, 2 Aug 2018 11:00:13 -0400 Subject: [PATCH 50/60] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 49647e0..1ebd025 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,8 @@ The next time a user in the appropriate group would like to start it, they shoul Don't use the `singularity instance.start` and `singularity instance.stop` commands, as the additional permissions fixing and database restoring or dumping won't run. +The startscript should write `my_readme.md` to the output directory you specify with instructions on running and connecting to the mindcontrol instance you've created with all of the appropriate paths and port numbers filled in. + ## Demo Check out the [demo](http://mindcontrol.herokuapp.com/). [This data is from the 1000 Functional Connectomes Project](http://fcon_1000.projects.nitrc.org/fcpClassic/FcpTable.html) From 9cee36e333826c73ce5208f93de3c93b0c381adb Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 2 Aug 2018 11:29:51 -0400 Subject: [PATCH 51/60] point fs_dir at bids_dir if fs_dir not specified --- start_singularity_mindcontrol.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index 1dc3b05..c62a2a4 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -866,6 +866,8 @@ def create_mindcontrol_entries(output_dir, subject, stats): build_command = f"singularity build {simg_path.absolute()} shub://Shotgunosine/mindcontrol" if bids_dir is None: bids_dir = freesurfer_dir + elif freesurfer_dir is None: + freesurfer_dir = bids_dir startcmd = f"singularity instance.start -B {logdir.absolute()}:/var/log/nginx" \ + f" -B {bids_dir.absolute()}:/mc_data" \ + f" -B {freesurfer_dir.absolute()}:/mc_fs" \ From 97d58b21e9a79a98e1b26bf82fd6ab035e9cd6ba Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Tue, 7 Aug 2018 16:28:06 -0400 Subject: [PATCH 52/60] add container name --- start_singularity_mindcontrol.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index c62a2a4..b7b8f6e 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -396,7 +396,7 @@ def write_startfile(startfile, workdir, cmd): startfile.write_text(script) -def write_stopfile(stopfile, workdir, group, cmd, meteor_port, run_stop=True): +def write_stopfile(stopfile, workdir, group, cmd, meteor_port, container_name, run_stop=True): #find scratch/singularity_home ! -group {group} -exec chmod 770 {{}} \; -exec chown :{group} {{}} \; if run_stop: @@ -407,8 +407,8 @@ def write_stopfile(stopfile, workdir, group, cmd, meteor_port, run_stop=True): echo "Saving previous database dump to log/simg_out/mindcontrol_database_${{DATE}}.tar.gz" tar -czf log/simg_out/mindcontrol_database_${{DATE}}.tar.gz log/simg_out/mindcontrol_database/ fi -singularity exec instance://mindcontrol mongodump --out=/output/mindcontrol_database --port={meteor_port+1} --gzip -singularity exec instance://mindcontrol mongod --dbpath=/home/${{USER}}/mindcontrol/.meteor/local/db --shutdown +singularity exec instance://{container_name} mongodump --out=/output/mindcontrol_database --port={meteor_port+1} --gzip +singularity exec instance://{container_name} mongod --dbpath=/home/${{USER}}/mindcontrol/.meteor/local/db --shutdown {cmd} echo "Waiting 30 seconds for everything to finish writing" sleep 30 @@ -567,9 +567,10 @@ def create_mindcontrol_entries(output_dir, subject, stats): ' if needed.') parser.add_argument('group', help='Name of the group under which mindcontrol directories should be created') + parser.add_argument('container_name', + help='Name for the container. Should be unique.') parser.add_argument('--sing_out_dir', - default='.', - help='Directory to bulid singularirty image and files in.') + help='Directory to bulid singularirty image and files in. Dafaults to ./[container_name]') parser.add_argument('--custom_settings', help='Path to custom settings json') parser.add_argument('--freesurfer', action='store_true', @@ -626,7 +627,11 @@ def create_mindcontrol_entries(output_dir, subject, stats): raise Exception("This command must be run with a umask of 2, run" " 'umask 002' to set the umask then try" " this command again") - sing_out_dir = args.sing_out_dir + container_name = args.container_name + if args.sing_out_dir is None: + sing_out_dir = container_name + else: + sing_out_dir = args.sing_out_dir if args.custom_settings is not None: custom_settings = Path(args.custom_settings) @@ -647,7 +652,6 @@ def create_mindcontrol_entries(output_dir, subject, stats): layout = BIDSLayout(bids_dir.as_posix()) except ValueError as e: print("Invalid bids directory, skipping none freesurfer files. BIDS error:", e) - else: bids_dir = None @@ -861,7 +865,7 @@ def create_mindcontrol_entries(output_dir, subject, stats): f"rm /tmp/mongodb-{meteor_port + 1}.sock" ]) else: - stop_cmd = "singularity instance.stop mindcontrol" + stop_cmd = f"singularity instance.stop {container_name}" build_command = f"singularity build {simg_path.absolute()} shub://Shotgunosine/mindcontrol" if bids_dir is None: @@ -878,9 +882,9 @@ def create_mindcontrol_entries(output_dir, subject, stats): + f" -B {mcsetdir.absolute()}:/mc_settings" \ + f" -B {mc_hdir.absolute()}:/home/singularity_home" \ + f" -H {mc_hdir.absolute().as_posix() + '_'}${{USER}}:/home/${{USER}} {simg_path.absolute()}" \ - + " mindcontrol" + + f" {container_name}" write_startfile(startfile, basedir, startcmd) - write_stopfile(stopfile, basedir, mc_gnam, stop_cmd, meteor_port, allow_pidns) + write_stopfile(stopfile, basedir, mc_gnam, stop_cmd, meteor_port, container_name, allow_pidns) cmd = f"/bin/bash {startfile.absolute()}" if not args.no_server: readme_str += "## Sinularity image was built with this comand \n" From 5e03e8756f4062e2e249d4e391fffe41b33dbd3a Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Tue, 7 Aug 2018 16:30:15 -0400 Subject: [PATCH 53/60] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1ebd025..302304c 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ You'll need to have a python 3 environment with access to nipype, pybids, freesu Run start_singularity_mindcontrol.py. You can see its documentation with `python start_singularity_mindcontrol.py -h`. The script is now set up to take the name of a linux group as it's first argument. Any member of this group will be able to run the mindcontrol instance and access the mindcontrol files. ``` -python start_singularity_mindcontrol.py [name of the group you want to won mindcontrol files] —freesurfer_dir [path to directory containing subdirectories for all the subjects] --sing_out_dir [path where mindcontrol can output files] --freesurfer +python start_singularity_mindcontrol.py [name of the group you want to won mindcontrol files] [name you want to give container] —freesurfer_dir [path to directory containing subdirectories for all the subjects] --sing_out_dir [path where mindcontrol can output files] --freesurfer ``` This command does a number of things: From 80cacc701210191808264dc65b6ab3a515e5e114 Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Wed, 31 Oct 2018 14:29:55 -0400 Subject: [PATCH 54/60] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 302304c..01ce659 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ You'll need to have a python 3 environment with access to nipype, pybids, freesu Run start_singularity_mindcontrol.py. You can see its documentation with `python start_singularity_mindcontrol.py -h`. The script is now set up to take the name of a linux group as it's first argument. Any member of this group will be able to run the mindcontrol instance and access the mindcontrol files. ``` -python start_singularity_mindcontrol.py [name of the group you want to won mindcontrol files] [name you want to give container] —freesurfer_dir [path to directory containing subdirectories for all the subjects] --sing_out_dir [path where mindcontrol can output files] --freesurfer +python start_singularity_mindcontrol.py [name of the group you want to own mindcontrol files] [name you want to give container] —freesurfer_dir [path to directory containing subdirectories for all the subjects] --sing_out_dir [path where mindcontrol can output files] --freesurfer ``` This command does a number of things: From c1697505e9283e5cd2ce423c08be8d2cd556ed79 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 31 Oct 2018 16:12:11 -0400 Subject: [PATCH 55/60] explicitly create bindpoint targets --- imports/docker/Dockerfile_services | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services index 32572f1..ec2222e 100644 --- a/imports/docker/Dockerfile_services +++ b/imports/docker/Dockerfile_services @@ -69,7 +69,8 @@ RUN chown mindcontrol:mindcontrol /home/entrypoint.sh &&\ chown -R mindcontrol:mindcontrol /home/mindcontrol &&\ chmod -R a+rx /home/mindcontrol &&\ chmod 777 /var/log/nginx /var/cache/nginx /opt /etc &&\ - mkdir -p /mc_data /mc_startup_data /mc_settings /mc_fs &&\ + mkdir -p /mc_data /mc_startup_data /mc_settings /mc_fs /output /opt/settings /home/singularity_home &&\ + chmot 777 /output /opt/settings /home/singularity_home &&\ chown mindcontrol:mindcontrol /mc_data /mc_startup_data /mc_settings /mc_fs &&\ rm -f /etc/nginx/nginx.conf &&\ ln -s /opt/settings/auth.htpasswd /etc/nginx/auth.htpasswd &&\ From 7935efc252ae53788dcaabad1ece15f5b16988c8 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Wed, 31 Oct 2018 16:36:10 -0400 Subject: [PATCH 56/60] explicitly create bindpoint targets --- imports/docker/Dockerfile_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services index ec2222e..5577253 100644 --- a/imports/docker/Dockerfile_services +++ b/imports/docker/Dockerfile_services @@ -70,7 +70,7 @@ RUN chown mindcontrol:mindcontrol /home/entrypoint.sh &&\ chmod -R a+rx /home/mindcontrol &&\ chmod 777 /var/log/nginx /var/cache/nginx /opt /etc &&\ mkdir -p /mc_data /mc_startup_data /mc_settings /mc_fs /output /opt/settings /home/singularity_home &&\ - chmot 777 /output /opt/settings /home/singularity_home &&\ + chmod -R 777 /output /opt/settings /home/singularity_home &&\ chown mindcontrol:mindcontrol /mc_data /mc_startup_data /mc_settings /mc_fs &&\ rm -f /etc/nginx/nginx.conf &&\ ln -s /opt/settings/auth.htpasswd /etc/nginx/auth.htpasswd &&\ From 6b0bb4eae928857361b4e53c2ef53d3f3e36eed4 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 1 Nov 2018 13:34:04 -0400 Subject: [PATCH 57/60] take files out of /home --- imports/docker/Dockerfile_services | 9 ++++----- imports/docker/Singularity | 24 ++++++++++++------------ start_singularity_mindcontrol.py | 8 ++++---- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services index 5577253..1cdcd6a 100644 --- a/imports/docker/Dockerfile_services +++ b/imports/docker/Dockerfile_services @@ -69,8 +69,8 @@ RUN chown mindcontrol:mindcontrol /home/entrypoint.sh &&\ chown -R mindcontrol:mindcontrol /home/mindcontrol &&\ chmod -R a+rx /home/mindcontrol &&\ chmod 777 /var/log/nginx /var/cache/nginx /opt /etc &&\ - mkdir -p /mc_data /mc_startup_data /mc_settings /mc_fs /output /opt/settings /home/singularity_home &&\ - chmod -R 777 /output /opt/settings /home/singularity_home &&\ + mkdir -p /mc_data /mc_startup_data /mc_settings /mc_fs /output /opt/settings /mc_files/singularity_home &&\ + chmod -R 777 /output /opt/settings /mc_files/singularity_home &&\ chown mindcontrol:mindcontrol /mc_data /mc_startup_data /mc_settings /mc_fs &&\ rm -f /etc/nginx/nginx.conf &&\ ln -s /opt/settings/auth.htpasswd /etc/nginx/auth.htpasswd &&\ @@ -86,8 +86,7 @@ RUN cd ${MC_DIR}/mindcontrol &&\ meteor npm install --save bcrypt &&\ meteor remove meteorhacks:aggregate &&\ meteor add sakulstra:aggregate &&\ - meteor reset + meteor reset &&\ + mv /home/mindcontrol /mc_files/mindcontrol #git clone https://github.com/clowdcontrol/mindcontrol.git ${MC_DIR}/mindcontrol - - diff --git a/imports/docker/Singularity b/imports/docker/Singularity index d440f31..a2eccae 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -7,17 +7,17 @@ From: mindcontrol:latest %startscript export HOME=$(find /home/ -maxdepth 1 -writable -not -name "singularity_home") - if [ ! -d /home/singularity_home/mindcontrol ] || [ ! -d /home/singularity_home/.meteor ] || [ ! -d /home/singularity_home/.cordova ] ; then + if [ ! -d /mc_files/singularity_home/mindcontrol ] || [ ! -d /mc_files/singularity_home/.meteor ] || [ ! -d /mc_files/singularity_home/.cordova ] ; then echo "Copying meteor files into singularity_home" > /output/out - rsync -rlD /home/mindcontrol/mindcontrol /home/singularity_home/ > /output/rsync 2>&1 - chmod -R 770 /home/singularity_home/mindcontrol - echo "/home/mindcontrol/mindcontrol copied to $HOME" >> /output/out - rsync -rlD /home/mindcontrol/.meteor /home/singularity_home/ >> /output/rsync 2>&1 - chmod -R 770 /home/singularity_home/.meteor - echo "/home/mindcontrol/.meteor copied to $HOME" >> /output/out - rsync -rlD /home/mindcontrol/.cordova /home/singularity_home/ >> /output/rsync 2>&1 - chmod -R 770 /home/singularity_home/.cordova - echo "/home/mindcontrol/.cordova copied to $HOME" >> /output/out + rsync -rlD /mc_files/mindcontrol/mindcontrol /mc_files/singularity_home/ > /output/rsync 2>&1 + chmod -R 770 /mc_files/singularity_home/mindcontrol + echo "/mc_files/mindcontrol/mindcontrol copied to $HOME" >> /output/out + rsync -rlD /mc_files/mindcontrol/.meteor /mc_files/singularity_home/ >> /output/rsync 2>&1 + chmod -R 770 /mc_files/singularity_home/.meteor + echo "/mc_files/mindcontrol/.meteor copied to $HOME" >> /output/out + rsync -rlD /mc_files/mindcontrol/.cordova /mc_files/singularity_home/ >> /output/rsync 2>&1 + chmod -R 770 /mc_files/singularity_home/.cordova + echo "/mc_files/mindcontrol/.cordova copied to $HOME" >> /output/out fi cd $HOME/mindcontrol # grab proper meteor port from settings file @@ -26,8 +26,8 @@ From: mindcontrol:latest # Check if we need to reset meteor DB_OWNER="NONE" - if [ -d /home/singularity_home/mindcontrol/.meteor/local/db ] ; then - DB_OWNER=$(stat -c %U /home/singularity_home/mindcontrol/.meteor/local/db) + if [ -d /mc_files/singularity_home/mindcontrol/.meteor/local/db ] ; then + DB_OWNER=$(stat -c %U /mc_files/singularity_home/mindcontrol/.meteor/local/db) fi RESTORE_DB=0 if [ ${DB_OWNER} != $(stat -c %U $HOME) ] && [ ${DB_OWNER} != "NONE" ] ; then diff --git a/start_singularity_mindcontrol.py b/start_singularity_mindcontrol.py index b7b8f6e..0aa32fd 100644 --- a/start_singularity_mindcontrol.py +++ b/start_singularity_mindcontrol.py @@ -387,9 +387,9 @@ def write_startfile(startfile, workdir, cmd): if [ ! -d scratch/singularity_home_${{USER}} ]; then mkdir scratch/singularity_home_${{USER}} cd scratch/singularity_home_${{USER}} - ln -s ../singularity_home/.cordova - ln -s ../singularity_home/.meteor - ln -s ../singularity_home/mindcontrol + ln -s /mc_files/singularity_home/.cordova + ln -s /mc_files/singularity_home/.meteor + ln -s /mc_files/singularity_home/mindcontrol fi {cmd} """ @@ -880,7 +880,7 @@ def create_mindcontrol_entries(output_dir, subject, stats): + f" -B {nginx_scratch.absolute()}:/var/cache/nginx" \ + f" -B {simg_out.absolute()}:/output" \ + f" -B {mcsetdir.absolute()}:/mc_settings" \ - + f" -B {mc_hdir.absolute()}:/home/singularity_home" \ + + f" -B {mc_hdir.absolute()}:/mc_files/singularity_home" \ + f" -H {mc_hdir.absolute().as_posix() + '_'}${{USER}}:/home/${{USER}} {simg_path.absolute()}" \ + f" {container_name}" write_startfile(startfile, basedir, startcmd) From 0d1a360f07bb6b861c32927906c41e06363e5177 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 1 Nov 2018 13:48:44 -0400 Subject: [PATCH 58/60] fix perms --- imports/docker/Dockerfile_services | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services index 1cdcd6a..1909c3c 100644 --- a/imports/docker/Dockerfile_services +++ b/imports/docker/Dockerfile_services @@ -70,8 +70,8 @@ RUN chown mindcontrol:mindcontrol /home/entrypoint.sh &&\ chmod -R a+rx /home/mindcontrol &&\ chmod 777 /var/log/nginx /var/cache/nginx /opt /etc &&\ mkdir -p /mc_data /mc_startup_data /mc_settings /mc_fs /output /opt/settings /mc_files/singularity_home &&\ - chmod -R 777 /output /opt/settings /mc_files/singularity_home &&\ - chown mindcontrol:mindcontrol /mc_data /mc_startup_data /mc_settings /mc_fs &&\ + chmod -R 777 /output /opt/settings &&\ + chown -R mindcontrol:mindcontrol /mc_data /mc_startup_data /mc_settings /mc_fs /mc_files &&\ rm -f /etc/nginx/nginx.conf &&\ ln -s /opt/settings/auth.htpasswd /etc/nginx/auth.htpasswd &&\ ln -s /opt/settings/nginx.conf /etc/nginx/nginx.conf &&\ From 624f640a82d85d217763a26f6ece09b44a0e1829 Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 1 Nov 2018 14:37:19 -0400 Subject: [PATCH 59/60] symlink mindcontrol home to mc_files --- imports/docker/Dockerfile_services | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/imports/docker/Dockerfile_services b/imports/docker/Dockerfile_services index 1909c3c..bb198ce 100644 --- a/imports/docker/Dockerfile_services +++ b/imports/docker/Dockerfile_services @@ -62,7 +62,10 @@ ENV LC_ALL C COPY entrypoint_nginx.sh /home/entrypoint.sh #COPY ndmg_launch.sh /home/ndmg_launch.sh -RUN useradd --create-home --home-dir ${MC_DIR} mindcontrol +RUN useradd mindcontrol &&\ + mkdir -p /mc_files/mindcontrol &&\ + chown -R mindcontrol:mindcontrol mc_files &&\ + ln -s /mc_files/mindcontrol /home/mindcontrol RUN chown mindcontrol:mindcontrol /home/entrypoint.sh &&\ chmod +x /home/entrypoint.sh &&\ mkdir -p ${MC_DIR}/mindcontrol &&\ @@ -86,7 +89,6 @@ RUN cd ${MC_DIR}/mindcontrol &&\ meteor npm install --save bcrypt &&\ meteor remove meteorhacks:aggregate &&\ meteor add sakulstra:aggregate &&\ - meteor reset &&\ - mv /home/mindcontrol /mc_files/mindcontrol + meteor reset #git clone https://github.com/clowdcontrol/mindcontrol.git ${MC_DIR}/mindcontrol From 272f8de372ce8511859a2e4960b3e636764d8f9d Mon Sep 17 00:00:00 2001 From: Shotgunosine Date: Thu, 1 Nov 2018 15:33:26 -0400 Subject: [PATCH 60/60] fix setting home directory --- imports/docker/Singularity | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imports/docker/Singularity b/imports/docker/Singularity index a2eccae..8a8e6f4 100644 --- a/imports/docker/Singularity +++ b/imports/docker/Singularity @@ -6,7 +6,7 @@ From: mindcontrol:latest Mainteiner Dylan Nielson\ %startscript - export HOME=$(find /home/ -maxdepth 1 -writable -not -name "singularity_home") + export HOME=$(find /home/ -maxdepth 1 -mindepth 1 -writable -not -name "singularity_home") if [ ! -d /mc_files/singularity_home/mindcontrol ] || [ ! -d /mc_files/singularity_home/.meteor ] || [ ! -d /mc_files/singularity_home/.cordova ] ; then echo "Copying meteor files into singularity_home" > /output/out rsync -rlD /mc_files/mindcontrol/mindcontrol /mc_files/singularity_home/ > /output/rsync 2>&1