notebooks/run_fmriprepv21.0.0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Requirements\n",
    "- fmriprep singularity image called fmriprep_v21.0.0.simg in CATD-ReliabilityAnalysis/singularity_images\n",
    "- freesurfer licesnse called license.txt in CATD-ReliabilityAnalysis/singularity_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_12097/417483069.py:17: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import display, HTML\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#notebook { padding-top:0px !important; } .container { width:100% !important; } .end_space { min-height:0px !important; } </style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import shutil\n",
    "from bids import BIDSLayout\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from io import StringIO\n",
    "import re\n",
    "import nibabel as nb\n",
    "import json\n",
    "import copy\n",
    "import os\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.max_colwidth', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>\"\n",
    "    + \"#notebook { padding-top:0px !important; } \" \n",
    "    + \".container { width:100% !important; } \"\n",
    "    + \".end_space { min-height:0px !important; } \"\n",
    "    + \"</style>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_root = Path('..')\n",
    "bids_dir = project_root / 'data/bids_defaced'\n",
    "derivatives_dir = project_root / 'data/derivatives'\n",
    "swarm_cmd_dir = project_root / 'swarm/fmriprep/swarm_cmds'\n",
    "sing_img_dir = Path('../singularity_images')\n",
    "image_path = (sing_img_dir/'fmriprep_v21.0.0.simg').as_posix()\n",
    "fs_licence_path = sing_img_dir/'license.txt'\n",
    "assert fs_licence_path.exists()\n",
    "fs_licence_path = fs_licence_path.as_posix()\n",
    "\n",
    "# change this for other fmriprep runs\n",
    "#run_name = 'rn_aroma'\n",
    "#fmriprep_out = derivatives_dir / 'fmriprep' / run_name\n",
    "\n",
    "#cmd_file = swarm_cmd_dir / run_name\n",
    "#swarm_log_dir = project_root / ('swarm/fmriprep/swarm_logs' + run_name)\n",
    "jobids = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 18.2 ms, sys: 4.47 ms, total: 22.6 ms\n",
      "Wall time: 228 ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/MBDU/mid_share/fitlins_env/lib/python3.9/site-packages/bids/layout/validation.py:46: UserWarning: The ability to pass arguments to BIDSLayout that control indexing is likely to be removed in future; possibly as early as PyBIDS 0.14. This includes the `config_filename`, `ignore`, `force_index`, and `index_metadata` arguments. The recommended usage pattern is to initialize a new BIDSLayoutIndexer with these arguments, and pass it to the BIDSLayout via the `indexer` argument.\n",
      "  warnings.warn(\"The ability to pass arguments to BIDSLayout that control \"\n"
     ]
    }
   ],
   "source": [
    "database_path='/data/MBDU/mid_share/notebooks/pybids131'\n",
    "%time layout = BIDSLayout(bids_dir, database_path=database_path, ignore=[], force_index=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "subjects = layout.get(return_type='id', target='subject', suffix='T1w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def jobhist(jid):\n",
    "    jh = !jobhist {jid}\n",
    "    \n",
    "    jt = []\n",
    "    nblanks=0\n",
    "    for ll in jh:\n",
    "        if nblanks >=2 and ll != '':\n",
    "            jt.append(ll)\n",
    "        if ll == '':\n",
    "            nblanks+=1\n",
    "    jhdf = pd.read_csv(StringIO('\\n'.join(jt)), delim_whitespace=True)\n",
    "    states = [\"RUNNING\", \"COMPLETED\", \"FAILED\", \"CANCELLED\", \"TIMEOUT\"]\n",
    "    print(\"Job statuses:\")\n",
    "    for st in states:\n",
    "        stdf = jhdf.query('State == @st')\n",
    "        print(f\"{st}: {len(stdf)}\")\n",
    "    return jhdf\n",
    "\n",
    "\n",
    "def subj_status(jhdf, subjects, statuses):\n",
    "    if len(jhdf) != len(subjects):\n",
    "        raise ValueError('jhdf and subjects should have the same length')\n",
    "    status_runs = np.array(jhdf.loc[jhdf.State.isin(statuses), 'Jobid'].str.split('_').str[1]).astype(int)\n",
    "    status_subjs = np.array(subjects)[status_runs]\n",
    "    return status_subjs\n",
    "\n",
    "def put_back(orig_run_name, run_name, project_root, worked):\n",
    "    bids_dir = project_root / 'data/bids'\n",
    "    derivatives_dir = project_root / 'data/derivatives'\n",
    "    orig_fmriprep_out = derivatives_dir / 'fmriprep' / orig_run_name\n",
    "    rr_fmriprep_out = derivatives_dir / 'fmriprep' / run_name\n",
    "    orig_fails = derivatives_dir / 'fmriprep' / (orig_run_name + '_failure_archive')\n",
    "    assert orig_fails != rr_fmriprep_out\n",
    "    orig_fails.mkdir(exist_ok=True)\n",
    "    sing_cmds = []\n",
    "    for sid in worked:\n",
    "        rr_out_dir = rr_fmriprep_out / f'sub-{sid}'\n",
    "        orig_out_dir = orig_fmriprep_out / f'sub-{sid}'\n",
    "        orig_fail_dir = orig_fails / f'sub-{sid}'\n",
    "        if orig_out_dir.exists():\n",
    "            shutil.move(orig_out_dir, orig_fail_dir)\n",
    "        assert ~orig_out_dir.exists()\n",
    "        shutil.move(rr_out_dir, orig_out_dir)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#known_hard_subjects = ['24028', '22477']\n",
    "known_hard_subjects = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Move successfull runs back in to directory for original run\n",
    "# orig_run_name = 'rn_aroma_20190910'\n",
    "# run_name = 'rn_aroma_20190910_hard'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_completers(run_name, input_subjects):\n",
    "    jhdf = jobhist(jobids[run_name])\n",
    "    return subj_status(jhdf, input_subjects, statuses=['COMPLETED'])\n",
    "\n",
    "def get_fails(run_name, input_subjects):\n",
    "    jhdf = jobhist(jobids[run_name])\n",
    "    return subj_status(jhdf, input_subjects, statuses=['FAILED', 'CANCELLED', 'TIMEOUT'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run with fmripep 21.0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_sing_cmd(subj, image_path, bids_dir, fs_license_path, subj_out_dir,\n",
    "                  longitudinal=False, aroma=True, dummy_scans=4,\n",
    "                  save_work_directory=False, verbose=False, melodic_dims=None,\n",
    "                  echo_idx=None, task=None, anat_only=False, bids_filter_file=None,\n",
    "                  output_spaces=None, fs_subjects_dir=None, mnitobold=False, tedana=False,\n",
    "                  rest_infos=None):\n",
    "    if output_spaces is None:\n",
    "        output_spaces = ['MNI152NLin2009cAsym:res-2', 'fsaverage']\n",
    "    if tedana and (rest_infos is None):\n",
    "        raise ValueError(\"If you want tedana commands created, you must past a list of dictionaries of resting scan info.\")\n",
    "    elif tedana:\n",
    "        for ri in rest_infos:\n",
    "            if ri['subject'] != str(subj):\n",
    "                raise ValueError(f\"Rest infos should match the subj input. subj={subj}, rest_info['subject'] = {ri['subject']}.\")\n",
    "    output_spaces = ' '.join(output_spaces)\n",
    "    #    singularity run --cleanenv {image_path} {bids_dir} \\\n",
    "    #    singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep {image_path} {bids_dir} \\\n",
    "\n",
    "    sing_cmd = f'export TMPDIR=/lscratch/$SLURM_JOB_ID && \\\n",
    "    export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&\\\n",
    "    mkdir -p $TMPDIR/out && \\\n",
    "    mkdir -p $TMPDIR/wrk && \\\n",
    "    singularity run --cleanenv {image_path} {bids_dir} \\\n",
    "    $TMPDIR/out participant \\\n",
    "    --participant_label {subj} \\\n",
    "    -w $TMPDIR/wrk \\\n",
    "    --nprocs $SLURM_CPUS_PER_TASK \\\n",
    "    --mem_mb $SLURM_MEM_PER_NODE \\\n",
    "    --fs-license-file {fs_licence_path} \\\n",
    "    --output-spaces {output_spaces} \\\n",
    "    --dummy-scans {dummy_scans} \\\n",
    "    --me-output-echos'\n",
    "    if anat_only:\n",
    "        sing_cmd += \" --anat-only\" \n",
    "    if bids_filter_file:\n",
    "        sing_cmd += f\" --bids-filter-file {bids_filter_file.as_posix()}\"\n",
    "    if longitudinal:\n",
    "        sing_cmd += \" --longitudinal\"\n",
    "    if aroma:\n",
    "        sing_cmd += \" --use-aroma --error-on-aroma-warnings\"\n",
    "        if melodic_dims is not None:\n",
    "            sing_cmd += f\" --aroma-melodic-dimensionality {melodic_dims}\"\n",
    "    if echo_idx is not None:\n",
    "        sing_cmd += f\" --echo-idx {echo_idx}\"\n",
    "    if task is not None:\n",
    "        sing_cmd += f\" --task-id {task}\"\n",
    "    if verbose:\n",
    "        sing_cmd += \" -vvv\"\n",
    "    sing_cmd += '; '\n",
    "    sing_cmd += \"FPSTATUS=$? ; \" # Save the exit status of fmri prep, but allow the rsync to run so we can see the log\n",
    "    statuses = ['FPSTATUS']\n",
    "    \n",
    "    if tedana:\n",
    "        # make collect_fmriprep command\n",
    "        cf_cmd =  f'singularity exec --cleanenv \\\n",
    "        {image_path} python /data/MBDU/mid_share/code/collect_fmriprep.py \\\n",
    "        -i $TMPDIR/out/fmriprep \\\n",
    "        -o $TMPDIR/wrk/fmriprep_wf \\\n",
    "        -s {subj}; \\\n",
    "        CFSTATUS=$? ; '\n",
    "        statuses.append('CFSTATUS')\n",
    "        \n",
    "        # one command per resting state sequence\n",
    "        td_cmds = []\n",
    "        for rix, ri in enumerate(rest_infos):\n",
    "            d = ' '.join([f'$TMPDIR/out/fmriprep/{pp}' for pp in ri['preproc_names']])\n",
    "            e = ' '.join([str(et) for et in ri['echo_times']])\n",
    "            out_dir = f'$TMPDIR/out/tedana/sub-{ri[\"subject\"]}/ses-{ri[\"session\"]}'\n",
    "            prefix = ri['preproc_names'][0].split('/')[-1].split('_echo')[0]\n",
    "            gscontrol = 'gsr mir'\n",
    "            td_cmd = f'mkdir -p {out_dir}; tedana -d {d} -e {e} --prefix {prefix} --out-dir {out_dir} --gscontrol {gscontrol}; \\\n",
    "            TD{rix}STATUS=$? ; '\n",
    "            statuses.append(f'TD{rix}STATUS')\n",
    "            td_cmds.append(td_cmd)\n",
    "            \n",
    "        sing_cmd += cf_cmd + ' '.join(td_cmds)\n",
    "    \n",
    "    if not mnitobold:\n",
    "        sing_cmd += f\" rsync -ach $TMPDIR/out {subj_out_dir} ;\"\n",
    "        if save_work_directory:\n",
    "            subj_work_dir = subj_out_dir / \"wrk\"\n",
    "            sing_cmd += f\" rsync -ach $TMPDIR/wrk {subj_work_dir} ; \"\n",
    "            sing_cmd += f\" chown -R :MBDU {subj_work_dir} ; \"\n",
    "\n",
    "        sing_cmd += f\" chown -R :MBDU {subj_out_dir} ; \"\n",
    "    else:\n",
    "        sing_cmd += 'mkdir $TMPDIR/mnitobold; '\n",
    "        sing_cmd += f' singularity exec --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep \\\n",
    "                    {image_path} /data/MBDU/midla/notebooks/code/run_mnitobold.sh '\n",
    "        sing_cmd += ' $TMPDIR $TMPDIR/mnitobold '\n",
    "        sing_cmd += ' /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_T1w.nii.gz /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_desc-carpet_dseg.nii.gz'\n",
    "        sing_cmd += ' --omp-nthreads=$SLURM_CPUS_PER_TASK --mem-gb=$SLURM_MEM_PER_NODE ;'\n",
    "        sing_cmd += \" STATUS=$? ; \" # Save the exit status of fmri prep, but allow the rsync to run so we can see the log\n",
    "        sing_cmd += f' rsync -ach $TMPDIR/mnitobold/ /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-{subj} ;'\n",
    "        sing_cmd += f' chown -R :MBDU /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-{subj} ;'\n",
    "    \n",
    "    status_cmds = ' ' + ' ; '.join([f'echo \"{status} is ${status}\"' for status in statuses]) + ' ; '\n",
    "    status_cmds += ' ' + ' && '.join([f'(exit ${status})' for status in statuses])\n",
    "    sing_cmd += status_cmds # exit a subshell with the saved status so that hpc correctly identifies it as a success or failure\n",
    "\n",
    "    return sing_cmd\n",
    "\n",
    "\n",
    "def write_swarm_file(run_name, project_root, image_path,\n",
    "                     fs_license_path, subjects, layout=None, \n",
    "                     sing_sess=False, defaced=True,\n",
    "                     update=False, bids_filter=None, multiecho_masking_test=False,\n",
    "                     fs_subjects_run=None, **kwargs):\n",
    "    if defaced:\n",
    "        bids_dir = project_root / 'data/bids_defaced'\n",
    "    elif multiecho_masking_test:\n",
    "        bids_dir = project_root / 'data/multiecho_masking_test'\n",
    "    else:\n",
    "        bids_dir = project_root / 'data/bids'\n",
    "    derivatives_dir = project_root / 'data/derivatives'\n",
    "    swarm_cmd_dir = project_root / 'swarm/fmriprep/swarm_cmds'\n",
    "    swarm_cmd_dir.mkdir(parents=True, exist_ok=True)\n",
    "    fmriprep_out = derivatives_dir / 'fmriprep' / run_name\n",
    "    if fs_subjects_run is not None:\n",
    "        fs_base_dir = derivatives_dir / fs_subjects_run\n",
    "    cmd_file = swarm_cmd_dir / run_name\n",
    "    swarm_log_dir = project_root / 'swarm/fmriprep/swarm_logs' / run_name\n",
    "    if (bids_filter is not None) and not sing_sess:\n",
    "        fmriprep_out.mkdir(parents=True, exist_ok=True)\n",
    "        bids_filter_file = fmriprep_out / 'bids_filter'\n",
    "        bids_filter_file.write_text(json.dumps(bids_filter, indent=2))\n",
    "        kwargs['bids_filter_file'] = bids_filter_file\n",
    "    sing_cmds = []\n",
    "    old_subjs = []\n",
    "    for sid in subjects:\n",
    "        if fs_subjects_run is not None:\n",
    "            fs_subjects_dir = fs_base_dir / f'sub-{sid}/out/freesurfer'\n",
    "        else:\n",
    "            fs_subjects_dir = None\n",
    "        if sing_sess:\n",
    "            if layout is None:\n",
    "                raise ValueError(\"Must pass a layout if you're running single session.\")\n",
    "            if bids_filter is None:\n",
    "                bids_filter = {\n",
    "                               't1w': {'datatype': 'anat', 'suffix': 'T1w'},\n",
    "                              }\n",
    "            sessions = layout.get(return_type='id', target='session', subject=sid, **bids_filter['t1w'])\n",
    "            \n",
    "            for sesid in sessions:\n",
    "                ses_out_dir = fmriprep_out / f'sub-{sid}/ses-{sesid}'\n",
    "                ses_out_dir.mkdir(parents=True, exist_ok=True)\n",
    "                ses_filter = copy.deepcopy(bids_filter)\n",
    "                for modality in ses_filter.keys():\n",
    "                    ses_filter[modality]['session'] = sesid\n",
    "                ses_filter_file = ses_out_dir / f'sub-{sid}_ses-{sesid}.filter'\n",
    "                ses_filter_file.write_text(json.dumps(ses_filter, indent=2))\n",
    "                kwargs['bids_filter_file'] = ses_filter_file\n",
    "                sing_cmd = make_sing_cmd(sid, image_path, bids_dir, fs_licence_path,\n",
    "                                         ses_out_dir,fs_subjects_dir=fs_subjects_dir,\n",
    "                                         **kwargs)\n",
    "                sing_cmds.append(sing_cmd)\n",
    "        else:\n",
    "            subj_out_dir = fmriprep_out / f'sub-{sid}'\n",
    "            if update and (subj_out_dir / f'out/fmriprep/sub-{sid}.html').exists():\n",
    "                print(f\"Skipping {sid} because results for the subject exist.\")\n",
    "                old_subjs.append(sid)\n",
    "            else:\n",
    "                subj_out_dir.mkdir(parents=True, exist_ok=True)\n",
    "                # TODO: make sure the ones we think failed, actually failed\n",
    "                # get needed tedana inputs\n",
    "                if kwargs.get('tedana', False):\n",
    "                    rest_scans = sorted((bids_dir / f'sub-{sid}').glob('ses-*/func/*rest*echo-1*.nii.gz'))\n",
    "                    rest_infos = []\n",
    "                    for rs in rest_scans:\n",
    "                        rest_info = layout.parse_file_entities(rs.as_posix())\n",
    "                        rest_info['echos'] = []\n",
    "                        rest_info['echo_times'] = []\n",
    "                        rest_info['preproc_names'] = []\n",
    "                        rs_parent = rs.parent\n",
    "                        echos = sorted(rs_parent.glob(rs.parts[-1].replace('echo-1', 'echo-*')))\n",
    "                        for echo in echos:\n",
    "                            echo_info = layout.parse_file_entities(echo.as_posix())\n",
    "                            rest_info['echos'].append(echo_info['echo'])\n",
    "                            echo_time = json.loads(Path(echo.as_posix().replace('.nii.gz', '.json')).read_text())['EchoTime']\n",
    "                            rest_info['echo_times'].append(echo_time*1000)\n",
    "                            preproc_name = '/'.join(list(echo.parts[-4:-1]) + [echo.parts[-1].replace('_bold', '_space-scanner_desc-partialPreproc_bold')])\n",
    "                            rest_info['preproc_names'].append(preproc_name)\n",
    "                        rest_infos.append(rest_info)\n",
    "                    kwargs['rest_infos'] = rest_infos\n",
    "                sing_cmd = make_sing_cmd(sid, image_path, bids_dir, fs_licence_path, subj_out_dir,\n",
    "                                         fs_subjects_dir=fs_subjects_dir, **kwargs)\n",
    "                sing_cmds.append(sing_cmd)\n",
    "\n",
    "    cmd_file = Path(cmd_file.as_posix())\n",
    "    cmd_file.write_text('\\n'.join(sing_cmds))\n",
    "\n",
    "    tmp = cmd_file.read_text().split('\\n')\n",
    "    for i, tt in enumerate(tmp):\n",
    "        print(tt)\n",
    "        if i > 5:\n",
    "            break\n",
    "    if not sing_sess:\n",
    "        assert len(tmp) == (len(subjects) - len(old_subjs))\n",
    "    \n",
    "    return cmd_file, swarm_log_dir, tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Code below is messy, but reflects running fmriprep and debugging issues to get as many participants as possible run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#run_name = 'fmriprep_v21.0.0'\n",
    "#subjects = [20900, 22228, 22293, 22686, 22698, 22892, 23495, 23549, 24065, 23924, 23867, 23783, 23674, 23550]\n",
    "run_name = 'fmriprep_v21.0.0_fmapaffinefix'\n",
    "#subjects = [21250, 23399] #23399 failed, testing 21250 where files are not modified to see if it works\n",
    "subjects = [23399,23565,23649,23674,23772,23777,23809,23859,23919,23927,23937]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'known_hard_subjects' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [60]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m easy_subjects \u001b[38;5;241m=\u001b[39m [ss \u001b[38;5;28;01mfor\u001b[39;00m ss \u001b[38;5;129;01min\u001b[39;00m subjects \u001b[38;5;28;01mif\u001b[39;00m ss \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m known_hard_subjects]\n",
      "Input \u001b[0;32mIn [60]\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[0;32m----> 1\u001b[0m easy_subjects \u001b[38;5;241m=\u001b[39m [ss \u001b[38;5;28;01mfor\u001b[39;00m ss \u001b[38;5;129;01min\u001b[39;00m subjects \u001b[38;5;28;01mif\u001b[39;00m ss \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m \u001b[43mknown_hard_subjects\u001b[49m]\n",
      "\u001b[0;31mNameError\u001b[0m: name 'known_hard_subjects' is not defined"
     ]
    }
   ],
   "source": [
    "easy_subjects = [ss for ss in subjects if ss not in known_hard_subjects]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v21.0.0.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23399     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4     --me-output-echos --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/bids_filter --use-aroma --error-on-aroma-warnings -vvv; FPSTATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23399 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23399 ;  echo \"FPSTATUS is $FPSTATUS\" ;  (exit $FPSTATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v21.0.0.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23565     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4     --me-output-echos --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/bids_filter --use-aroma --error-on-aroma-warnings -vvv; FPSTATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23565 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23565 ;  echo \"FPSTATUS is $FPSTATUS\" ;  (exit $FPSTATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v21.0.0.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23649     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4     --me-output-echos --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/bids_filter --use-aroma --error-on-aroma-warnings -vvv; FPSTATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23649 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23649 ;  echo \"FPSTATUS is $FPSTATUS\" ;  (exit $FPSTATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v21.0.0.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23674     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4     --me-output-echos --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/bids_filter --use-aroma --error-on-aroma-warnings -vvv; FPSTATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23674 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23674 ;  echo \"FPSTATUS is $FPSTATUS\" ;  (exit $FPSTATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v21.0.0.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23772     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4     --me-output-echos --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/bids_filter --use-aroma --error-on-aroma-warnings -vvv; FPSTATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23772 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23772 ;  echo \"FPSTATUS is $FPSTATUS\" ;  (exit $FPSTATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v21.0.0.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23777     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4     --me-output-echos --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/bids_filter --use-aroma --error-on-aroma-warnings -vvv; FPSTATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23777 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23777 ;  echo \"FPSTATUS is $FPSTATUS\" ;  (exit $FPSTATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v21.0.0.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23809     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4     --me-output-echos --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/bids_filter --use-aroma --error-on-aroma-warnings -vvv; FPSTATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23809 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23809 ;  echo \"FPSTATUS is $FPSTATUS\" ;  (exit $FPSTATUS)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprep_v21.0.0_fmapaffinefix',\n",
       " PosixPath('/data/MBDU/mid_share/swarm/fmriprep/swarm_cmds/fmriprep_v21.0.0_fmapaffinefix'),\n",
       " PosixPath('/data/MBDU/mid_share/swarm/fmriprep/swarm_logs/fmriprep_v21.0.0_fmapaffinefix'))"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  subjects,\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=False, aroma=True, tedana=False)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 247 -t 56 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name} --sbatch --constraint=cpu56\n",
    "# jobids[run_name] = jobids[run_name][0]\n",
    "# jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[23399, 23565, 23649, 23674, 23772, 23777, 23809, 23859, 23919, 23927, 23937]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subjects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/data/MBDU/mid_share/swarm/fmriprep/swarm_cmds/fmriprep_v21.0.0_fmapaffinefix')"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cmd_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'35895179'"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 120 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "#manually modified /data/MBDU/mid_share/swarm/fmriprep/swarm_cmds/fmriprep_v21.0.0_fmapaffinefix to take out /gs4 had some issues with that\n",
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:100 -g 120 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get running failed_jobids with memory overruns\n",
    "jhdf = jobhist(19602308)\n",
    "\n",
    "failed_jobids = jhdf.loc[jhdf['State'] == 'RUNNING']\n",
    "failed_jobids = failed_jobids.loc[failed_jobids['Jobid'] != '19461934_74']\n",
    "failed_jobids = failed_jobids.Jobid\n",
    "\n",
    "for ji in failed_jobids:\n",
    "    !scancel {ji}\n",
    "\n",
    "statuses=['FAILED', 'CANCELLED', 'TIMEOUT']\n",
    "if len(jhdf) != len(subjects):\n",
    "    raise ValueError('jhdf and subjects should have the same length')\n",
    "status_runs = np.array(jhdf.loc[jhdf.State.isin(statuses), 'Jobid'].str.split('_').str[1]).astype(int)\n",
    "status_subjs = np.array(subjects)[status_runs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job statuses:\n",
      "RUNNING: 0\n",
      "COMPLETED: 7\n",
      "FAILED: 0\n",
      "CANCELLED: 0\n",
      "TIMEOUT: 4\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Jobid</th>\n",
       "      <th>Partition</th>\n",
       "      <th>State</th>\n",
       "      <th>Nodes</th>\n",
       "      <th>CPUs</th>\n",
       "      <th>Walltime</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>MemReq</th>\n",
       "      <th>MemUsed</th>\n",
       "      <th>Nodelist</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>35895179_0</td>\n",
       "      <td>norm</td>\n",
       "      <td>COMPLETED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>08:53:12</td>\n",
       "      <td>120.0GB/node</td>\n",
       "      <td>27.4GB</td>\n",
       "      <td>cn0900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35895179_1</td>\n",
       "      <td>norm</td>\n",
       "      <td>COMPLETED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>09:11:21</td>\n",
       "      <td>120.0GB/node</td>\n",
       "      <td>29.4GB</td>\n",
       "      <td>cn1014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35895179_2</td>\n",
       "      <td>norm</td>\n",
       "      <td>COMPLETED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>08:00:04</td>\n",
       "      <td>120.0GB/node</td>\n",
       "      <td>27.7GB</td>\n",
       "      <td>cn3098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35895179_3</td>\n",
       "      <td>norm</td>\n",
       "      <td>COMPLETED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>08:06:18</td>\n",
       "      <td>120.0GB/node</td>\n",
       "      <td>31.5GB</td>\n",
       "      <td>cn3150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35895179_4</td>\n",
       "      <td>norm</td>\n",
       "      <td>COMPLETED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>08:30:20</td>\n",
       "      <td>120.0GB/node</td>\n",
       "      <td>30.8GB</td>\n",
       "      <td>cn3159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>35895179_5</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>2-00:00:03</td>\n",
       "      <td>120.0GB/node</td>\n",
       "      <td>35.8GB</td>\n",
       "      <td>cn3238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>35895179_6</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>2-00:00:03</td>\n",
       "      <td>120.0GB/node</td>\n",
       "      <td>36.1GB</td>\n",
       "      <td>cn3351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>35895179_7</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>2-00:00:03</td>\n",
       "      <td>120.0GB/node</td>\n",
       "      <td>33.3GB</td>\n",
       "      <td>cn3376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>35895179_8</td>\n",
       "      <td>norm</td>\n",
       "      <td>COMPLETED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>07:50:43</td>\n",
       "      <td>120.0GB/node</td>\n",
       "      <td>20.2GB</td>\n",
       "      <td>cn3382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>35895179_9</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>2-00:00:03</td>\n",
       "      <td>120.0GB/node</td>\n",
       "      <td>38.0GB</td>\n",
       "      <td>cn3396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>35895179_10</td>\n",
       "      <td>norm</td>\n",
       "      <td>COMPLETED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>07:25:48</td>\n",
       "      <td>120.0GB/node</td>\n",
       "      <td>21.3GB</td>\n",
       "      <td>cn3402</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Jobid Partition      State  Nodes  CPUs    Walltime     Runtime        MemReq MemUsed Nodelist\n",
       "0    35895179_0      norm  COMPLETED      1    24  2-00:00:00    08:53:12  120.0GB/node  27.4GB   cn0900\n",
       "1    35895179_1      norm  COMPLETED      1    24  2-00:00:00    09:11:21  120.0GB/node  29.4GB   cn1014\n",
       "2    35895179_2      norm  COMPLETED      1    24  2-00:00:00    08:00:04  120.0GB/node  27.7GB   cn3098\n",
       "3    35895179_3      norm  COMPLETED      1    24  2-00:00:00    08:06:18  120.0GB/node  31.5GB   cn3150\n",
       "4    35895179_4      norm  COMPLETED      1    24  2-00:00:00    08:30:20  120.0GB/node  30.8GB   cn3159\n",
       "5    35895179_5      norm    TIMEOUT      1    24  2-00:00:00  2-00:00:03  120.0GB/node  35.8GB   cn3238\n",
       "6    35895179_6      norm    TIMEOUT      1    24  2-00:00:00  2-00:00:03  120.0GB/node  36.1GB   cn3351\n",
       "7    35895179_7      norm    TIMEOUT      1    24  2-00:00:00  2-00:00:03  120.0GB/node  33.3GB   cn3376\n",
       "8    35895179_8      norm  COMPLETED      1    24  2-00:00:00    07:50:43  120.0GB/node  20.2GB   cn3382\n",
       "9    35895179_9      norm    TIMEOUT      1    24  2-00:00:00  2-00:00:03  120.0GB/node  38.0GB   cn3396\n",
       "10  35895179_10      norm  COMPLETED      1    24  2-00:00:00    07:25:48  120.0GB/node  21.3GB   cn3402"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#jobhist(19602308)\n",
    "#jobhist(35832400)\n",
    "#jobhist(35838388)\n",
    "#jobhist(35868168)\n",
    "jobhist(35895179)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23777, 23809, 23859, 23927)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#subjects[72],subjects[74]\n",
    "subjects[5],subjects[6],subjects[7],subjects[9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v21.0.0.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23777     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4     --me-output-echos --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/bids_filter --use-aroma --error-on-aroma-warnings -vvv; FPSTATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23777 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23777 ;  echo \"FPSTATUS is $FPSTATUS\" ;  (exit $FPSTATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v21.0.0.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23809     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4     --me-output-echos --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/bids_filter --use-aroma --error-on-aroma-warnings -vvv; FPSTATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23809 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23809 ;  echo \"FPSTATUS is $FPSTATUS\" ;  (exit $FPSTATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v21.0.0.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23859     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4     --me-output-echos --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/bids_filter --use-aroma --error-on-aroma-warnings -vvv; FPSTATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23859 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23859 ;  echo \"FPSTATUS is $FPSTATUS\" ;  (exit $FPSTATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v21.0.0.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23927     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4     --me-output-echos --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/bids_filter --use-aroma --error-on-aroma-warnings -vvv; FPSTATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23927 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0_fmapaffinefix/sub-23927 ;  echo \"FPSTATUS is $FPSTATUS\" ;  (exit $FPSTATUS)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprep_v21.0.0_fmapaffinefix',\n",
       " PosixPath('/data/MBDU/mid_share/swarm/fmriprep/swarm_cmds/fmriprep_v21.0.0_fmapaffinefix'),\n",
       " PosixPath('/data/MBDU/mid_share/swarm/fmriprep/swarm_logs/fmriprep_v21.0.0_fmapaffinefix'))"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_name = 'fmriprep_v21.0.0_fmapaffinefix'\n",
    "#these subjects timed out after 2 days, also disk storage\n",
    "#subjects = [23399, 23565, 23649, 23674, 23772, 23777, 23809, 23859, 23919, 23927, 23937]\n",
    "subjects = [23777,23809,23859,23927]\n",
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  subjects,\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=False, aroma=True)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'36071960'"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:1000 -g 200 -t 30 --module singularity,webproxy --time 96:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job statuses:\n",
      "RUNNING: 0\n",
      "COMPLETED: 4\n",
      "FAILED: 0\n",
      "CANCELLED: 0\n",
      "TIMEOUT: 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Jobid</th>\n",
       "      <th>Partition</th>\n",
       "      <th>State</th>\n",
       "      <th>Nodes</th>\n",
       "      <th>CPUs</th>\n",
       "      <th>Walltime</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>MemReq</th>\n",
       "      <th>MemUsed</th>\n",
       "      <th>Nodelist</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36071960_0</td>\n",
       "      <td>norm</td>\n",
       "      <td>COMPLETED</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>4-00:00:00</td>\n",
       "      <td>07:53:53</td>\n",
       "      <td>200.0GB/node</td>\n",
       "      <td>36.8GB</td>\n",
       "      <td>cn3630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>36071960_1</td>\n",
       "      <td>norm</td>\n",
       "      <td>COMPLETED</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>4-00:00:00</td>\n",
       "      <td>10:07:42</td>\n",
       "      <td>200.0GB/node</td>\n",
       "      <td>36.4GB</td>\n",
       "      <td>cn3631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36071960_2</td>\n",
       "      <td>norm</td>\n",
       "      <td>COMPLETED</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>4-00:00:00</td>\n",
       "      <td>14:59:16</td>\n",
       "      <td>200.0GB/node</td>\n",
       "      <td>33.9GB</td>\n",
       "      <td>cn3624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>36071960_3</td>\n",
       "      <td>norm</td>\n",
       "      <td>COMPLETED</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>4-00:00:00</td>\n",
       "      <td>08:21:45</td>\n",
       "      <td>200.0GB/node</td>\n",
       "      <td>33.1GB</td>\n",
       "      <td>cn3625</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Jobid Partition      State  Nodes  CPUs    Walltime   Runtime        MemReq MemUsed Nodelist\n",
       "0  36071960_0      norm  COMPLETED      1    30  4-00:00:00  07:53:53  200.0GB/node  36.8GB   cn3630\n",
       "1  36071960_1      norm  COMPLETED      1    30  4-00:00:00  10:07:42  200.0GB/node  36.4GB   cn3631\n",
       "2  36071960_2      norm  COMPLETED      1    30  4-00:00:00  14:59:16  200.0GB/node  33.9GB   cn3624\n",
       "3  36071960_3      norm  COMPLETED      1    30  4-00:00:00  08:21:45  200.0GB/node  33.1GB   cn3625"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobhist(36071960)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Input \u001b[0;32mIn [71]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m run_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfmriprepv20.2.2_fails_2\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m      2\u001b[0m bids_filter \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m      3\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mt1w\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m      4\u001b[0m                     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdatatype\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124manat\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m      5\u001b[0m                     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mreconstruction\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprenorm\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m      6\u001b[0m                     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msuffix\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mT1w\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m      7\u001b[0m                 }}\n\u001b[1;32m      8\u001b[0m cmd_file, swarm_log_dir, _cmds \u001b[38;5;241m=\u001b[39m write_swarm_file(run_name,\n\u001b[1;32m      9\u001b[0m                                                   project_root,\n\u001b[1;32m     10\u001b[0m                                                   image_path,\n\u001b[1;32m     11\u001b[0m                                                   fs_licence_path,\n\u001b[0;32m---> 12\u001b[0m                                                   [\u001b[43msubjects\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m72\u001b[39;49m\u001b[43m]\u001b[49m,subjects[\u001b[38;5;241m74\u001b[39m]],\n\u001b[1;32m     13\u001b[0m                                                   layout\u001b[38;5;241m=\u001b[39mlayout,\n\u001b[1;32m     14\u001b[0m                                                   sing_sess\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m     15\u001b[0m                                                   save_work_directory\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m     16\u001b[0m                                                   verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m     17\u001b[0m                                                   anat_only\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m     18\u001b[0m                                                   bids_filter\u001b[38;5;241m=\u001b[39mbids_filter,\n\u001b[1;32m     19\u001b[0m                                                   longitudinal\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, aroma\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m     21\u001b[0m run_name, cmd_file, swarm_log_dir\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "run_name = 'fmriprepv20.2.2_fails_2'\n",
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  [subjects[72],subjects[74]],\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=False, aroma=True)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "disk_fail_subjects = ['sub-23490',\n",
    " 'sub-23513',\n",
    " 'sub-23544',\n",
    " 'sub-23550',\n",
    " 'sub-23554',\n",
    " 'sub-23565',\n",
    " 'sub-23574',\n",
    " 'sub-23593',\n",
    " 'sub-23607',\n",
    " 'sub-23625',\n",
    " 'sub-23641',\n",
    " 'sub-23672',\n",
    " 'sub-23745',\n",
    " 'sub-23746',\n",
    " 'sub-23767',\n",
    " 'sub-23772',\n",
    " 'sub-23783',\n",
    " 'sub-23809',\n",
    " 'sub-23810',\n",
    " 'sub-23814',\n",
    " 'sub-23830',\n",
    " 'sub-23859',\n",
    " 'sub-23867',\n",
    " 'sub-23927',\n",
    " 'sub-23951',\n",
    " 'sub-23952',\n",
    " 'sub-23962',\n",
    " 'sub-23988',\n",
    " 'sub-24028',\n",
    " 'sub-24054',\n",
    " 'sub-24059',\n",
    " 'sub-24112']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23399     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23399 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23399 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23428     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23428 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23428 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23457     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23457 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23457 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23458     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23458 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23458 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23490     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23490 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23490 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23495     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23495 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23495 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23501     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23501 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem/sub-23501 ; \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.2.2_lowmem',\n",
       " PosixPath('/data/MBDU/mid_share/swarm/fmriprep/swarm_cmds/fmriprepv20.2.2_lowmem'),\n",
       " PosixPath('/data/MBDU/mid_share/swarm/fmriprep/swarm_logs/fmriprepv20.2.2_lowmem'))"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_name = 'fmriprepv20.2.2_lowmem'\n",
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  subjects[32:],\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=False, aroma=True)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/data/MBDU/mid_share/swarm/fmriprep/swarm_logs/fmriprepv20.2.2_lowmem')"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'20067587'"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 65 -t 25 --module singularity,webproxy --time 24:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'fmriprepv20.2.2_lowmem'"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job statuses:\n",
      "RUNNING: 0\n",
      "COMPLETED: 157\n",
      "FAILED: 0\n",
      "CANCELLED: 0\n",
      "TIMEOUT: 13\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "170"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#lowmem\n",
    "len(jobhist(20067587))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job statuses:\n",
      "RUNNING: 0\n",
      "COMPLETED: 157\n",
      "FAILED: 0\n",
      "CANCELLED: 0\n",
      "TIMEOUT: 13\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Jobid</th>\n",
       "      <th>Partition</th>\n",
       "      <th>State</th>\n",
       "      <th>Nodes</th>\n",
       "      <th>CPUs</th>\n",
       "      <th>Walltime</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>MemReq</th>\n",
       "      <th>MemUsed</th>\n",
       "      <th>Nodelist</th>\n",
       "      <th>subjects</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20067587_1</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:26</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>60.7GB</td>\n",
       "      <td>cn0744</td>\n",
       "      <td>23428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>20067587_11</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:25</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>65.0GB</td>\n",
       "      <td>cn0331</td>\n",
       "      <td>23520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>20067587_13</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:12</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>59.6GB</td>\n",
       "      <td>cn0156</td>\n",
       "      <td>23540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>20067587_17</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:07</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>65.0GB</td>\n",
       "      <td>cn2007</td>\n",
       "      <td>23549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>20067587_23</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:33</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>64.2GB</td>\n",
       "      <td>cn0145</td>\n",
       "      <td>23564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>20067587_34</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:28</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>65.0GB</td>\n",
       "      <td>cn0096</td>\n",
       "      <td>23638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>20067587_40</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:28</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>61.1GB</td>\n",
       "      <td>cn0641</td>\n",
       "      <td>23652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>20067587_42</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:28</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>61.1GB</td>\n",
       "      <td>cn0775</td>\n",
       "      <td>23660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>20067587_52</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:04</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>60.8GB</td>\n",
       "      <td>cn0208</td>\n",
       "      <td>23732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>20067587_68</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:26</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>35.0GB</td>\n",
       "      <td>cn1697</td>\n",
       "      <td>23798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>20067587_70</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:26</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>65.0GB</td>\n",
       "      <td>cn1996</td>\n",
       "      <td>23810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>20067587_75</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>61.1GB</td>\n",
       "      <td>cn2927</td>\n",
       "      <td>23823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>20067587_87</td>\n",
       "      <td>norm</td>\n",
       "      <td>TIMEOUT</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>1-00:00:16</td>\n",
       "      <td>65.0GB/node</td>\n",
       "      <td>14.3GB</td>\n",
       "      <td>cn0239</td>\n",
       "      <td>23859</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Jobid Partition    State  Nodes  CPUs    Walltime     Runtime       MemReq MemUsed Nodelist subjects\n",
       "1    20067587_1      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:26  65.0GB/node  60.7GB   cn0744    23428\n",
       "11  20067587_11      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:25  65.0GB/node  65.0GB   cn0331    23520\n",
       "13  20067587_13      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:12  65.0GB/node  59.6GB   cn0156    23540\n",
       "17  20067587_17      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:07  65.0GB/node  65.0GB   cn2007    23549\n",
       "23  20067587_23      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:33  65.0GB/node  64.2GB   cn0145    23564\n",
       "34  20067587_34      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:28  65.0GB/node  65.0GB   cn0096    23638\n",
       "40  20067587_40      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:28  65.0GB/node  61.1GB   cn0641    23652\n",
       "42  20067587_42      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:28  65.0GB/node  61.1GB   cn0775    23660\n",
       "52  20067587_52      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:04  65.0GB/node  60.8GB   cn0208    23732\n",
       "68  20067587_68      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:26  65.0GB/node  35.0GB   cn1697    23798\n",
       "70  20067587_70      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:26  65.0GB/node  65.0GB   cn1996    23810\n",
       "75  20067587_75      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:00  65.0GB/node  61.1GB   cn2927    23823\n",
       "87  20067587_87      norm  TIMEOUT      1    26  1-00:00:00  1-00:00:16  65.0GB/node  14.3GB   cn0239    23859"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jhdf = jobhist(20067587)\n",
    "jhdf['subjects'] = subjects[32:]\n",
    "jhdf.loc[jhdf.State == 'TIMEOUT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job statuses:\n",
      "RUNNING: 0\n",
      "COMPLETED: 157\n",
      "FAILED: 0\n",
      "CANCELLED: 0\n",
      "TIMEOUT: 13\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['23428', '23520', '23540', '23549', '23564', '23638', '23652',\n",
       "       '23660', '23732', '23798', '23810', '23823', '23859'], dtype='<U5')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids['fmriprepv20.2.2_lowmem'] = 20067587\n",
    "fails = get_fails('fmriprepv20.2.2_lowmem',subjects[32:])\n",
    "fails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23428     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23428 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23428 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23520     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23520 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23520 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23540     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23540 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23540 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23549     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23549 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23549 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23564     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23564 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23564 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23638     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23638 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23638 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.2.simg /data/MBDU/mid_share/data/bids_defaced     $TMPDIR/out participant     --participant_label 23652     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/bids_filter --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23652 ; chown -R :MBDU /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2_lowmem_fails1/sub-23652 ; \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.2.2_lowmem_fails1',\n",
       " PosixPath('/data/MBDU/mid_share/swarm/fmriprep/swarm_cmds/fmriprepv20.2.2_lowmem_fails1'),\n",
       " PosixPath('/data/MBDU/mid_share/swarm/fmriprep/swarm_logs/fmriprepv20.2.2_lowmem_fails1'))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_name = 'fmriprepv20.2.2_lowmem_fails1'\n",
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  fails,\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=False, aroma=True)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'20203123'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 100 -t 35 --module singularity,webproxy --time 24:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job statuses:\n",
      "RUNNING: 13\n",
      "COMPLETED: 0\n",
      "FAILED: 0\n",
      "CANCELLED: 0\n",
      "TIMEOUT: 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Jobid</th>\n",
       "      <th>Partition</th>\n",
       "      <th>State</th>\n",
       "      <th>Nodes</th>\n",
       "      <th>CPUs</th>\n",
       "      <th>Walltime</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>MemReq</th>\n",
       "      <th>MemUsed</th>\n",
       "      <th>Nodelist</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20203123_0</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>03:05:19</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn3505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20203123_1</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>03:05:19</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn1641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20203123_2</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>03:05:19</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn1930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20203123_3</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>02:31:50</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn3449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20203123_4</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>02:31:50</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn1576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20203123_5</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>02:31:50</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn1742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>20203123_6</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>02:31:50</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn1787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>20203123_7</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>02:31:50</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn1801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>20203123_8</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>02:31:50</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn1871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>20203123_9</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>02:31:50</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>20203123_10</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>02:31:50</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn2185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>20203123_11</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>02:26:25</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn3277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>20203123_12</td>\n",
       "      <td>norm</td>\n",
       "      <td>RUNNING</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1-00:00:00</td>\n",
       "      <td>02:26:25</td>\n",
       "      <td>100.0GB/node</td>\n",
       "      <td>-</td>\n",
       "      <td>cn1854</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Jobid Partition    State  Nodes  CPUs    Walltime   Runtime        MemReq MemUsed Nodelist\n",
       "0    20203123_0      norm  RUNNING      1    36  1-00:00:00  03:05:19  100.0GB/node       -   cn3505\n",
       "1    20203123_1      norm  RUNNING      1    36  1-00:00:00  03:05:19  100.0GB/node       -   cn1641\n",
       "2    20203123_2      norm  RUNNING      1    36  1-00:00:00  03:05:19  100.0GB/node       -   cn1930\n",
       "3    20203123_3      norm  RUNNING      1    36  1-00:00:00  02:31:50  100.0GB/node       -   cn3449\n",
       "4    20203123_4      norm  RUNNING      1    36  1-00:00:00  02:31:50  100.0GB/node       -   cn1576\n",
       "5    20203123_5      norm  RUNNING      1    36  1-00:00:00  02:31:50  100.0GB/node       -   cn1742\n",
       "6    20203123_6      norm  RUNNING      1    36  1-00:00:00  02:31:50  100.0GB/node       -   cn1787\n",
       "7    20203123_7      norm  RUNNING      1    36  1-00:00:00  02:31:50  100.0GB/node       -   cn1801\n",
       "8    20203123_8      norm  RUNNING      1    36  1-00:00:00  02:31:50  100.0GB/node       -   cn1871\n",
       "9    20203123_9      norm  RUNNING      1    36  1-00:00:00  02:31:50  100.0GB/node       -   cn2014\n",
       "10  20203123_10      norm  RUNNING      1    36  1-00:00:00  02:31:50  100.0GB/node       -   cn2185\n",
       "11  20203123_11      norm  RUNNING      1    36  1-00:00:00  02:26:25  100.0GB/node       -   cn3277\n",
       "12  20203123_12      norm  RUNNING      1    36  1-00:00:00  02:26:25  100.0GB/node       -   cn1854"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobhist(20203123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 23544     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_23544/bids_filter --longitudinal -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_23544/sub-23544 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_23544/sub-23544 ;  (exit $STATUS)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.1.0_20200528_2mm_clifix_noaroma_23544',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_23544'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_23544'))"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_name = 'fmriprepv20.1.0_20200528_2mm_clifix_noaroma_23544'\n",
    "\n",
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  ['23544'],\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=True, aroma=False, t2scoreg=False)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'58972033'"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/multiecho_masking_test      $TMPDIR/out participant     --participant_label 23638     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_23638/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_23638/sub-23638 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_23638/sub-23638/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_23638/sub-23638 ;  (exit $STATUS)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.1.0_20200528_2mm_clifix_23638',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprepv20.1.0_20200528_2mm_clifix_23638'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm_clifix_23638'))"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_name = 'fmriprepv20.1.0_20200528_2mm_clifix_23638'\n",
    "\n",
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  ['23638'],\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=True,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=True, aroma=True, t2scoreg=False, multiecho_masking_test=True)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'59023751'"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/data/MBDU/midla')"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "project_root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/data/MBDU/singularity_images/fmriprep_20.0.5.simg'"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 24028     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_hard/bids_filter -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_hard/sub-24028 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_hard/sub-24028 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 22477     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_hard/bids_filter -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_hard/sub-22477 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_hard/sub-22477 ;  (exit $STATUS)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.1.0_20200528_2mm_clifix_noaroma_hard',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_hard'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_hard'))"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_name = 'fmriprepv20.1.0_20200528_2mm_clifix_noaroma_hard'\n",
    "\n",
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  known_hard_subjects,\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=False, aroma=False, t2scoreg=False)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'58972034'"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fmriprepv20.0.5_20190325_2mm_2_hard': '58869771',\n",
       " 'fmriprepv20.1.0_20200528_2mm_56cpulotsofmem_test': '58889357',\n",
       " 'fmriprepv20.1.0_20200528_2mm_clifix_test': '58892193',\n",
       " 'fmriprepv20.1.0_20200528_2mm_clifix': '58900755',\n",
       " 'fmriprepv20.1.0_20200528_2mm_clifix_23544': '58900758',\n",
       " 'fmriprepv20.1.0_20200528_2mm_clifix_hard': '58919112',\n",
       " 'fmriprepv20.1.0_20200528_2mm_clifix_noaroma': '58971492',\n",
       " 'fmriprepv20.1.0_20200528_2mm_clifix_noaroma_23544': '58972033',\n",
       " 'fmriprepv20.1.0_20200528_2mm_clifix_noaroma_hard': '58972034'}"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'58813266'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids['fmriprepv20.1.0_20200528_2mm']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job statuses:\n",
      "RUNNING: 0\n",
      "COMPLETED: 86\n",
      "FAILED: 101\n",
      "CANCELLED: 0\n"
     ]
    }
   ],
   "source": [
    "jhdf = jobhist(jobids['fmriprepv20.1.0_20200528_2mm_clifix'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "jhdf['subject'] = easy_subjects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Jobid</th>\n",
       "      <th>Partition</th>\n",
       "      <th>State</th>\n",
       "      <th>Nodes</th>\n",
       "      <th>CPUs</th>\n",
       "      <th>Walltime</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>MemReq</th>\n",
       "      <th>MemUsed</th>\n",
       "      <th>Nodelist</th>\n",
       "      <th>subject</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>58813266_52</td>\n",
       "      <td>norm</td>\n",
       "      <td>FAILED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>11:15:59</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>32.7GB</td>\n",
       "      <td>cn3548</td>\n",
       "      <td>23613</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Jobid Partition   State  Nodes  CPUs    Walltime   Runtime       MemReq MemUsed Nodelist subject\n",
       "52  58813266_52      norm  FAILED      1    24  2-00:00:00  11:15:59  64.0GB/node  32.7GB   cn3548   23613"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jhdf.query('subject == \"23613\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm_23544')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "swarm_logs = Path('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm_23544')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_paths = []\n",
    "for jid in jhdf.Jobid:\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "cc_pat = re.compile('cpu_count = ([0-9]*)')\n",
    "task_pat = re.compile('func_preproc_ses_[0-9][0-9]_task_(.*)_run_[0-9]_echo_[0-9]_wf')\n",
    "echo_pat = re.compile('func_preproc_ses_[0-9][0-9]_task_.*_run_[0-9]_echo_([0-9])_wf')\n",
    "run_pat = re.compile('func_preproc_ses_[0-9][0-9]_task_.*_run_([0-9])_echo_[0-9]_wf')\n",
    "error_pat = re.compile('Error on \"(.*)\"')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "jdf = []\n",
    "for ix, ss in enumerate(easy_subjects):\n",
    "    job_info ={}\n",
    "    jobid = f'58900755_{ix}'\n",
    "    job_info['subject'] = ss\n",
    "    job_info['jobid'] = jobid\n",
    "    job_info['log_path'] = f'/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm_clifix/fmriprepv20.1.0_20200528_2mm_clifix_{jobid}.o'\n",
    "    log = Path(job_info['log_path']).read_text()\n",
    "    job_info['cpu_count'] = int(cc_pat.search(log).groups()[0])\n",
    "    \n",
    "    try:\n",
    "        \n",
    "        job_info['errors'] = error_pat.findall(log)\n",
    "        job_info['em1'] = job_info['errors'][0]\n",
    "        job_info['nerrors'] = len(job_info['errors'])\n",
    "        job_info['task'] = task_pat.findall(job_info['em1'])[0]\n",
    "        job_info['echo'] = echo_pat.findall(job_info['em1'])[0]\n",
    "        job_info['run'] = run_pat.findall(job_info['em1'])[0]\n",
    "\n",
    "    except IndexError:\n",
    "        pass\n",
    "    jdf.append(job_info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "jdf = pd.DataFrame(jdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "jdf['State'] = 'FAILED'\n",
    "jdf.loc[jdf.nerrors.isnull(), 'State'] = 'COMPLETED'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_df = jdf.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_df['error'] = em_df.em1.str.split('.').str[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_df.loc[em_df.error.isnull(), ['error', 'task', 'echo', 'run']] = 'NA'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>n</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cpu_count</th>\n",
       "      <th>State</th>\n",
       "      <th>error</th>\n",
       "      <th>task</th>\n",
       "      <th>echo</th>\n",
       "      <th>run</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">32</th>\n",
       "      <th>COMPLETED</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">FAILED</th>\n",
       "      <th>ica_aroma</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ica_aroma_confound_extraction</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">56</th>\n",
       "      <th>COMPLETED</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">FAILED</th>\n",
       "      <th>dvars</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">ica_aroma</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">rest</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ica_aroma_confound_extraction</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                  n\n",
       "cpu_count State     error                         task echo run    \n",
       "32        COMPLETED NA                            NA   NA   NA   58\n",
       "          FAILED    ica_aroma                     rest 1    1    33\n",
       "                    ica_aroma_confound_extraction rest 1    1    15\n",
       "56        COMPLETED NA                            NA   NA   NA   28\n",
       "          FAILED    dvars                         rest 1    1     1\n",
       "                    ica_aroma                     rest 1    1    26\n",
       "                                                            2     1\n",
       "                    ica_aroma_confound_extraction rest 1    1    24"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "em_df.groupby(['cpu_count', 'State', 'error', 'task', 'echo', 'run'])[['jobid']].count().rename(columns={'jobid':'n'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "aroma_failed = em_df.query('State == \"FAILED\"').subject.unique()\n",
    "run_name = 'fmriprepv20.1.0_20200528_2mm_clifix_aromafail'\n",
    "image_path = (sing_img_dir/'fmriprep_20.1.1.simg').as_posix()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.1.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 20900     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-20900 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-20900/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-20900 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.1.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 21748     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-21748 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-21748/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-21748 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.1.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 22228     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-22228 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-22228/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-22228 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.1.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 22279     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-22279 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-22279/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-22279 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.1.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 22686     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-22686 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-22686/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_aromafail/sub-22686 ;  (exit $STATUS)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.1.0_20200528_2mm_clifix_aromafail',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprepv20.1.0_20200528_2mm_clifix_aromafail'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm_clifix_aromafail'))"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  aroma_failed[:5],\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=True,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=True, aroma=True, t2scoreg=False)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'59352656'"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "jhdf['log_path'] = jhdf.Jobid.apply(lambda x:f'/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm_clifix/fmriprepv20.1.0_20200528_2mm_clifix_{x}.o')\n",
    "jhdf['cpu_count'] = jhdf.log_path.apply(lambda x: int(cc_pat.findall(Path(x).read_text())[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>n</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cpu_count</th>\n",
       "      <th>State</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <th>FAILED</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">56</th>\n",
       "      <th>CANCELLED</th>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COMPLETED</th>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FAILED</th>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">72</th>\n",
       "      <th>CANCELLED</th>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COMPLETED</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FAILED</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      n\n",
       "cpu_count State        \n",
       "32        FAILED      5\n",
       "56        CANCELLED  45\n",
       "          COMPLETED  67\n",
       "          FAILED     47\n",
       "72        CANCELLED   8\n",
       "          COMPLETED  10\n",
       "          FAILED      5"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jhdf.groupby(['cpu_count', 'State'])[['Jobid']].count().rename(columns={'Jobid':'n'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "task_pat = re.compile('func_preproc_ses_[0-9][0-9]_task_(.*)_run_[0-9]_echo_[0-9]_wf')\n",
    "echo_pat = re.compile('func_preproc_ses_[0-9][0-9]_task_.*_run_[0-9]_echo_([0-9])_wf')\n",
    "run_pat = re.compile('func_preproc_ses_[0-9][0-9]_task_.*_run_([0-9])_echo_[0-9]_wf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "error_messages = []\n",
    "for lp in jhdf.log_path:\n",
    "    log = Path(lp).read_text()\n",
    "    res = {}\n",
    "    try:\n",
    "        \n",
    "        res['errors'] = error_pat.findall(log)\n",
    "        res['em1'] = res['errors'][0]\n",
    "        res['nerrors'] = len(res['errors'])\n",
    "        res['task'] = task_pat.findall(res['em1'])[0]\n",
    "        res['echo'] = echo_pat.findall(res['em1'])[0]\n",
    "        res['run'] = run_pat.findall(res['em1'])[0]\n",
    "\n",
    "    except IndexError:\n",
    "        pass\n",
    "    error_messages.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_df = pd.concat([jhdf, pd.DataFrame(error_messages)], axis=1)\n",
    "em_df['error'] = em_df.em1.str.split('.').str[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_df.loc[em_df.State == 'CANCELLED', 'State'] = 'MEMORY'\n",
    "em_df.loc[em_df.error.isnull(), ['error', 'task', 'echo', 'run']] = 'NA'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>n</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cpu_count</th>\n",
       "      <th>State</th>\n",
       "      <th>error</th>\n",
       "      <th>task</th>\n",
       "      <th>echo</th>\n",
       "      <th>run</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <th>FAILED</th>\n",
       "      <th>ica_aroma</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">56</th>\n",
       "      <th>COMPLETED</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">FAILED</th>\n",
       "      <th>ica_aroma</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ica_aroma_confound_extraction</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smooth</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">MEMORY</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ica_aroma</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ica_aroma_confound_extraction</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">72</th>\n",
       "      <th>COMPLETED</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">FAILED</th>\n",
       "      <th>ica_aroma</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ica_aroma_confound_extraction</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">MEMORY</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ica_aroma</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ica_aroma_confound_extraction</th>\n",
       "      <th>rest</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                  n\n",
       "cpu_count State     error                         task echo run    \n",
       "32        FAILED    ica_aroma                     rest 1    1     5\n",
       "56        COMPLETED NA                            NA   NA   NA   67\n",
       "          FAILED    ica_aroma                     rest 1    1    30\n",
       "                    ica_aroma_confound_extraction rest 1    1    16\n",
       "                    smooth                        rest 1    1     1\n",
       "          MEMORY    NA                            NA   NA   NA   15\n",
       "                    ica_aroma                     rest 1    1    21\n",
       "                    ica_aroma_confound_extraction rest 1    1     8\n",
       "72        COMPLETED NA                            NA   NA   NA   10\n",
       "          FAILED    ica_aroma                     rest 1    1     3\n",
       "                    ica_aroma_confound_extraction rest 1    1     2\n",
       "          MEMORY    NA                            NA   NA   NA    1\n",
       "                    ica_aroma                     rest 1    1     6\n",
       "                    ica_aroma_confound_extraction rest 1    1     1"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "em_df.groupby(['cpu_count', 'State', 'error', 'task', 'echo', 'run'])[['Jobid']].count().rename(columns={'Jobid':'n'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Jobid</th>\n",
       "      <th>Partition</th>\n",
       "      <th>State</th>\n",
       "      <th>Nodes</th>\n",
       "      <th>CPUs</th>\n",
       "      <th>Walltime</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>MemReq</th>\n",
       "      <th>MemUsed</th>\n",
       "      <th>Nodelist</th>\n",
       "      <th>subject</th>\n",
       "      <th>log_path</th>\n",
       "      <th>cpu_count</th>\n",
       "      <th>echo</th>\n",
       "      <th>em1</th>\n",
       "      <th>errors</th>\n",
       "      <th>nerrors</th>\n",
       "      <th>run</th>\n",
       "      <th>task</th>\n",
       "      <th>error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>58813266_13</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>21:59:13</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>64.0GB</td>\n",
       "      <td>cn1083</td>\n",
       "      <td>22698</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_13.o</td>\n",
       "      <td>72</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>58813266_20</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>21:50:58</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>58.7GB</td>\n",
       "      <td>cn3185</td>\n",
       "      <td>23108</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_20.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>58813266_33</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>21:08:35</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>64.0GB</td>\n",
       "      <td>cn3171</td>\n",
       "      <td>23519</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_33.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>58813266_34</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>21:08:35</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>64.0GB</td>\n",
       "      <td>cn3281</td>\n",
       "      <td>23520</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_34.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>58813266_40</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>21:02:58</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>64.0GB</td>\n",
       "      <td>cn3206</td>\n",
       "      <td>23550</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_40.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>58813266_41</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>21:02:58</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>64.0GB</td>\n",
       "      <td>cn3295</td>\n",
       "      <td>23553</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_41.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>58813266_42</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>21:02:58</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>57.5GB</td>\n",
       "      <td>cn3349</td>\n",
       "      <td>23554</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_42.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>58813266_44</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>21:02:58</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>63.4GB</td>\n",
       "      <td>cn3481</td>\n",
       "      <td>23564</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_44.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>58813266_45</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>21:02:58</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>61.1GB</td>\n",
       "      <td>cn3585</td>\n",
       "      <td>23565</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_45.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>58813266_71</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>20:08:04</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>64.0GB</td>\n",
       "      <td>cn3486</td>\n",
       "      <td>23732</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_71.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>58813266_85</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>20:08:04</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>64.0GB</td>\n",
       "      <td>cn1775</td>\n",
       "      <td>23783</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_85.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>58813266_95</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>20:08:04</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>62.8GB</td>\n",
       "      <td>cn1835</td>\n",
       "      <td>23825</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_95.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>58813266_127</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>20:08:03</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>64.0GB</td>\n",
       "      <td>cn2131</td>\n",
       "      <td>23951</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_127.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>58813266_148</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>20:02:05</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>64.0GB</td>\n",
       "      <td>cn1747</td>\n",
       "      <td>24072</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_148.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>58813266_152</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>20:02:05</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>64.0GB</td>\n",
       "      <td>cn1963</td>\n",
       "      <td>24088</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_152.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>58813266_154</td>\n",
       "      <td>norm</td>\n",
       "      <td>MEMORY</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>20:02:05</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>64.0GB</td>\n",
       "      <td>cn2002</td>\n",
       "      <td>24093</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_154.o</td>\n",
       "      <td>56</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Jobid Partition   State  Nodes  CPUs    Walltime   Runtime       MemReq MemUsed Nodelist subject                                                                                                             log_path  cpu_count echo  em1 errors  nerrors run task error\n",
       "13    58813266_13      norm  MEMORY      1    24  2-00:00:00  21:59:13  64.0GB/node  64.0GB   cn1083   22698   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_13.o         72   NA  NaN     []      NaN  NA   NA    NA\n",
       "20    58813266_20      norm  MEMORY      1    24  2-00:00:00  21:50:58  64.0GB/node  58.7GB   cn3185   23108   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_20.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "33    58813266_33      norm  MEMORY      1    24  2-00:00:00  21:08:35  64.0GB/node  64.0GB   cn3171   23519   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_33.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "34    58813266_34      norm  MEMORY      1    24  2-00:00:00  21:08:35  64.0GB/node  64.0GB   cn3281   23520   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_34.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "40    58813266_40      norm  MEMORY      1    24  2-00:00:00  21:02:58  64.0GB/node  64.0GB   cn3206   23550   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_40.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "41    58813266_41      norm  MEMORY      1    24  2-00:00:00  21:02:58  64.0GB/node  64.0GB   cn3295   23553   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_41.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "42    58813266_42      norm  MEMORY      1    24  2-00:00:00  21:02:58  64.0GB/node  57.5GB   cn3349   23554   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_42.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "44    58813266_44      norm  MEMORY      1    24  2-00:00:00  21:02:58  64.0GB/node  63.4GB   cn3481   23564   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_44.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "45    58813266_45      norm  MEMORY      1    24  2-00:00:00  21:02:58  64.0GB/node  61.1GB   cn3585   23565   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_45.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "71    58813266_71      norm  MEMORY      1    24  2-00:00:00  20:08:04  64.0GB/node  64.0GB   cn3486   23732   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_71.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "85    58813266_85      norm  MEMORY      1    24  2-00:00:00  20:08:04  64.0GB/node  64.0GB   cn1775   23783   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_85.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "95    58813266_95      norm  MEMORY      1    24  2-00:00:00  20:08:04  64.0GB/node  62.8GB   cn1835   23825   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_95.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "127  58813266_127      norm  MEMORY      1    24  2-00:00:00  20:08:03  64.0GB/node  64.0GB   cn2131   23951  /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_127.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "148  58813266_148      norm  MEMORY      1    24  2-00:00:00  20:02:05  64.0GB/node  64.0GB   cn1747   24072  /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_148.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "152  58813266_152      norm  MEMORY      1    24  2-00:00:00  20:02:05  64.0GB/node  64.0GB   cn1963   24088  /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_152.o         56   NA  NaN     []      NaN  NA   NA    NA\n",
       "154  58813266_154      norm  MEMORY      1    24  2-00:00:00  20:02:05  64.0GB/node  64.0GB   cn2002   24093  /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_154.o         56   NA  NaN     []      NaN  NA   NA    NA"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "em_df.query(\"State == 'MEMORY' & error == 'NA'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Jobid</th>\n",
       "      <th>Partition</th>\n",
       "      <th>State</th>\n",
       "      <th>Nodes</th>\n",
       "      <th>CPUs</th>\n",
       "      <th>Walltime</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>MemReq</th>\n",
       "      <th>MemUsed</th>\n",
       "      <th>Nodelist</th>\n",
       "      <th>subject</th>\n",
       "      <th>log_path</th>\n",
       "      <th>cpu_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>58813266_15</td>\n",
       "      <td>norm</td>\n",
       "      <td>FAILED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>16:53:46</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>44.1GB</td>\n",
       "      <td>cn0156</td>\n",
       "      <td>22749</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_15.o</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>58813266_144</td>\n",
       "      <td>norm</td>\n",
       "      <td>FAILED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>15:33:21</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>43.4GB</td>\n",
       "      <td>cn0220</td>\n",
       "      <td>24060</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_144.o</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>58813266_145</td>\n",
       "      <td>norm</td>\n",
       "      <td>FAILED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>15:44:26</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>37.2GB</td>\n",
       "      <td>cn0362</td>\n",
       "      <td>24064</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_145.o</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>58813266_146</td>\n",
       "      <td>norm</td>\n",
       "      <td>FAILED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>16:01:31</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>47.3GB</td>\n",
       "      <td>cn0738</td>\n",
       "      <td>24065</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_146.o</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>58813266_179</td>\n",
       "      <td>norm</td>\n",
       "      <td>FAILED</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2-00:00:00</td>\n",
       "      <td>12:16:34</td>\n",
       "      <td>64.0GB/node</td>\n",
       "      <td>28.4GB</td>\n",
       "      <td>cn0207</td>\n",
       "      <td>24186</td>\n",
       "      <td>/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_179.o</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Jobid Partition   State  Nodes  CPUs    Walltime   Runtime       MemReq MemUsed Nodelist subject                                                                                                             log_path  cpu_count\n",
       "15    58813266_15      norm  FAILED      1    24  2-00:00:00  16:53:46  64.0GB/node  44.1GB   cn0156   22749   /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_15.o         32\n",
       "144  58813266_144      norm  FAILED      1    24  2-00:00:00  15:33:21  64.0GB/node  43.4GB   cn0220   24060  /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_144.o         32\n",
       "145  58813266_145      norm  FAILED      1    24  2-00:00:00  15:44:26  64.0GB/node  37.2GB   cn0362   24064  /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_145.o         32\n",
       "146  58813266_146      norm  FAILED      1    24  2-00:00:00  16:01:31  64.0GB/node  47.3GB   cn0738   24065  /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_146.o         32\n",
       "179  58813266_179      norm  FAILED      1    24  2-00:00:00  12:16:34  64.0GB/node  28.4GB   cn0207   24186  /data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm/fmriprepv20.1.0_20200528_2mm_58813266_179.o         32"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jhdf.query(\"cpu_count == 32 & State == 'FAILED'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_name = 'fmriprepv20.1.0_20200528_2mm_'\n",
    "\n",
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  known_hard_subjects,\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=False, aroma=True, t2scoreg=False)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job statuses:\n",
      "RUNNING: 1\n",
      "COMPLETED: 100\n",
      "FAILED: 88\n",
      "CANCELLED: 0\n"
     ]
    }
   ],
   "source": [
    "jhdf = jobhist(jobids['fmriprepv20.0.5_20190325_2mm'])\n",
    "bad_states = ['FAILED', 'CANCELLED']\n",
    "failed_runs = np.array(jhdf.loc[jhdf.State.isin(bad_states), 'Jobid'].str.split('_').str[1]).astype(int)\n",
    "failed_subjs = np.array(subjects)[failed_runs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_name = 'fmriprepv20.0.5_20190325_2mm_2_hard'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "known_hard_subjects = ['24028', '22477']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 24028     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_hard/bids_filter --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_hard/sub-24028 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_hard/sub-24028 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 22477     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_hard/bids_filter --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_hard/sub-22477 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_hard/sub-22477 ;  (exit $STATUS)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.0.5_20190325_2mm_2_hard',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprepv20.0.5_20190325_2mm_2_hard'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.0.5_20190325_2mm_2_hard'))"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  known_hard_subjects,\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=False, aroma=True, t2scoreg=False)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'58869771'"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_name = 'fmriprepv20.0.5_20190325_2mm_2_fails'\n",
    "image_path = (sing_img_dir/'fmriprep_20.0.5.simg').as_posix()\n",
    "failed_subjs = ['22698',\n",
    "'23015',\n",
    "'23495',\n",
    "'23516',\n",
    "'23544',\n",
    "'23546',\n",
    "'23564',\n",
    "'23641',\n",
    "'23652',\n",
    "'23988']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 22698     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-22698 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-22698 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 23015     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23015 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23015 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 23495     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23495 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23495 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 23516     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23516 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23516 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 23544     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23544 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23544 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 23546     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23546 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23546 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 23564     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23564 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm_2_fails/sub-23564 ;  (exit $STATUS)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.0.5_20190325_2mm_2_fails',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprepv20.0.5_20190325_2mm_2_fails'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.0.5_20190325_2mm_2_fails'))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  failed_subjs,\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=True, aroma=True, t2scoreg=False)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'58759467'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs3,/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 21111     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings --t2s-coreg -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-21111 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-21111 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs3,/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 21748     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings --t2s-coreg -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-21748 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-21748 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs3,/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 22228     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings --t2s-coreg -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-22228 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-22228 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs3,/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 22279     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings --t2s-coreg -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-22279 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-22279 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs3,/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 22293     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings --t2s-coreg -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-22293 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-22293 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs3,/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 22409     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings --t2s-coreg -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-22409 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-22409 ;  (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs3,/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_20.0.5.simg /data/MBDU/midla/data/bids      $TMPDIR/out participant     --participant_label 22477     -w $TMPDIR/wrk     --nthreads $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 fsaverage     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/bids_filter --longitudinal --use-aroma --error-on-aroma-warnings --t2s-coreg -vvv ;     STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-22477 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.0.5_20190325_2mm/sub-22477 ;  (exit $STATUS)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.0.5_20190325_2mm',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprepv20.0.5_20190325_2mm'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.0.5_20190325_2mm'))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  failed_subjs,\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=True, aroma=True, t2scoreg=True)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'5'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name+'_2'] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name+'_2'] = jobids[run_name][0]\n",
    "jobids[run_name+'_2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'5'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name+'_2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "rerun = ['21669', '22127', '22477', '22749', '23015', '23108', '23298',\n",
    "       '23303', '23516', '23528', '23557', '23611', '23717', '23718',\n",
    "       '23720', '23753', '23770', '23783', '23809', '23813', '23825',\n",
    "       '23834', '23843', '23852', '23857', '23886', '23890', '23895',\n",
    "       '23900', '23905', '23906', '23919', '23927', '23935', '23943',\n",
    "       '23957', '23969', '24028', '24047', '24052', '24059', '24060',\n",
    "       '24064', '24065', '24077', '24078', '24088', '24093', '24101',\n",
    "       '24110', '24111', '24116', '24123', '24137', '24138', '24156',\n",
    "       '24158', '24167', '24176', '24177', '24180', '24186', '24204',\n",
    "       '24226', '24264']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_name = 'fmriprepv20.0.5_20190325_2mm'\n",
    "image_path = (sing_img_dir/'fmriprep_20.0.5.simg').as_posix()\n",
    "\n",
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  subjects,\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=False,\n",
    "                                                  verbose=True,\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=True, aroma=True, t2scoreg=True)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "ename": "EmptyDataError",
     "evalue": "No columns to parse from file",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mEmptyDataError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-d09a2c02ee07>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mjhdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjobhist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjobids\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mrun_name\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'_2'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mfailed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msubj_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjhdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubjects\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatuses\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'FAILED'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'CANCELLED'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-d19fa9332c97>\u001b[0m in \u001b[0;36mjobhist\u001b[0;34m(jid)\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mll\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m             \u001b[0mnblanks\u001b[0m\u001b[0;34m+=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m     \u001b[0mjhdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mStringIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdelim_whitespace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m     \u001b[0mstates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"RUNNING\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"COMPLETED\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"FAILED\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"CANCELLED\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Job statuses:\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/gpfs/gsfs11/users/MBDU/midla/env/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    700\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    701\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    703\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    704\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/gpfs/gsfs11/users/MBDU/midla/env/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    428\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 429\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    431\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/gpfs/gsfs11/users/MBDU/midla/env/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    893\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    896\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/gpfs/gsfs11/users/MBDU/midla/env/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1120\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1121\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1122\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1123\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1124\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/gpfs/gsfs11/users/MBDU/midla/env/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1851\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'usecols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1852\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1853\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1854\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1855\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mEmptyDataError\u001b[0m: No columns to parse from file"
     ]
    }
   ],
   "source": [
    "jhdf = jobhist(jobids[run_name+'_2'])\n",
    "failed = subj_status(jhdf, subjects, statuses=['FAILED', 'CANCELLED'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rerun fmriprep with freesurfer directory and bold output while saving work directory so I can see how to pull out the transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_name = 'fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "easy_subjects = [ss for ss in subjects if ss not in known_hard_subjects]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 20900     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/bids_filter --longitudinal -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/sub-20900 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/sub-20900/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/sub-20900 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 21111     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/bids_filter --longitudinal -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/sub-21111 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/sub-21111/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/sub-21111 ; \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output'))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  easy_subjects[:2],\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=True,\n",
    "                                                  verbose=True,\n",
    "                                                  output_spaces = ['MNI152NLin2009cAsym:res-2', 'func'],\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=True, aroma=False, t2scoreg=False,\n",
    "                                                  fs_subjects_run='fmriprepv20.1.0_20200528_2mm_clifix_noaroma')\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'62826584'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 20900     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/bids_filter --longitudinal -vvv; mkdir $TMPDIR/mnitobold;  singularity exec --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep                     /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/notebooks/code/run_mnitobold.sh  $TMPDIR $TMPDIR/mnitobold  /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_T1w.nii.gz /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_desc-carpet_dseg.nii.gz --omp-nthreads=$SLURM_CPUS_PER_TASK --mem-gb=$SLURM_MEM_PER_NODE ; STATUS=$? ;  rsync -ach $TMPDIR/mnitobold/ /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-20900 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-20900 ; (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 21111     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/bids_filter --longitudinal -vvv; mkdir $TMPDIR/mnitobold;  singularity exec --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep                     /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/notebooks/code/run_mnitobold.sh  $TMPDIR $TMPDIR/mnitobold  /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_T1w.nii.gz /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_desc-carpet_dseg.nii.gz --omp-nthreads=$SLURM_CPUS_PER_TASK --mem-gb=$SLURM_MEM_PER_NODE ; STATUS=$? ;  rsync -ach $TMPDIR/mnitobold/ /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-21111 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-21111 ; (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 21669     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/bids_filter --longitudinal -vvv; mkdir $TMPDIR/mnitobold;  singularity exec --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep                     /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/notebooks/code/run_mnitobold.sh  $TMPDIR $TMPDIR/mnitobold  /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_T1w.nii.gz /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_desc-carpet_dseg.nii.gz --omp-nthreads=$SLURM_CPUS_PER_TASK --mem-gb=$SLURM_MEM_PER_NODE ; STATUS=$? ;  rsync -ach $TMPDIR/mnitobold/ /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-21669 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-21669 ; (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 21723     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/bids_filter --longitudinal -vvv; mkdir $TMPDIR/mnitobold;  singularity exec --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep                     /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/notebooks/code/run_mnitobold.sh  $TMPDIR $TMPDIR/mnitobold  /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_T1w.nii.gz /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_desc-carpet_dseg.nii.gz --omp-nthreads=$SLURM_CPUS_PER_TASK --mem-gb=$SLURM_MEM_PER_NODE ; STATUS=$? ;  rsync -ach $TMPDIR/mnitobold/ /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-21723 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-21723 ; (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 21748     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/bids_filter --longitudinal -vvv; mkdir $TMPDIR/mnitobold;  singularity exec --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep                     /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/notebooks/code/run_mnitobold.sh  $TMPDIR $TMPDIR/mnitobold  /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_T1w.nii.gz /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_desc-carpet_dseg.nii.gz --omp-nthreads=$SLURM_CPUS_PER_TASK --mem-gb=$SLURM_MEM_PER_NODE ; STATUS=$? ;  rsync -ach $TMPDIR/mnitobold/ /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-21748 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-21748 ; (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 22127     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/bids_filter --longitudinal -vvv; mkdir $TMPDIR/mnitobold;  singularity exec --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep                     /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/notebooks/code/run_mnitobold.sh  $TMPDIR $TMPDIR/mnitobold  /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_T1w.nii.gz /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_desc-carpet_dseg.nii.gz --omp-nthreads=$SLURM_CPUS_PER_TASK --mem-gb=$SLURM_MEM_PER_NODE ; STATUS=$? ;  rsync -ach $TMPDIR/mnitobold/ /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-22127 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-22127 ; (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 22228     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/bids_filter --longitudinal -vvv; mkdir $TMPDIR/mnitobold;  singularity exec --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep                     /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/notebooks/code/run_mnitobold.sh  $TMPDIR $TMPDIR/mnitobold  /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_T1w.nii.gz /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_desc-carpet_dseg.nii.gz --omp-nthreads=$SLURM_CPUS_PER_TASK --mem-gb=$SLURM_MEM_PER_NODE ; STATUS=$? ;  rsync -ach $TMPDIR/mnitobold/ /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-22228 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/mnitobold/run3/sub-22228 ; (exit $STATUS)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output'))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_name = 'fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output'\n",
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  easy_subjects,\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=True,\n",
    "                                                  verbose=True,\n",
    "                                                  output_spaces = ['MNI152NLin2009cAsym:res-2', 'func'],\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=True, aroma=False, t2scoreg=False, mnitobold=True,\n",
    "                                                  fs_subjects_run='fmriprepv20.1.0_20200528_2mm_clifix_noaroma')\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'64536703'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cmd_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids_defaced      $TMPDIR/out participant     --participant_label 22477     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/bids_filter --longitudinal -vvv; mkdir $TMPDIR/mnitobold;  singularity exec --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep                     /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/notebooks/code/run_mnitobold.sh  $TMPDIR $TMPDIR/mnitobold  /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_T1w.nii.gz /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_desc-carpet_dseg.nii.gz --omp-nthreads=$SLURM_CPUS_PER_TASK --mem-gb=$SLURM_MEM_PER_NODE ; STATUS=$? ;  rsync -avch $TMPDIR/mnitobold/ /data/MBDU/midla/data/derivatives/mnitobold/run1/sub-22477 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/mnitobold/run1/sub-22477 ; (exit $STATUS)\n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv --bind /data/MBDU/nielsond/fmriprep/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/data/bids_defaced      $TMPDIR/out participant     --participant_label 23638     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --bids-filter-file /data/MBDU/midla/data/derivatives/fmriprep/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output/bids_filter --longitudinal -vvv; mkdir $TMPDIR/mnitobold;  singularity exec --cleanenv --bind /data/MBDU/nielsond/fmriprep-fix-cli-parser/fmriprep/:/usr/local/miniconda/lib/python3.7/site-packages/fmriprep                     /data/MBDU/singularity_images/fmriprep_20.1.0.simg /data/MBDU/midla/notebooks/code/run_mnitobold.sh  $TMPDIR $TMPDIR/mnitobold  /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_T1w.nii.gz /data/MBDU/midla/data/templates/tpl-MNI152NLin2009cAsym_res-02_desc-carpet_dseg.nii.gz --omp-nthreads=$SLURM_CPUS_PER_TASK --mem-gb=$SLURM_MEM_PER_NODE ; STATUS=$? ;  rsync -avch $TMPDIR/mnitobold/ /data/MBDU/midla/data/derivatives/mnitobold/run1/sub-23638 ; chown -R :MBDU /data/MBDU/midla/data/derivatives/mnitobold/run1/sub-23638 ; (exit $STATUS)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output'))"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_name = 'fmriprepv20.1.0_20200528_2mm_clifix_noaroma_bold_output'\n",
    "bids_filter = {\n",
    "                \"t1w\": {\n",
    "                    \"datatype\": \"anat\",\n",
    "                    \"reconstruction\": \"prenorm\",\n",
    "                    \"suffix\": \"T1w\"\n",
    "                }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  [22477, 23638],\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=True,\n",
    "                                                  verbose=True,\n",
    "                                                  defaced=False,\n",
    "                                                  output_spaces = ['MNI152NLin2009cAsym:res-2', 'func'],\n",
    "                                                  anat_only=False,\n",
    "                                                  bids_filter=bids_filter,\n",
    "                                                  longitudinal=True, aroma=False, t2scoreg=False, mnitobold=True,\n",
    "                                                    \n",
    "                                                  fs_subjects_run='fmriprepv20.1.0_20200528_2mm_clifix_noaroma')\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'63265649'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run with lts-rc to test mutli-echo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.0rc0.simg /data/MBDU/midla/data/bids_defaced     $TMPDIR/out participant     --participant_label 20900     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --longitudinal --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-20900 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-20900/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-20900 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.0rc0.simg /data/MBDU/midla/data/bids_defaced     $TMPDIR/out participant     --participant_label 21111     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --longitudinal --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-21111 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-21111/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-21111 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.0rc0.simg /data/MBDU/midla/data/bids_defaced     $TMPDIR/out participant     --participant_label 21669     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --longitudinal --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-21669 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-21669/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-21669 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.0rc0.simg /data/MBDU/midla/data/bids_defaced     $TMPDIR/out participant     --participant_label 22477     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --longitudinal --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-22477 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-22477/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-22477 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.0rc0.simg /data/MBDU/midla/data/bids_defaced     $TMPDIR/out participant     --participant_label 23638     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --longitudinal --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-23638 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-23638/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output/sub-23638 ; \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output'))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_path = (sing_img_dir/'fmriprep_v20.2.0rc0.simg').as_posix()\n",
    "\n",
    "run_name = 'fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output'\n",
    "# bids_filter = {\n",
    "#                 \"t1w\": {\n",
    "#                     \"datatype\": \"anat\",\n",
    "#                     \"reconstruction\": \"prenorm\",\n",
    "#                     \"suffix\": \"T1w\"\n",
    "#                 }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  ['20900', '21111', '21669', '22477', '23638'],\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=True,\n",
    "                                                  verbose=True,\n",
    "                                                  defaced=True,\n",
    "                                                  output_spaces = ['MNI152NLin2009cAsym:res-2', 'func'],\n",
    "                                                  anat_only=False,\n",
    "                                                  longitudinal=True, aroma=True, t2scoreg=False)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'64649157'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['20900', '21111', '21669']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "easy_subjects[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.0rc0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 20900     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --longitudinal --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-20900 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-20900/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-20900 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.0rc0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 21111     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --longitudinal --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-21111 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-21111/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-21111 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.0rc0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 21669     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --longitudinal --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-21669 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-21669/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-21669 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.0rc0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 22477     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --longitudinal --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-22477 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-22477/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-22477 ; \n",
      "export TMPDIR=/lscratch/$SLURM_JOB_ID &&     export SINGULARITY_BINDPATH=\"/gs4,/gs5,/gs6,/gs7,/gs8,/gs9,/gs10,/gs11,/spin1,/scratch,/fdb,/data,/lscratch\" &&    mkdir -p $TMPDIR/out &&     mkdir -p $TMPDIR/wrk &&     singularity run --cleanenv /data/MBDU/singularity_images/fmriprep_v20.2.0rc0.simg /data/MBDU/midla/data/bids     $TMPDIR/out participant     --participant_label 23638     -w $TMPDIR/wrk     --nprocs $SLURM_CPUS_PER_TASK     --mem_mb $SLURM_MEM_PER_NODE     --fs-license-file /data/MBDU/singularity_images/license.txt     --output-spaces MNI152NLin2009cAsym:res-2 func     --dummy-scans 4 --longitudinal --use-aroma --error-on-aroma-warnings -vvv;  STATUS=$? ;  rsync -ach $TMPDIR/out /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-23638 ; rsync -ach $TMPDIR/wrk /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-23638/wrk ;  chown -R :MBDU /data/MBDU/midla/data/derivatives/fmriprep/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface/sub-23638 ; \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface',\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_cmds/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface'),\n",
       " PosixPath('/data/MBDU/midla/swarm/fmriprep/swarm_logs/fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface'))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_path = (sing_img_dir/'fmriprep_v20.2.0rc0.simg').as_posix()\n",
    "\n",
    "run_name = 'fmriprep_v20.2.0rc0_2mm_clifix_noaroma_bold_output_nodeface'\n",
    "# bids_filter = {\n",
    "#                 \"t1w\": {\n",
    "#                     \"datatype\": \"anat\",\n",
    "#                     \"reconstruction\": \"prenorm\",\n",
    "#                     \"suffix\": \"T1w\"\n",
    "#                 }}\n",
    "cmd_file, swarm_log_dir, _cmds = write_swarm_file(run_name,\n",
    "                                                  project_root,\n",
    "                                                  image_path,\n",
    "                                                  fs_licence_path,\n",
    "                                                  ['20900', '21111', '21669', '22477', '23638'],\n",
    "                                                  layout=layout,\n",
    "                                                  sing_sess=False,\n",
    "                                                  save_work_directory=True,\n",
    "                                                  verbose=True,\n",
    "                                                  defaced=False,\n",
    "                                                  output_spaces = ['MNI152NLin2009cAsym:res-2', 'func'],\n",
    "                                                  anat_only=False,\n",
    "                                                  longitudinal=True, aroma=True, t2scoreg=False)\n",
    "\n",
    "run_name, cmd_file, swarm_log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'64654916'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:200 -g 64 -t 24 --module singularity,webproxy --time 48:00:00 --logdir {swarm_log_dir} --job-name {run_name}\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}