write rsync swarm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mbdu_bids_root = Path('/data/MBDU/ABCD/BIDS/NKI_script/MID')\n",
    "dsst_bids_root = Path('/data/ABCD_DSST/bids_20190215/')\n",
    "rsync_swarm_file = Path('/data/ABCD_DSST/swarms/rsync_swarm/rsync_swarm')\n",
    "rsync_swarm_log = Path('/data/ABCD_DSST/swarms/rsync_swarm/logs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "subs = sorted(list(mbdu_bids_root.glob('sub-*')))\n",
    "cmd_template = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmds = []\n",
    "for sub in subs:\n",
    "    tmp = (sub / 'tmp')\n",
    "    rest = (sub / 'ses-1' / 'rest')\n",
    "    dest = (dsst_bids_root / sub.parts[-1])\n",
    "    if (not tmp.exists()) & (not rest.exists()) & (not dest.exists()):\n",
    "        cmd = f'rsync -ach {sub} {dsst_bids_root}/'\n",
    "        cmds.append(cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/data/ABCD_DSST/bids_20190215/sub-NDARINVZZZP87KR')"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ".ex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9799"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rsync_swarm_file.write_text('\\n'.join(cmds[:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV007W6H7B /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV00CY2MDM /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV00HEV6HB /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV00LH735Y /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV00LJVZK2 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV00R4TXET /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV00U4FTRU /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV00X2TBWJ /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV014RTM1V /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0191C80U /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV019DXLU4 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV01AJ15N9 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV01D03VR7 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV01ELX9L6 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV01NAYMZH /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV01RGTWD2 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV02JXJUZG /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV04BFG4JM /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV04JY6730 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV04P0G6LK /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV05ATJ1V1 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV05CA3VX5 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV05X0LM1N /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV06A9DLE9 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV06BKAHN5 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV06DJHEV0 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV06DP74KL /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV06DX0E7R /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV06LWXY67 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV07CMT243 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV07K2M5P4 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV07RAHHYH /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV07UK1DPH /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV07XXFHDK /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV086114AT /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0889M0JE /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV08GBDG8X /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV08K0R9C4 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV08K8VF33 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV08P1JKNE /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV08P3GNDV /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV08R2PTT1 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV08UNGWZK /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV094UXV0J /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV09AEBLZH /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV09AKV10T /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV09T2EBX4 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV09VBPCLU /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV09ZE6UUK /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0A4P0LWM /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0A4ZDYNL /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0A6WVRZY /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0A87RKWD /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0AHJHA5D /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0APJMRD1 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0AYJMFMW /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0AZFZ65C /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0B7UGM1D /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0BAVEL0T /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0BEPJHU1 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0C471G23 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0C765WK4 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0CBPKF6W /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0CCMBWPE /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0CF1U8X8 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0CFR33F0 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0CKA3YZX /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0CP9XGTP /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0D0C239B /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0D5J9T8P /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0D83M5VE /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0DC9BJZK /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0DG8XR2P /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0DKWEM1A /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0E350J5D /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0EV57VEX /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0EVRF8BN /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0EWGP0U5 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0F78WV5U /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0FBX6KWX /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0FM9MUTU /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0FUC15ZB /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0FV544H9 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0FZ26GB1 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0G37YGY1 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0G42TBW1 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0GGJJ4H9 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0GND16RW /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0GUTM6AM /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0GWJ7GFF /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0GZ334XA /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0H05G0TR /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0HALXUFY /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0HL5D3DF /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0HVEVD01 /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0J6LY05U /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0JW2EU4V /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0JWEE23L /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0JWTGKAD /data/ABCD_DSST/bids_20190215/',\n",
       " 'rsync -ach /data/MBDU/ABCD/BIDS/NKI_script/MID/sub-NDARINV0JXBZBV9 /data/ABCD_DSST/bids_20190215/']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rsync_swarm_file.read_text().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20462860\r\n"
     ]
    }
   ],
   "source": [
    "# Submitting a test\n",
    "!swarm -f {rsync_swarm_file} -g 5 -t 2 -b 24 --partition norm,quick --logdir {rsync_swarm_log} --time 00:10:00"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5099"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rsync_swarm_file.write_text('\\n'.join(cmds[100:]))\n",
    "len(rsync_swarm_file.read_text().split('\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20463478\r\n"
     ]
    }
   ],
   "source": [
    "# Submitting for real\n",
    "!swarm -f {rsync_swarm_file} -g 5 -t 2 -b 24 --partition norm,quick --logdir {rsync_swarm_log} --time 00:10:00"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check to see how many new subs are done\n",
    "cmds = []\n",
    "for sub in subs:\n",
    "    tmp = (sub / 'tmp')\n",
    "    rest = (sub / 'ses-1' / 'rest')\n",
    "    dest = (dsst_bids_root / sub.parts[-1])\n",
    "    if (not tmp.exists()) & (not rest.exists()) & (not dest.exists()):\n",
    "        cmd = f'rsync -ach {sub} {dsst_bids_root}/'\n",
    "        cmds.append(cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2646"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rsync_swarm_file.write_text('\\n'.join(cmds))\n",
    "len(rsync_swarm_file.read_text().split('\\n'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20500070\r\n"
     ]
    }
   ],
   "source": [
    "# Submitting \n",
    "!swarm -f {rsync_swarm_file} -g 24 -t 2 -b 24 --partition norm,quick --logdir {rsync_swarm_log} --time 00:10:00"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check to see how many new subs are done\n",
    "cmds = []\n",
    "for sub in subs:\n",
    "    tmp = (sub / 'tmp')\n",
    "    rest = (sub / 'ses-1' / 'rest')\n",
    "    dest = (dsst_bids_root / sub.parts[-1])\n",
    "    if (not tmp.exists()) & (not rest.exists()) & (not dest.exists()):\n",
    "        cmd = f'rsync -ach {sub} {dsst_bids_root}/'\n",
    "        cmds.append(cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rsync_swarm_file.write_text('\\n'.join(cmds))\n",
    "len(rsync_swarm_file.read_text().split('\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20544480\r\n"
     ]
    }
   ],
   "source": [
    "# Submitting \n",
    "!swarm -f {rsync_swarm_file} -g 5 -t 2 -b 24 --partition norm,quick --logdir {rsync_swarm_log} --time 00:10:00"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "subs = sorted(list(mbdu_bids_root.glob('sub-*')))\n",
    "\n",
    "# check to see how many new subs are done\n",
    "cmds = []\n",
    "for sub in subs:\n",
    "    tmp = (sub / 'tmp')\n",
    "    rest = (sub / 'ses-1' / 'rest')\n",
    "    dest = (dsst_bids_root / sub.parts[-1])\n",
    "    if (not tmp.exists()) & (not rest.exists()) & (not dest.exists()):\n",
    "        cmd = f'rsync -ach {sub} {dsst_bids_root}/'\n",
    "        cmds.append(cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1284"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rsync_swarm_file.write_text('\\n'.join(cmds))\n",
    "len(rsync_swarm_file.read_text().split('\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20590619\r\n"
     ]
    }
   ],
   "source": [
    "!swarm -f {rsync_swarm_file} -g 5 -t 2 -b 24 --partition norm,quick --logdir {rsync_swarm_log} --time 00:10:00"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}