pipeline-timing-new-segmentation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cab7fcc7-bad4-4f3b-b83b-4977fec6414e",
   "metadata": {},
   "source": [
    "## This file contains all code necessary to time segmentation methods, across multiple scans and different subsets of recordings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21462e40-a632-4387-8c96-d861f3eb22d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pipeline_timing import pipeline_with_dataload\n",
    "from profiling_util import average_profilings, sort_files, plot_profiling_params\n",
    "from segmentation.segmentation_pipelines import extract_masks_adapted_time_profiling, save_as_memmap\n",
    "import suite2p\n",
    "from pipeline_memray_new import perform_raster_correction, perform_motion_correction\n",
    "from caiman import load_memmap\n",
    "import numpy as np\n",
    "from numpy import std\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20e86f95-9eb2-4427-ae83-acc529d28c90",
   "metadata": {},
   "source": [
    "Run this cell if utils or segmentation_pipelines files get updated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3348fb52-2867-484e-bcab-89096f58b716",
   "metadata": {},
   "outputs": [],
   "source": [
    "from importlib import reload\n",
    "import profiling_util\n",
    "reload(profiling_util)\n",
    "from profiling_util import *\n",
    "import segmentation.segmentation_pipelines\n",
    "reload(segmentation.segmentation_pipelines)\n",
    "from segmentation.segmentation_pipelines import extract_masks_adapted_time_profiling, save_as_memmap"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e928cf6e-f5eb-4d72-b8aa-015c52d79909",
   "metadata": {},
   "source": [
    "### define consolidated functions for running all code necessary for running segmentation pipelines inside of timing wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10bb9c19-a2df-46aa-8b5e-f32bd13058c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def caiman_analysis(scan, num_frames, num_calls, directory_modifier='', params=None):\n",
    "\n",
    "    if params is None:\n",
    "        print('Using default caiman params')\n",
    "        params = {'num_background_components': 1,\n",
    "        'merge_threshold': 0.7,\n",
    "        'fps': 8.3091,\n",
    "        'init_on_patches': True,\n",
    "        'proportion_patch_overlap': 0.2,\n",
    "        'num_components_per_patch': 6,\n",
    "        'init_method': 'greedy_roi',\n",
    "        'patch_size': [20.0, 20.0],\n",
    "        'soma_diameter': [3.2, 3.2],\n",
    "        'num_processes': 8,\n",
    "        'num_pixels_per_process': 10000}\n",
    "\n",
    "    print('Performing raster and motion correction for caiman')\n",
    "    scan = perform_raster_correction(scan, \n",
    "                                     temporal_fill_fraction=1, in_place=False)\n",
    "    scan = perform_motion_correction(scan, in_place=False)\n",
    "    print('Finished raster and motion correction')\n",
    "\n",
    "    if not os.path.isdir(f'segmentation/data/caiman/segmentation results{directory_modifier}'):\n",
    "            os.mkdir(f'segmentation/data/caiman/segmentation results{directory_modifier}')\n",
    "        \n",
    "    for frame_length in num_frames:\n",
    "    \n",
    "        test_scan = (scan[...,select_middle_frames(scan, frame_length)]).copy()\n",
    "        save_path = f'segmentation/data/caiman/segmentation results{directory_modifier}/frames{frame_length}'\n",
    "        if not os.path.isdir(save_path):\n",
    "            os.mkdir(save_path)\n",
    "        mmap_filename = save_as_memmap(test_scan, base_name=\"data/time_profiling_data/caiman_mmaps/caiman\").filename\n",
    "        mmap_scan, (image_height, image_width), num_frames = load_memmap(mmap_filename)\n",
    "        save_timing_results_file = f'data/time_profiling_data/caiman/{directory_modifier}average_profile_{frame_length}frames_{num_calls}runs.prof'\n",
    "        profile_function = lambda : extract_masks_adapted_time_profiling(test_scan, mmap_scan, **params)\n",
    "        result, individual_times, last_output = average_profilings(profile_function, num_calls, save_timing_results_file, save_seg_results=save_path)\n",
    "        #os.remove(mmap_scan)\n",
    "        print(f\"Averaged {num_calls} function calls, at a time of {round(result.total_tt, 2)} seconds per call. Program run times had a standard deviation of {round(std(individual_times), 3)} seconds.\")\n",
    "        print(f'Saved results in {save_timing_results_file}.')\n",
    "\n",
    "def s2p_analysis(scan, num_frames, num_calls, directory_modifier='', tau=1.3, fs=8.3091):\n",
    "    raw_scan = np.moveaxis(scan, -1, 0)\n",
    "    reg_scan = np.zeros_like(raw_scan)\n",
    "\n",
    "    # perform suite2p registration\n",
    "    ops = suite2p.default_ops()\n",
    "    ops['tau'] = tau\n",
    "    ops['fs'] = fs\n",
    "    #ops['data_path'] = ['./segmentation/data/tiffs/']\n",
    "    #ops['tiff_list'] = ['2scanMovieOneFieldAllFrames.tif']\n",
    "    ops['save_path0'] = f'segmentation/data/suite2p/segmentation results{directory_modifier}'\n",
    "    #ops['reg_tif'] = True\n",
    "    #ops = suite2p.io.tiff_to_binary(ops)\n",
    "    #f_reg2 = suite2p.io.BinaryFile(ops['Ly'],ops['Lx'], filename=ops['reg_file'], n_frames=ops['nframes'])\n",
    "    outputs = suite2p.registration_wrapper(f_reg=reg_scan, f_raw=raw_scan, ops=ops)\n",
    "\n",
    "    ops = suite2p.registration.save_registration_outputs_to_ops(outputs, ops)\n",
    "    meanImgE = suite2p.registration.compute_enhanced_mean_image(ops['meanImg'].astype(np.float32), ops)\n",
    "    ops['meanImgE'] = meanImgE\n",
    "\n",
    "    def run_s2p_save_path(ops, f_reg, classfile, run_num):\n",
    "        ops['save_path0'] = ops['save_path0'] + f'/test{run_num}'\n",
    "        if not os.path.isdir(ops['save_path0']):\n",
    "            os.mkdir(ops['save_path0'])\n",
    "        output_ops, stat = suite2p.detection_wrapper(f_reg=f_reg, ops=ops, classfile=classfile)\n",
    "        stat, F, Fneu, F_chan2, Fneu_chan2 = suite2p.extraction_wrapper(stat, f_reg, f_reg_chan2=None, ops=output_ops)\n",
    "        iscell = suite2p.classify(stat=stat, classfile=classfile)\n",
    "        dF = F.copy() - output_ops['neucoeff']*Fneu\n",
    "        dF = suite2p.extraction.preprocess(\n",
    "            F=dF,\n",
    "            baseline=output_ops['baseline'],\n",
    "            win_baseline=output_ops['win_baseline'],\n",
    "            sig_baseline=output_ops['sig_baseline'],\n",
    "            fs=output_ops['fs'],\n",
    "            prctile_baseline=output_ops['prctile_baseline']\n",
    "        )\n",
    "        spks = suite2p.extraction.oasis(F=dF, batch_size=output_ops['batch_size'], tau=output_ops['tau'], fs=output_ops['fs'])\n",
    "    \n",
    "        # save results\n",
    "        fpath = output_ops['save_path0']\n",
    "        np.save(os.path.join(fpath, 'stat.npy'), stat)\n",
    "        np.save(os.path.join(fpath, 'iscell.npy'), iscell)\n",
    "        np.save(os.path.join(fpath, 'F.npy'), F)\n",
    "        np.save(os.path.join(fpath, 'Fneu.npy'), Fneu)\n",
    "        np.save(os.path.join(fpath, 'spks.npy'), spks)\n",
    "        np.save(os.path.join(fpath, 'ops.npy'), output_ops)\n",
    "        print(f'saved ops.npy to {os.path.join(fpath, \"ops.npy\")}')\n",
    "\n",
    "    if not os.path.isdir(f'segmentation/data/suite2p/segmentation results{directory_modifier}'):\n",
    "            os.mkdir(f'segmentation/data/suite2p/segmentation results{directory_modifier}')\n",
    "    classfile = suite2p.classification.builtin_classfile\n",
    "    for frame_length in num_frames:\n",
    "\n",
    "        save_path = f'segmentation/data/suite2p/segmentation results{directory_modifier}/frames{frame_length}'\n",
    "        if not os.path.isdir(save_path):\n",
    "            os.mkdir(save_path)\n",
    "        #ops['frames_include'] = frame_length\n",
    "        ops['save_path0'] = save_path\n",
    "        save_timing_results_file = f'data/time_profiling_data/suite2p/{directory_modifier}_average_profile_{frame_length}frames_{num_calls}runs.prof'\n",
    "        profile_function = lambda index : run_s2p_save_path(ops.copy(), reg_scan[select_middle_frames(reg_scan[:,0,0], frame_length),:,:], classfile, index)\n",
    "        result, individual_times, last_output = average_profilings(profile_function, num_calls, save_timing_results_file, save_seg_results=False, input_index=True)\n",
    "        print(f\"Averaged {num_calls} function calls, at a time of {round(result.total_tt, 2)} seconds per call. Program run times had a standard deviation of {round(std(individual_times), 3)} seconds.\")\n",
    "        print(f'Saved results in {save_timing_results_file}.')\n",
    "\n",
    "def caiman_s2p_analysis(filename, frame_lengths, num_calls, directory_modifier='', skip_frames=0, caiman_params=None, s2p_params=(1.3, 8.3091), perform_caiman=True, perform_s2p=True):\n",
    "\n",
    "    print('Loading in scan')\n",
    "    if isinstance(filename, str):\n",
    "        scan = np.load(filename)\n",
    "    else:\n",
    "        scan = filename\n",
    "    if skip_frames > 0:\n",
    "        print(f'keeping every {skip_frames}th frame')\n",
    "        scan = scan[:,:,::skip_frames]\n",
    "    scan -= scan.min()\n",
    "    \n",
    "\n",
    "    interpret_last_frame_length = False\n",
    "    if frame_lengths[-1] < 0:\n",
    "        interpret_last_frame_length = True\n",
    "        frame_lengths[-1] = scan.shape[-1]\n",
    "\n",
    "    if perform_caiman:\n",
    "        print(f'Starting caiman analysis with frame lengths {frame_lengths}')\n",
    "        caiman_analysis(scan.copy(), frame_lengths, num_calls, directory_modifier, caiman_params)\n",
    "        print('Finished caiman analysis')\n",
    "    if perform_s2p:\n",
    "        print(f'Starting suite2p analysis with frame lengths {frame_lengths}')\n",
    "        s2p_analysis(scan, frame_lengths, num_calls, directory_modifier, s2p_params[0], s2p_params[1])\n",
    "        print('Done')\n",
    "\n",
    "    if interpret_last_frame_length:\n",
    "        frame_lengths[-1] = -1\n",
    "    \n",
    "def select_middle_frames(scan, return_frames, skip_rows=0, skip_cols=0):\n",
    "    # Load some frames from the middle of the scan\n",
    "    num_frames = scan.shape[-1]\n",
    "    middle_frame = int(np.floor(num_frames / 2))\n",
    "    frames = slice(max(middle_frame - int(return_frames/2), 0), middle_frame + int(return_frames/2))\n",
    "    #last_row = -scan.shape[0] if skip_rows == 0 else skip_rows\n",
    "    #last_col = -scan.shape[1] if skip_cols == 0 else skip_cols\n",
    "    #mini_scan = scan[skip_rows:-last_row, skip_cols:-last_col, frames]\n",
    "\n",
    "    return frames"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf93390a-782b-47e4-b173-eb555a941666",
   "metadata": {},
   "source": [
    "if reading from a pickled dataframe, load it in and pass it to the timing wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7392e453-17f7-407a-bb96-2d21ebcaeb56",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "# is there any way to read this dataframe in chunks (since it is very large)?\n",
    "with open('./raster_motion_miniscan_export01.pkl', 'rb') as f:\n",
    "    data = f.read()\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c5d5090-304d-4e79-8a1c-01d75d59998d",
   "metadata": {},
   "source": [
    "## This cell runs the entire wrapper functions defined above"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36e12e41-d369-4db6-bbc3-a1e924fede67",
   "metadata": {},
   "source": [
    "### parameters:\n",
    "#### filenames: list of files where data you want to analyze are saved. Can also pass in references to numpy arrays if the data is already loaded in\n",
    "#### frame_lengths: list of positive intgers defining how many frames the segmentation should be performed on. The last entry can be -1 if you want to analyze all frames in the recording\n",
    "#### num_calls: number of times to perform segmentation (all function calls will be averaged for timing purposes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e27adea-bfc5-4ada-bafb-83e3921158be",
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = ['./segmentation/test_scan1.npy', './segmentation/test_scan2.npy', './segmentation/onefield_miniscan.npy']\n",
    "frame_lengths = [500,1000,2000,3000,5000,7000, 10000,20000,-1]\n",
    "num_calls = 2\n",
    "skip_frames = [15,13,11,9,7,5,3,1]\n",
    "for i in range(2):\n",
    "    scan = np.load(filenames[i])\n",
    "    for skip_frame in skip_frames:\n",
    "        \n",
    "        caiman_s2p_analysis(scan[:,:,::skip_frame].copy(), frame_lengths=[-1], num_calls=num_calls, skip_frames=0, directory_modifier=f'scan{i+1}skip{skip_frame}')\n",
    "        #caiman_s2p_analysis(filenames[i], frame_lengths=[-1], num_calls, directory_modifier=f'scan{i+1}')\n",
    "# do I also need to include fps and calcium indicator data\n",
    "#for i in range(len(data)):\n",
    "#    try:\n",
    "#        caiman_params=data.loc[:, 'params'][i]\n",
    "#        print('pulled caiman params')\n",
    "#        print(caiman_params)\n",
    "#    except Exception:\n",
    "#        caiman_params=None\n",
    "#    caiman_s2p_analysis(data.loc[:, 'mini_scan'][i], frame_lengths, num_calls, directory_modifier=f'scan{i+1}', caiman_params=caiman_params) #, caiman_params=data.loc['params'][i]\n",
    "scan = np.load(filenames[2])\n",
    "for skip_frame in skip_frames:\n",
    "    caiman_s2p_analysis(scan[:,:,::skip_frame].copy(), frame_lengths=[-1], num_calls=num_calls, skip_frames=0, directory_modifier=f'skip{skip_frame}')\n",
    "#caiman_s2p_analysis(filenames[2], frame_lengths, num_calls=5, directory_modifier='')#,'original_file')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "420547a2-684d-458a-b5fc-a2bfdfc4bf53",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'segmentation/onefield_miniscan.npy'\n",
    "frame_lengths = [500,1000,2000,3000,5000,7000, 10000,20000,-1]\n",
    "num_calls = 5\n",
    "caiman_s2p_analysis(filename, frame_lengths, num_calls, directory_modifier='', perform_caiman=False)#,'original_file')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0205e31-e281-44a3-ac53-ad91ccfc47f1",
   "metadata": {},
   "source": [
    "Use helper function that will perform segmentation {num_calls} times and average the times. Results will be saved to the directory specified in average_profilings. The cell below contains the basic outline for profiling a function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "217f72cf-4804-4c39-b4f2-3c941e2aafb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_calls = 5\n",
    "profile_function = lambda : extract_masks_adapted_time_profiling(test_scan, mmap_scan, **params)\n",
    "result, individual_times, last_output = average_profilings(profile_function, num_calls, f'data/time_profiling_data/average_profile_{frame_length}frames_{num_calls}runs.prof')\n",
    "print(f\"Averaged {num_calls} function calls, at a time of {round(result.total_tt, 2)} seconds per call. Program run times had a standard deviation of {round(std(individual_times), 3)} seconds.\")\n",
    "print(f'Saved results in time_profiling_data/average_profile_{frame_length}frames_{num_calls}runs.prof.')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddf512f4-5ab2-4fc3-a74a-678ce91f1b34",
   "metadata": {},
   "source": [
    "The cell below is adapted to profile segmentation of variable length videos performed by CaImAn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "418deca6-1ace-4495-91d2-94f53d11b3f6",
   "metadata": {},
   "source": [
    "The cells below first perform suite2p registration and then profile segmentation by suite2p on variable length videos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4a1f5b7-91f0-4b9c-b0ab-f1ae9fa76974",
   "metadata": {},
   "outputs": [],
   "source": [
    "###### verify output of function\n",
    "import matplotlib.pyplot as plt\n",
    "masks, traces, background_masks, background_traces, raw_traces = last_output\n",
    "plt.imshow(masks.sum(axis=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02378eea-3f6f-4fdb-84fc-ea063eecae8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_profiling_params(individual_times, result.total_tt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ec0617d-3045-472a-859a-4ea4f8e311bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('masks_new.npy', masks)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9af598c-7e12-4e33-b4e0-3871ce3364e9",
   "metadata": {},
   "source": [
    "def print_plain(lines):\n",
    "    for line in lines:\n",
    "        print(line.replace('\\n', ''))\n",
    "print_plain([\n",
    "    \"# Timing profiling usage:\\n\",\n",
    "    \"    - Run \\\"average_proflings\\\" function above\\n\",\n",
    "    \"    - Run \\\"snakeviz time_profiling_data/average_profile_{num_calls}runs.prof\\\" to visualize output in browser\\n\",\n",
    "    \"    - If there are instances of multiple function call timings overlapping with each other, can call \\\"tuna data/average_profile_{num_calls}runs.prof\\\" to account for these inconsistencies\\n\",\n",
    "    \"        - tuna also gives percentages of total run time for each function call\\n\",\n",
    "    \"    - If using multiprocessing (such as in segmentation), may need to use viztracer: \\\"viztracer --multi-processing pipeline_timing.py\\\"\\n\",\n",
    "    \"        - View results with \\\"vizviewer --flamegraph result.json\\\"\\n\",\n",
    "    \"\\n\",\n",
    "    \"Memory profiling usage:\\n\",\n",
    "    \"    - two libraries\\n\",\n",
    "    \"        - memory_profiler: samples program periodically so you can track memory usage over course of program\\n\",\n",
    "    \"            - \\\"mprof run --python python pipeline_mprof.py\\\"\\n\",\n",
    "    \"            - \\\"mprof plot --flame\\\"\\n\",\n",
    "    \"                - can get additional granularity into which functions are taking up memory by adding more @profile decorators in pipeline_profiling.py file\\n\",\n",
    "    \"            - clean up files afterwards with \\\"mprof clean\\\"\\n\",\n",
    "    \"            - can run all steps together with \\\"python run_mprof.py\\\"\\n\",\n",
    "    \"            - can use \\\"multiprocess\\\" and \\\"--include-children\\\" flags for multiprocess programs\\n\",\n",
    "    \"        - memray: provides memory size of largest objects at peak memory usage\\n\",\n",
    "    \"            - \\\"memray run pipeline_memray.py\\\"\\n\",\n",
    "    \"                - use \\\"--follow-fork\\\" tag for multiprocessing tracking\\n\",\n",
    "    \"            - generate graph outlining largest memory allocations during peak memory usage: \\\"memray flamegraph [output file name from run operator]\\\"\\n\",\n",
    "    \"            - open graph in browser: \\\"open [flamegraph file name]\\\"\\n\",\n",
    "    \"            - also outputs interesting memory profiling statistics: \\\"memray stats [output file name from run operator]\\\"\\n\",\n",
    "    \"            - can run all steps together with \\\"python run_memray.py\\\"\\n\",\n",
    "    \"    - can clean up files afterwards by running cell below\\n\",\n",
    "    \"\\n\",\n",
    "    \"To run any/all of the above:\\n\",\n",
    "    \"    - in command line, run \\\"python run_profiling.py [-m] [-p] [-t] [# of runs to average]\\n\",\n",
    "    \"        - m runs memray, p runs mprof, and t runs the time profiling\\n\",\n",
    "    \"        - note: may need to exit out of matplotlib plots to let program continue running (but files also get saved, so no need to worry)\\n\",\n",
    "    \"\\n\",\n",
    "    \"Scaling concerns:\\n\",\n",
    "    \"    - does cProfile work when running with multiple processors?\\n\",\n",
    "    \"        - we will figure out later\\n\",\n",
    "    \"        - viztracer definitely works, so may need to switch over to that\\n\",\n",
    "    \"    - are memory profiling scripts too modularized?\\n\",\n",
    "    \"        - having too many subfunctions may increase memory consumption\\n\",\n",
    "    \"        - just keep skeleton of high level functions, similar to how script will be run in real usage\\n\",\n",
    "    \"        - leave deep modularization to the timing script\"\n",
    "   ])\n",
    "\n",
    "# Timing profiling usage:\n",
    "    - Run \"average_proflings\" function above\n",
    "    - Run \"snakeviz time_profiling_data/average_profile_{num_calls}runs.prof\" to visualize output in browser\n",
    "    - If there are instances of multiple function call timings overlapping with each other, can call \"tuna data/average_profile_{num_calls}runs.prof\" to account for these inconsistencies\n",
    "        - tuna also gives percentages of total run time for each function call\n",
    "    - If using multiprocessing (such as in segmentation), may need to use viztracer: \"viztracer --multi-processing pipeline_timing.py\"\n",
    "        - View results with \"vizviewer --flamegraph result.json\"\n",
    "\n",
    "Memory profiling usage:\n",
    "    - two libraries\n",
    "        - memory_profiler: samples program periodically so you can track memory usage over course of program\n",
    "            - \"mprof run --python python pipeline_mprof.py\"\n",
    "            - \"mprof plot --flame\"\n",
    "                - can get additional granularity into which functions are taking up memory by adding more @profile decorators in pipeline_profiling.py file\n",
    "            - clean up files afterwards with \"mprof clean\"\n",
    "            - can run all steps together with \"python run_mprof.py\"\n",
    "            - can use \"multiprocess\" and \"--include-children\" flags for multiprocess programs\n",
    "        - memray: provides memory size of largest objects at peak memory usage\n",
    "            - \"memray run pipeline_memray.py\"\n",
    "                - use \"--follow-fork\" tag for multiprocessing tracking\n",
    "            - generate graph outlining largest memory allocations during peak memory usage: \"memray flamegraph [output file name from run operator]\"\n",
    "            - open graph in browser: \"open [flamegraph file name]\"\n",
    "            - also outputs interesting memory profiling statistics: \"memray stats [output file name from run operator]\"\n",
    "            - can run all steps together with \"python run_memray.py\"\n",
    "    - can clean up files afterwards by running cell below\n",
    "\n",
    "To run any/all of the above:\n",
    "    - in command line, run \"python run_profiling.py [-m] [-p] [-t] [# of runs to average]\n",
    "        - m runs memray, p runs mprof, and t runs the time profiling\n",
    "        - note: may need to exit out of matplotlib plots to let program continue running (but files also get saved, so no need to worry)\n",
    "\n",
    "Scaling concerns:\n",
    "    - does cProfile work when running with multiple processors?\n",
    "        - we will figure out later\n",
    "        - viztracer definitely works, so may need to switch over to that\n",
    "    - are memory profiling scripts too modularized?\n",
    "        - having too many subfunctions may increase memory consumption\n",
    "        - just keep skeleton of high level functions, similar to how script will be run in real usage\n",
    "        - leave deep modularization to the timing script\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b40eaaca-2f43-4925-84d7-c491e0313e79",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}