From 06b0b7d22d983a7f165777edee23fba11dcec64e Mon Sep 17 00:00:00 2001 From: ericagriffith Date: Tue, 13 Dec 2022 17:11:16 -0500 Subject: [PATCH 01/65] commented out legendKwargs title fontsize small line to allow these functions to run properly --- netpyne/plotting/plotLFPPSD.py | 2 +- netpyne/plotting/plotLFPTimeSeries.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/netpyne/plotting/plotLFPPSD.py b/netpyne/plotting/plotLFPPSD.py index 1aa8cb4c0..9fe3a1f0d 100644 --- a/netpyne/plotting/plotLFPPSD.py +++ b/netpyne/plotting/plotLFPPSD.py @@ -355,7 +355,7 @@ def plotLFPPSD( #legendKwargs['borderaxespad'] = 0.0 #legendKwargs['handlelength'] = 0.5 legendKwargs['fontsize'] = 'small' - legendKwargs['title_fontsize'] = 'small' + # legendKwargs['title_fontsize'] = 'small' ## EYG commenting out to resolve error 12/06/22 # add the legend if legend: diff --git a/netpyne/plotting/plotLFPTimeSeries.py b/netpyne/plotting/plotLFPTimeSeries.py index 551b51b59..e0e507ea2 100644 --- a/netpyne/plotting/plotLFPTimeSeries.py +++ b/netpyne/plotting/plotLFPTimeSeries.py @@ -285,7 +285,7 @@ def plotLFPTimeSeries( #legendKwargs['borderaxespad'] = 0.0 #legendKwargs['handlelength'] = 0.5 legendKwargs['fontsize'] = 'small' - legendKwargs['title_fontsize'] = 'small' + # legendKwargs['title_fontsize'] = 'small' # add the legend if legend: From b3d61a8d1d02275212d92e698d0333239b625b7c Mon Sep 17 00:00:00 2001 From: ericagriffith Date: Wed, 14 Dec 2022 15:18:28 -0500 Subject: [PATCH 02/65] updating prepareCSD() function to more closely parallel prepareLFP() --- netpyne/analysis/csd.py | 365 ++++++++++++++++++++---------------- netpyne/plotting/plotCSD.py | 2 + 2 files changed, 201 insertions(+), 166 deletions(-) diff --git a/netpyne/analysis/csd.py b/netpyne/analysis/csd.py index 13eb5f231..7a9308e48 100644 --- a/netpyne/analysis/csd.py +++ b/netpyne/analysis/csd.py @@ -3,41 +3,141 @@ """ -from __future__ import print_function -from __future__ import division -from __future__ import unicode_literals -from __future__ import absolute_import +# from __future__ import print_function +# from __future__ import division +# from __future__ import unicode_literals +# from __future__ import absolute_import -from future import standard_library -standard_library.install_aliases() +# from future import standard_library +# standard_library.install_aliases() -try: - basestring -except NameError: - basestring = str +# try: +# basestring +# except NameError: +# basestring = str import numpy as np -import scipy - -import matplotlib -from matplotlib import pyplot as plt -from matplotlib import ticker as ticker -import json -import sys -import os -from collections import OrderedDict -import warnings -from scipy.fftpack import hilbert -from scipy.signal import cheb2ord, cheby2, convolve, get_window, iirfilter, remez, decimate +from numbers import Number +# import scipy + +# import matplotlib +# from matplotlib import pyplot as plt +# from matplotlib import ticker as ticker +# import json +# import sys +# import os +# from collections import OrderedDict +# import warnings +# from scipy.fftpack import hilbert +# from scipy.signal import cheb2ord, cheby2, convolve, get_window, iirfilter, remez, decimate from .filter import lowpass,bandpass from .utils import exception, _saveFigData +def getbandpass( + lfps, + sampr, + minf=0.05, + maxf=300): + """ + Function to bandpass filter data + + Parameters + ---------- + lfps : list or array + LFP signal data arranged spatially in a column. + **Default:** *required* + + sampr : float + The data sampling rate. + **Default:** *required* + + minf : float + The high-pass filter frequency (Hz). + **Default:** ``0.05`` + + maxf : float + The low-pass filter frequency (Hz). + **Default:** ``300`` + + + Returns + ------- + data : array + The bandpass-filtered data. + + """ + + datband = [] + for i in range(len(lfps[0])):datband.append(bandpass(lfps[:,i], minf, maxf, df=sampr, zerophase=True)) + datband = np.array(datband) + return datband + +def Vaknin(x): + """ + Function to perform the Vaknin correction for CSD analysis + + Allows CSD to be performed on all N contacts instead of N-2 contacts (see Vaknin et al (1988) for more details). + + Parameters + ---------- + x : array + Data to be corrected. + **Default:** *required* + + + Returns + ------- + data : array + The corrected data. + + """ + + # Preallocate array with 2 more rows than input array + x_new = np.zeros((x.shape[0]+2, x.shape[1])) + + # Duplicate first and last row of x into first and last row of x_new + x_new[0, :] = x[0, :] + x_new[-1, :] = x[-1, :] + + # Duplicate all of x into middle rows of x_neww + x_new[1:-1, :] = x + + return x_new + +def removemean(x, ax=1): + """ + Function to subtract the mean from an array or list + + Parameters + ---------- + x : array + Data to be processed. + **Default:** *required* + + ax : int + The axis to remove the mean across. + **Default:** ``1`` + + + Returns + ------- + data : array + The processed data. + + """ + + mean = np.mean(x, axis=ax, keepdims=True) + x -= mean + + @exception def prepareCSD( sim=None, + electrodes=['avg', 'all'], + timeRange=None, pop=None, dt=None, sampr=None, @@ -46,8 +146,6 @@ def prepareCSD( maxf=300, vaknin=True, norm=False, - saveData=True, - getAllData=True, **kwargs): @@ -59,6 +157,13 @@ def prepareCSD( sim : NetPyNE object **Default:** ``None`` + electrodes : list of electrodes to look at CSD data + **Default:** ['avg', 'all'] + + timeRange: list + List of length two, with timeRange[0] as beginning of desired timeRange, and timeRange[1] as the end + **Default:** ``None`` retrieves timeRange = [0, sim.cfg.duration] + pop : str Retrieves CSD data from a specific cell population **Default:** ``None`` retrieves overall CSD data @@ -89,15 +194,8 @@ def prepareCSD( norm : bool Subtracts the mean from the CSD data - **Default:** ``False`` - - saveData : bool - Saves CSD data to sim object - **Default:** ``True`` + **Default:** ``False`` --> ``True`` - getAllData : bool - Returns CSDData as well as LFPData, sampr, spacing_um, and dt - **Default:** ``True`` """ print('Preparing CSD data... ') @@ -109,56 +207,44 @@ def prepareCSD( raise Exception('Cannot access sim') - ## Get LFP data from sim and instantiate as a numpy array - simDataCategories = sim.allSimData.keys() + # ## Get LFP data from sim and instantiate as a numpy array + # simDataCategories = sim.allSimData.keys() - if pop is None: - if 'LFP' in simDataCategories: - LFPData = np.array(sim.allSimData['LFP']) - else: - raise Exception('NO LFP DATA!! Need to re-run simulation with cfg.recordLFP enabled') + # if pop is None: + # if 'LFP' in simDataCategories: + # LFPData = np.array(sim.allSimData['LFP']) + # else: + # raise Exception('NO LFP DATA!! Need to re-run simulation with cfg.recordLFP enabled') + # else: + # if 'LFPPops' in simDataCategories: + # simLFPPops = sim.allSimData['LFPPops'].keys() + # if pop in simLFPPops: + # LFPData = sim.allSimData['LFPPops'][pop] + # else: + # raise Exception('No LFP data for ' + str(pop) + ' cell pop; CANNOT GENERATE CSD DATA OR PLOT') + # else: + # raise Exception('No pop-specific LFP data recorded! CANNOT GENERATE POP-SPECIFIC CSD DATA OR PLOT') + + + # Retrieve LFP data from sim + if pop and pop in sim.allSimData['LFPPops']: + lfpData = np.array(sim.allSimData['LFPPops'][pop]) else: - if 'LFPPops' in simDataCategories: - simLFPPops = sim.allSimData['LFPPops'].keys() - if pop in simLFPPops: - LFPData = sim.allSimData['LFPPops'][pop] - else: - raise Exception('No LFP data for ' + str(pop) + ' cell pop; CANNOT GENERATE CSD DATA OR PLOT') - else: - raise Exception('No pop-specific LFP data recorded! CANNOT GENERATE POP-SPECIFIC CSD DATA OR PLOT') + lfpData = np.array(sim.allSimData['LFP']) # time step used in simulation recording (in ms) if dt is None: dt = sim.cfg.recordStep - # Sampling rate of data recording during the simulation + # Calculate the sampling rate of data recording during the simulation if sampr is None: # divide by 1000.0 to turn denominator from units of ms to s sampr = 1.0/(dt/1000.0) # dt == sim.cfg.recordStep, unless specified otherwise by user - # Spacing between electrodes (in microns) - if spacing_um is None: - spacing_um = sim.cfg.recordLFP[1][1] - sim.cfg.recordLFP[0][1] - - # Convert spacing from microns to mm - spacing_mm = spacing_um/1000 - - print('dt, sampr, spacing_um, spacing_mm values determined') - - - ## This retrieves: - # LFPData (as an array) - # dt --> recording time step (in ms) - # sampr --> sampling rate of data recording (in Hz) - # spacing_um --> spacing btwn electrodes (in um) - - - #################################### - # Bandpass filter the LFP data with getbandpass() fx defined above - datband = getbandpass(LFPData,sampr,minf,maxf) + datband = getbandpass(lfpData,sampr,minf,maxf) # Take CSD along smaller dimension if datband.shape[0] > datband.shape[1]: @@ -170,126 +256,73 @@ def prepareCSD( if vaknin: datband = Vaknin(datband) + # norm data if norm: removemean(datband,ax=ax) - # now each column (or row) is an electrode -- take CSD along electrodes - CSDData = -np.diff(datband, n=2, axis=ax)/spacing_mm**2 - - - ##### SAVE DATA ####### - # Add CSDData to sim.allSimData for later access - if saveData: - if pop is None: - sim.allSimData['CSD'] = CSDData - else: - sim.allSimData['CSDPops'] = {} - sim.allSimData['CSDPops'][pop] = CSDData - - # return CSD_data or all data - if getAllData is True: - return CSDData, LFPData, sampr, spacing_um, dt - elif getAllData is False: - return CSDData - - -def getbandpass( - lfps, - sampr, - minf=0.05, - maxf=300): - """ - Function to bandpass filter data - - Parameters - ---------- - lfps : list or array - LFP signal data arranged spatially in a column. - **Default:** *required* - - sampr : float - The data sampling rate. - **Default:** *required* - - minf : float - The high-pass filter frequency (Hz). - **Default:** ``0.05`` - - maxf : float - The low-pass filter frequency (Hz). - **Default:** ``300`` - - Returns - ------- - data : array - The bandpass-filtered data. - - """ - - datband = [] - for i in range(len(lfps[0])):datband.append(bandpass(lfps[:,i], minf, maxf, df=sampr, zerophase=True)) + # Spacing between electrodes (in microns) + if spacing_um is None: + spacing_um = sim.cfg.recordLFP[1][1] - sim.cfg.recordLFP[0][1] + + # Convert spacing from microns to mm + spacing_mm = spacing_um/1000 - datband = np.array(datband) - return datband -def Vaknin(x): - """ - Function to perform the Vaknin correction for CSD analysis + # now each column (or row) is an electrode -- take CSD along electrodes + csdData = -np.diff(datband, n=2, axis=ax)/spacing_mm**2 - Allows CSD to be performed on all N contacts instead of N-2 contacts (see Vaknin et al (1988) for more details). - Parameters - ---------- - x : array - Data to be corrected. - **Default:** *required* + # Splice CSD data by timeRange, if relevant + if timeRange is None: + timeRange = [0, sim.cfg.recordStep] + else: + # lfpData = lfpData[int(timeRange[0]/sim.cfg.recordStep):int(timeRange[1]/sim.cfg.recordStep),:] + csdData = csdData[:,int(timeRange[0]/sim.cfg.recordStep):int(timeRange[1]/sim.cfg.recordStep)] - Returns - ------- - data : array - The corrected data. + # create the output data dictionary + data = {} + data['electrodes'] = {} + data['electrodes']['names'] = [] + data['electrodes']['locs'] = [] + data['electrodes']['csds'] = [] - """ + # create an array of the time steps and store in output data + t = np.arange(timeRange[0], timeRange[1], sim.cfg.recordStep) + data['t'] = t - # Preallocate array with 2 more rows than input array - x_new = np.zeros((x.shape[0]+2, x.shape[1])) - # Duplicate first and last row of x into first and last row of x_new - x_new[0, :] = x[0, :] - x_new[-1, :] = x[-1, :] + # electrode preparation (add average if needed) + if 'all' in electrodes: + electrodes.remove('all') + electrodes.extend(list(range(int(sim.net.recXElectrode.nsites)))) - # Duplicate all of x into middle rows of x_neww - x_new[1:-1, :] = x + for i, elec in enumerate(electrodes): + if elec == 'avg': + csdSignal = np.mean(csdData, axis=0) + loc = None + elif isinstance(elec, Number) and elec <= sim.net.recXElectrode.nsites: + csdSignal = csdData[elec,:] + loc = sim.cfg.recordLFP[elec] - return x_new -def removemean(x, ax=1): - """ - Function to subtract the mean from an array or list + data['electrodes']['names'].append(str(elec)) + data['electrodes']['locs'].append(loc) + data['electrodes']['csds'].append(csdSignal) ## <-- this can be turned into an array with np.array(data['electrodes']['csds'] -- NOTE: first row will be average -- data['electrodes']['csds'][0]) - Parameters - ---------- - x : array - Data to be processed. - **Default:** *required* + ## testing line + data['csdData'] = csdData + ### NOTE: + ### csd = np.array(data['electrodes']['csds']) + ### csd = np.array(csd) + ### csd[1:, :] == csdData['csdData'] ---> True, True, True... True - ax : int - The axis to remove the mean across. - **Default:** ``1`` + return data - Returns - ------- - data : array - The processed data. - """ - - mean = np.mean(x, axis=ax, keepdims=True) - x -= mean diff --git a/netpyne/plotting/plotCSD.py b/netpyne/plotting/plotCSD.py index 0bfde0d62..bcfd4e7a8 100644 --- a/netpyne/plotting/plotCSD.py +++ b/netpyne/plotting/plotCSD.py @@ -16,6 +16,7 @@ def plotCSD( dt=None, sampr=None, spacing_um=None, + norm=True, fontSize=12, ymax=None, figSize=(8,8), @@ -123,6 +124,7 @@ def plotCSD( dt=dt, sampr=sampr, spacing_um=spacing_um, + norm=True, getAllData=True, **kwargs) From a0618cfa859caa9e04d6f29e8047f1c2604d20ca Mon Sep 17 00:00:00 2001 From: ericagriffith Date: Wed, 14 Dec 2022 15:52:29 -0500 Subject: [PATCH 03/65] adding preparePSDCSD() function to csd script in netpyne/analysis --- netpyne/analysis/csd.py | 121 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/netpyne/analysis/csd.py b/netpyne/analysis/csd.py index 7a9308e48..7650feabd 100644 --- a/netpyne/analysis/csd.py +++ b/netpyne/analysis/csd.py @@ -323,6 +323,127 @@ def prepareCSD( return data +@exception +def preparePSDCSD( + csdData=None, + sim=None, + timeRange=None, + electrodes=['avg', 'all'], + pop=None, + minFreq=1, + maxFreq=100, + stepFreq=1, + normSignal=False, + normPSD=False, + transformMethod='morlet', + **kwargs + ): + + """ + Function to prepare data for plotting of power spectral density (PSD) of current source density (CSD) + """ + + + ## OLD ARGS --> + # NFFT=256, + # noverlap=128, + # nperseg=256, + # smooth=0, + # logy=False, + ## --> These were args in preparePSD for LFP but I bet these won't be relevant --> + # filtFreq=False, + # filtOrder=3, + # detrend=False, + + + + if not sim: + from .. import sim + + + data = prepareCSD( + sim=sim, + electrodes=electrodes, + timeRange=timeRange, + pop=pop, + **kwargs) + + # prepareCSD args covered under kwargs --> + # dt=dt, + # sampr=sampr, + # spacing_um=spacing_um, + # minf=minf, + # maxf=maxf, + # vaknin=vaknin, + # norm=norm, + + print('Preparing PSD CSD data...') + + + names = data['electrodes']['names'] + csds = data['electrodes']['csds'] # KEEP IN MIND THAT THE FIRST [0] IS AVERAGE ELECTRODE!!! + + allFreqs = [] + allSignal = [] + allNames = [] + + + # Used in both transforms + fs = int(1000.0/sim.cfg.recordStep) + + + for index, csd in enumerate(csds): + + # Morlet wavelet transform method + if transformMethod == 'morlet': + + from ..support.morlet import MorletSpec, index2ms + morletSpec = MorletSpec(csd, fs, freqmin=minFreq, freqmax=maxFreq, freqstep=stepFreq) + freqs = morletSpec.f + spec = morletSpec.TFR + signal = np.mean(spec, 1) + ylabel = 'Power' + + + allFreqs.append(freqs) + allSignal.append(signal) + allNames.append(names[index]) + + if normPSD: + vmax = np.max(allSignal) + for index, signal in enumerate(allSignal): + allSignal[index] = allSignal[index]/vmax + + + psdFreqs = [] + psdSignal = [] + + for index, name in enumerate(names): + freqs = allFreqs[index] + signal = allSignal[index] + + psdFreqs.append(freqs[freqs Date: Wed, 14 Dec 2022 15:53:35 -0500 Subject: [PATCH 04/65] importing preparePSDCSD from csd.py into init in analysis --- netpyne/analysis/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/netpyne/analysis/__init__.py b/netpyne/analysis/__init__.py index 8346fec1b..d28e6fb45 100644 --- a/netpyne/analysis/__init__.py +++ b/netpyne/analysis/__init__.py @@ -42,6 +42,7 @@ # Import CSD-related functions #from .csd import getCSD, plotCSD from .csd import prepareCSD +from .csd import preparePSDCSD # Import dipole-related functions from .dipole import plotDipole, plotEEG From ffd1ed5ef0ee697deff64c8147aef19fd8825443 Mon Sep 17 00:00:00 2001 From: ericagriffith Date: Wed, 14 Dec 2022 16:06:06 -0500 Subject: [PATCH 05/65] changing name of psd function to prepareCSDPSD --- netpyne/analysis/__init__.py | 2 +- netpyne/analysis/csd.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/netpyne/analysis/__init__.py b/netpyne/analysis/__init__.py index d28e6fb45..41a71cb18 100644 --- a/netpyne/analysis/__init__.py +++ b/netpyne/analysis/__init__.py @@ -42,7 +42,7 @@ # Import CSD-related functions #from .csd import getCSD, plotCSD from .csd import prepareCSD -from .csd import preparePSDCSD +from .csd import prepareCSDPSD # Import dipole-related functions from .dipole import plotDipole, plotEEG diff --git a/netpyne/analysis/csd.py b/netpyne/analysis/csd.py index 7650feabd..5046c933d 100644 --- a/netpyne/analysis/csd.py +++ b/netpyne/analysis/csd.py @@ -324,7 +324,7 @@ def prepareCSD( @exception -def preparePSDCSD( +def prepareCSDPSD( csdData=None, sim=None, timeRange=None, @@ -433,6 +433,26 @@ def preparePSDCSD( return data +### FUTURE WORK --> DEFINE FUNCTION prepareSpectrogram() ### +# @exception +# def prepareSpectrogram( +# sim=None, +# timeRange=None, +# electrodes=['avg', 'all'], +# pop=None, +# csdData=None, +# minFreq=1, +# maxFreq=100, +# stepFreq=1, +# normSignal=False, +# normPSD=False, +# normSpec=False, +# transformMethod='morlet', +# **kwargs): + +# """ +# Function to prepare data for plotting of the spectrogram +# """ From 6f0d73fce751684f05eef984bbd6328d4a3a2ccb Mon Sep 17 00:00:00 2001 From: ericagriffith Date: Wed, 14 Dec 2022 16:35:49 -0500 Subject: [PATCH 06/65] fixed small timeRange bug in prepareCSD and added plotCSDTimeSeries to init.py in plotting --- netpyne/analysis/csd.py | 18 +++++++++--------- netpyne/plotting/__init__.py | 3 +++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/netpyne/analysis/csd.py b/netpyne/analysis/csd.py index 5046c933d..0b388cf8f 100644 --- a/netpyne/analysis/csd.py +++ b/netpyne/analysis/csd.py @@ -272,15 +272,15 @@ def prepareCSD( # now each column (or row) is an electrode -- take CSD along electrodes - csdData = -np.diff(datband, n=2, axis=ax)/spacing_mm**2 + CSDData = -np.diff(datband, n=2, axis=ax)/spacing_mm**2 # Splice CSD data by timeRange, if relevant if timeRange is None: - timeRange = [0, sim.cfg.recordStep] + timeRange = [0, sim.cfg.duration] else: # lfpData = lfpData[int(timeRange[0]/sim.cfg.recordStep):int(timeRange[1]/sim.cfg.recordStep),:] - csdData = csdData[:,int(timeRange[0]/sim.cfg.recordStep):int(timeRange[1]/sim.cfg.recordStep)] + CSDData = CSDData[:,int(timeRange[0]/sim.cfg.recordStep):int(timeRange[1]/sim.cfg.recordStep)] # create the output data dictionary @@ -302,10 +302,10 @@ def prepareCSD( for i, elec in enumerate(electrodes): if elec == 'avg': - csdSignal = np.mean(csdData, axis=0) + csdSignal = np.mean(CSDData, axis=0) loc = None elif isinstance(elec, Number) and elec <= sim.net.recXElectrode.nsites: - csdSignal = csdData[elec,:] + csdSignal = CSDData[elec,:] loc = sim.cfg.recordLFP[elec] @@ -314,18 +314,18 @@ def prepareCSD( data['electrodes']['csds'].append(csdSignal) ## <-- this can be turned into an array with np.array(data['electrodes']['csds'] -- NOTE: first row will be average -- data['electrodes']['csds'][0]) ## testing line - data['csdData'] = csdData + data['CSDData'] = CSDData ### NOTE: ### csd = np.array(data['electrodes']['csds']) ### csd = np.array(csd) - ### csd[1:, :] == csdData['csdData'] ---> True, True, True... True + ### csd[1:, :] == CSDData['CSDData'] ---> True, True, True... True return data @exception def prepareCSDPSD( - csdData=None, + CSDData=None, sim=None, timeRange=None, electrodes=['avg', 'all'], @@ -440,7 +440,7 @@ def prepareCSDPSD( # timeRange=None, # electrodes=['avg', 'all'], # pop=None, -# csdData=None, +# CSDData=None, # minFreq=1, # maxFreq=100, # stepFreq=1, diff --git a/netpyne/plotting/__init__.py b/netpyne/plotting/__init__.py index 1c19ad26b..353cdf1af 100644 --- a/netpyne/plotting/__init__.py +++ b/netpyne/plotting/__init__.py @@ -24,6 +24,9 @@ from .plotShape import plotShape from .plotCSD import plotCSD +# NEW CSD PLOTTING FUNCTIONS TESTS!! +from .plotCSDTimeSeries import plotCSDTimeSeries + """ # ------------------------------------------------------------------------------------------------------------------- From daa665c71fbba899263898c569b0633300e296f7 Mon Sep 17 00:00:00 2001 From: ericagriffith Date: Wed, 14 Dec 2022 16:36:21 -0500 Subject: [PATCH 07/65] adding a new script that will plot the time series of the CSD data, similar to plotLFPTimeSeries --- netpyne/plotting/plotCSDTimeSeries.py | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 netpyne/plotting/plotCSDTimeSeries.py diff --git a/netpyne/plotting/plotCSDTimeSeries.py b/netpyne/plotting/plotCSDTimeSeries.py new file mode 100644 index 000000000..1d4b99cf3 --- /dev/null +++ b/netpyne/plotting/plotCSDTimeSeries.py @@ -0,0 +1,73 @@ +# Generate plots of CSD (current source density) and related analyses + + +from ..analysis.utils import exception + + +@exception +def plotCSDTimeSeries( + CSDData=None, + timeRange=None, + electrodes=['avg', 'all'], + pop=None, + **kwargs): + + """Function to produce a line plot of CSD electrode signals""" + + ### args I'm not sure what to do with yet --> + # axis=None, + + + + + # If there is no input data, get the data from the NetPyNE sim object + if CSDData is None: + if 'sim' not in kwargs: + from .. import sim + else: + sim = kwargs['sim'] + + CSDData = sim.analysis.prepareCSD( + sim=sim, + electrodes=electrodes, + timeRange=timeRange, + pop=pop, + **kwargs) + + ### ARGS THAT WILL PROBABLY / SHOULD BE COVERED UNDER KWARGS --> + # dt=None, + # sampr=None, + # spacing_um=None, + # minf=0.05, + # maxf=300, + # vaknin=True, + # norm=False, + + print('Plotting CSD time series...') + + + # If input is a dictionary, pull the data out of it + if type(CSDData) == dict: + + t = CSDData['t'] + csds = CSDData['electrodes']['csds'] + names = CSDData['electrodes']['names'] + locs = CSDData['electrodes']['locs'] + colors = CSDData.get('colors') + linewidths = CSDData.get('linewidths') + alphas = CSDData.get('alphas') + axisArgs = CSDData.get('axisArgs') + + # testData = {'colors': colors, 'linewidths': linewidths, 'alphas': alphas, 'axisArgs': axisArgs, + # 't': t, 'csds': csds, 'names': names, 'locs': locs} + + # return testData + + + + + + + + + From 4215d89333585e57e29b7e1f82d803dce98f8023 Mon Sep 17 00:00:00 2001 From: ericagriffith Date: Wed, 14 Dec 2022 17:00:06 -0500 Subject: [PATCH 08/65] expanded the csd time series plotting functions to produce a line plot similar to lfp time series plots --- netpyne/analysis/csd.py | 8 +- netpyne/plotting/plotCSDTimeSeries.py | 220 ++++++++++++++++++++++++-- 2 files changed, 214 insertions(+), 14 deletions(-) diff --git a/netpyne/analysis/csd.py b/netpyne/analysis/csd.py index 0b388cf8f..8bc842811 100644 --- a/netpyne/analysis/csd.py +++ b/netpyne/analysis/csd.py @@ -136,8 +136,8 @@ def removemean(x, ax=1): @exception def prepareCSD( sim=None, - electrodes=['avg', 'all'], timeRange=None, + electrodes=['avg', 'all'], pop=None, dt=None, sampr=None, @@ -157,13 +157,13 @@ def prepareCSD( sim : NetPyNE object **Default:** ``None`` - electrodes : list of electrodes to look at CSD data - **Default:** ['avg', 'all'] - timeRange: list List of length two, with timeRange[0] as beginning of desired timeRange, and timeRange[1] as the end **Default:** ``None`` retrieves timeRange = [0, sim.cfg.duration] + electrodes : list of electrodes to look at CSD data + **Default:** ['avg', 'all'] + pop : str Retrieves CSD data from a specific cell population **Default:** ``None`` retrieves overall CSD data diff --git a/netpyne/plotting/plotCSDTimeSeries.py b/netpyne/plotting/plotCSDTimeSeries.py index 1d4b99cf3..7aa22642d 100644 --- a/netpyne/plotting/plotCSDTimeSeries.py +++ b/netpyne/plotting/plotCSDTimeSeries.py @@ -2,20 +2,102 @@ from ..analysis.utils import exception - +import numpy as np +from .plotter import LinesPlotter @exception def plotCSDTimeSeries( CSDData=None, + axis=None, timeRange=None, electrodes=['avg', 'all'], pop=None, + separation=1.0, + orderInverse=True, + overlay=False, + scalebar=True, + legend=True, + colorList=None, + returnPlotter=False, **kwargs): - """Function to produce a line plot of CSD electrode signals""" + """" + NetPyNE Options + --------------- + sim : NetPyNE sim object + The *sim object* from which to get data. + + *Default:* ``None`` uses the current NetPyNE sim object + + + Parameters + ---------- + CSDData : dict, str + The data necessary to plot the CSD signals. + + *Default:* ``None`` uses ``analysis.prepareCSD`` to produce ``CSDData`` using the current NetPyNE sim object. + + If a *str* it must represent a file path to previously saved data. + + axis : matplotlib axis + The axis to plot into, allowing overlaying of plots. + + *Default:* ``None`` produces a new figure and axis. + + timeRange : list + Time range to include in the raster: ``[min, max]``. + + *Default:* ``None`` uses the entire simulation + + electrodes : list + A *list* of the electrodes to plot from. + + *Default:* ``['avg', 'all']`` plots each electrode as well as their average + + pop : str + A population name to calculate signals from. + + *Default:* ``None`` uses all populations. + + separation : float + Use to increase or decrease distance between signals on the plot. + + *Default:* ``1.0`` + + orderInverse : bool + Whether to invert the order of plotting. + + *Default:* ``True`` + + overlay : bool + Option to label signals with a color-matched overlay. - ### args I'm not sure what to do with yet --> - # axis=None, + *Default:* ``False`` + + scalebar : bool + Whether to to add a scalebar to the plot. + + *Default:* ``True`` + + legend : bool + Whether or not to add a legend to the plot. + + *Default:* ``True`` adds a legend. + + colorList : list + A *list* of colors to draw from when plotting. + + *Default:* ``None`` uses the default NetPyNE colorList. + + returnPlotter : bool + Whether to return the figure or the NetPyNE MetaFig object. + + *Default:* ``False`` returns the figure. + """ + + + + """Function to produce a line plot of CSD electrode signals""" @@ -29,8 +111,8 @@ def plotCSDTimeSeries( CSDData = sim.analysis.prepareCSD( sim=sim, - electrodes=electrodes, timeRange=timeRange, + electrodes=electrodes, pop=pop, **kwargs) @@ -43,6 +125,8 @@ def plotCSDTimeSeries( # vaknin=True, # norm=False, + + print('Plotting CSD time series...') @@ -58,15 +142,131 @@ def plotCSDTimeSeries( alphas = CSDData.get('alphas') axisArgs = CSDData.get('axisArgs') - # testData = {'colors': colors, 'linewidths': linewidths, 'alphas': alphas, 'axisArgs': axisArgs, - # 't': t, 'csds': csds, 'names': names, 'locs': locs} - - # return testData - + # Set up colors, linewidths, and alphas for the plots + if not colors: + if not colorList: + from .plotter import colorList + colors = colorList[0:len(csds)] + + if not linewidths: + linewidths = [1.0 for csd in csds] + + if not alphas: + alphas = [1.0 for csd in csds] + + + + # Create a dictionary to hold axis inputs + if not axisArgs: + axisArgs = {} + title = 'CSD Time Series Plot' + if pop: + title += ' - Population: ' + pop + axisArgs['title'] = title + axisArgs['xlabel'] = 'Time (ms)' + axisArgs['ylabel'] = 'CSD Signal (mV/(mm^2))' + + + # Link colors to traces, make avg plot black, add separation to traces + plotColors = [] + legendLabels = [] + colorIndex = 0 + offset = np.absolute(csds).max() * separation + + for index, (name, csd) in enumerate(zip(names, csds)): + legendLabels.append(name) + csds[index] = index * offset + csd + if orderInverse: + csds[index] = index * offset - csd + axisArgs['invert_yaxis'] = True + else: + csds[index] = index * offset + csd + if name == 'avg': + plotColors.append('black') + else: + plotColors.append(colors[colorIndex]) + colorIndex += 1 + + + # Create a dictionary with the inputs for a line plot + linesData = {} + linesData['x'] = t + linesData['y'] = csds + linesData['color'] = plotColors + linesData['marker'] = None + linesData['markersize'] = None + linesData['linewidth'] = linewidths + linesData['alpha'] = alphas + linesData['label'] = legendLabels + + + # If a kwarg matches a lines input key, use the kwarg value instead of the default + for kwarg in list(kwargs.keys()): + if kwarg in linesData: + linesData[kwarg] = kwargs[kwarg] + kwargs.pop(kwarg) + + # create Plotter object + linesPlotter = LinesPlotter(data=linesData, kind='CSDTimeSeries', axis=axis, **axisArgs, **kwargs) + metaFig = linesPlotter.metafig + + + # Set up the default legend settings + legendKwargs = {} + legendKwargs['title'] = 'Electrodes' + legendKwargs['loc'] = 'upper right' + legendKwargs['fontsize'] = 'small' + + + # add the legend + if legend: + axisArgs['legend'] = legendKwargs + + # add the scalebar + if scalebar: + args = {} + args['hidey'] = True + args['matchy'] = True + args['hidex'] = False + args['matchx'] = False + args['sizex'] = 0 + args['sizey'] = 1.0 + args['ymax'] = 0.25 * offset + args['unitsy'] = 'mm - FIX' #'$\mu$V' + args['scaley'] = 1000.0 + args['loc'] = 3 + args['pad'] = 0.5 + args['borderpad'] = 0.5 + args['sep'] = 3 + args['prop'] = None + args['barcolor'] = "black" + args['barwidth'] = 2 + args['space'] = 0.25 * offset + + axisArgs['scalebar'] = args + + + if overlay: + kwargs['tightLayout'] = False + for index, name in enumerate(names): + linesPlotter.axis.text(t[0]-0.07*(t[-1]-t[0]), (index*offset), name, color=plotColors[index], ha='center', va='center', fontsize='large', fontweight='bold') + linesPlotter.axis.spines['top'].set_visible(False) + linesPlotter.axis.spines['right'].set_visible(False) + linesPlotter.axis.spines['left'].set_visible(False) + if len(names) > 1: + linesPlotter.fig.text(0.05, 0.4, 'CSD electrode', color='k', ha='left', va='bottom', fontsize='large', rotation=90) + + # Generate the figure + CSDTimeSeriesPlot = linesPlotter.plot(**axisArgs, **kwargs) + # Default is to return the figure, but you can also return the plotter + if returnPlotter: + return metaFig + else: + return CSDTimeSeriesPlot From 31c751ffd3e2e5b3778f5762b1e5a8109ba57bbe Mon Sep 17 00:00:00 2001 From: ericagriffith Date: Wed, 14 Dec 2022 17:22:12 -0500 Subject: [PATCH 09/65] adding script to plot CSD PSD, similar to LFP PSD plotting script --- netpyne/plotting/__init__.py | 1 + netpyne/plotting/plotCSDPSD.py | 104 +++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 netpyne/plotting/plotCSDPSD.py diff --git a/netpyne/plotting/__init__.py b/netpyne/plotting/__init__.py index 353cdf1af..b1b07ac49 100644 --- a/netpyne/plotting/__init__.py +++ b/netpyne/plotting/__init__.py @@ -26,6 +26,7 @@ # NEW CSD PLOTTING FUNCTIONS TESTS!! from .plotCSDTimeSeries import plotCSDTimeSeries +from .plotCSDPSD import plotCSDPSD """ diff --git a/netpyne/plotting/plotCSDPSD.py b/netpyne/plotting/plotCSDPSD.py new file mode 100644 index 000000000..c5afd030a --- /dev/null +++ b/netpyne/plotting/plotCSDPSD.py @@ -0,0 +1,104 @@ +# Generate plots of CSD PSD and related analyses + + +from ..analysis.utils import exception + +@exception +def plotCSDPSD( + PSDData=None, + timeRange=None, + electrodes=['avg', 'all'], + pop=None, + minFreq=1, + maxFreq=100, + stepFreq=1, + normSignal=False, + normPSD=False, + transformMethod='morlet', + colorList=None, + **kwargs): + + """Function to produce a plot of CSD Power Spectral Density (PSD) data + + NetPyNE Options + --------------- + sim : NetPyNE sim object + The *sim object* from which to get data. + + *Default:* ``None`` uses the current NetPyNE sim object + + Parameters + ---------- + + colorList : list + A *list* of colors to draw from when plotting. + + *Default:* ``None`` uses the default NetPyNE colorList. + + + Returns + ------- + LFPPSDPlot : *matplotlib figure* + By default, returns the *figure*. If ``returnPlotter`` is ``True``, instead returns the NetPyNE MetaFig. + """ + + + # If there is no input data, get the data from the NetPyNE sim object + if PSDData is None: + if 'sim' not in kwargs: + from .. import sim + else: + sim = kwargs['sim'] + + PSDData = sim.analysis.prepareCSDPSD( + CSDData=None, + sim=sim, + timeRange=timeRange, + electrodes=electrodes, + pop=pop, + minFreq=minFreq, + maxFreq=maxFreq, + stepFreq=stepFreq, + normSignal=normSignal, + normPSD=normPSD, + transformMethod=transformMethod, + **kwargs + ) + + # CSDData=None, + # sim=None, + # timeRange=None, + # electrodes=['avg', 'all'], + # pop=None, + # minFreq=1, + # maxFreq=100, + # stepFreq=1, + # normSignal=False, + # normPSD=False, + # transformMethod='morlet', + # **kwargs + # ): + + print('Plotting CSD power spectral density (PSD)...') + + # If input is a dictionary, pull the data out of it + if type(PSDData) == dict: + + freqs = PSDData['psdFreqs'] + freq = freqs[0] + psds = PSDData['psdSignal'] + names = PSDData['psdNames'] + colors = PSDData.get('colors') + linewidths = PSDData.get('linewidths') + alphas = PSDData.get('alphas') + axisArgs = PSDData.get('axisArgs') + + + # Set up colors, linewidths, and alphas for the plots + if not colors: + if not colorList: + from .plotter import colorList + colors = colorList[0:len(psds)] + + + From c42085ee7fa41139861524531512074721203364 Mon Sep 17 00:00:00 2001 From: ericagriffith Date: Wed, 14 Dec 2022 17:43:00 -0500 Subject: [PATCH 10/65] expanded on csd psd plotting to produce similar plot(s) when compared to lfp psd plotting --- netpyne/analysis/csd.py | 2 +- netpyne/plotting/plotCSDPSD.py | 150 ++++++++++++++++++++++++++++++++- 2 files changed, 150 insertions(+), 2 deletions(-) diff --git a/netpyne/analysis/csd.py b/netpyne/analysis/csd.py index 8bc842811..a58ba7986 100644 --- a/netpyne/analysis/csd.py +++ b/netpyne/analysis/csd.py @@ -377,7 +377,7 @@ def prepareCSDPSD( # vaknin=vaknin, # norm=norm, - print('Preparing PSD CSD data...') + print('Preparing CSD PSD data...') names = data['electrodes']['names'] diff --git a/netpyne/plotting/plotCSDPSD.py b/netpyne/plotting/plotCSDPSD.py index c5afd030a..d700b82a4 100644 --- a/netpyne/plotting/plotCSDPSD.py +++ b/netpyne/plotting/plotCSDPSD.py @@ -2,20 +2,31 @@ from ..analysis.utils import exception +import numpy as np +import math +from .plotter import LinesPlotter +from .plotter import MultiPlotter + @exception def plotCSDPSD( PSDData=None, + axis=None, timeRange=None, electrodes=['avg', 'all'], pop=None, + separation=1.0, + roundOffset=True, minFreq=1, maxFreq=100, stepFreq=1, normSignal=False, normPSD=False, transformMethod='morlet', + orderInverse=True, + legend=True, colorList=None, + returnPlotter=False, **kwargs): """Function to produce a plot of CSD Power Spectral Density (PSD) data @@ -30,15 +41,44 @@ def plotCSDPSD( Parameters ---------- + axis : matplotlib axis + The axis to plot into, allowing overlaying of plots. + + *Default:* ``None`` produces a new figure and axis. + + separation : float + Use to increase or decrease distance between signals on the plot. + + *Default:* ``1.0`` + + roundOffset : bool + Attempts to line up PSD signals with gridlines + + *Default:* ``True`` + + orderInverse : bool + Whether to invert the order of plotting. + + *Default:* ``True`` + + legend : bool + Whether or not to add a legend to the plot. + + *Default:* ``True`` adds a legend. + colorList : list A *list* of colors to draw from when plotting. *Default:* ``None`` uses the default NetPyNE colorList. + returnPlotter : bool + Whether to return the figure or the NetPyNE MetaFig object. + + *Default:* ``False`` returns the figure. Returns ------- - LFPPSDPlot : *matplotlib figure* + CSDPSDPlot : *matplotlib figure* By default, returns the *figure*. If ``returnPlotter`` is ``True``, instead returns the NetPyNE MetaFig. """ @@ -100,5 +140,113 @@ def plotCSDPSD( from .plotter import colorList colors = colorList[0:len(psds)] + if not linewidths: + linewidths = [1.0 for name in names] + + if not alphas: + alphas = [1.0 for name in names] + + # Create a dictionary to hold axis inputs + if not axisArgs: + axisArgs = {} + title = 'CSD Power Spectral Density' + if pop: + title += ' - Population: ' + pop + axisArgs['title'] = title + axisArgs['xlabel'] = 'Frequency (Hz)' + axisArgs['ylabel'] = 'Power (mVˆ2 / Hz) -- NOTE: CORRECT??' ## ensure this is correct? + + + if axis != 'multi': + axisArgs['grid'] = {'which': 'both'} + + + # Link colors to traces, make avg plot black, add separation to traces + plotColors = [] + legendLabels = [] + colorIndex = 0 + offset = np.absolute(psds).max() * separation + + if axis == 'multi': + offset = 0 + roundOffset = False + + if roundOffset: + sigfigs = 1 + if type(roundOffset) == int: + sigfigs = roundOffset + offset = round(offset, sigfigs - int(math.floor(math.log10(abs(offset)))) - 1) + + for index, (name, psd) in enumerate(zip(names, psds)): + legendLabels.append(name) + if orderInverse: + psds[index] = index * offset - psd + axisArgs['invert_yaxis'] = True + else: + psds[index] = index * offset + psd + if name == 'avg': + plotColors.append('black') + else: + plotColors.append(colors[colorIndex]) + colorIndex += 1 + + + # Create a dictionary with the inputs for a line plot + linesData = {} + linesData['x'] = freq + linesData['y'] = psds + linesData['color'] = plotColors + linesData['marker'] = None + linesData['markersize'] = None + linesData['linewidth'] = None + linesData['alpha'] = None + linesData['label'] = legendLabels + + # If a kwarg matches a lines input key, use the kwarg value instead of the default + for kwarg in list(kwargs.keys()): + if kwarg in linesData: + linesData[kwarg] = kwargs[kwarg] + kwargs.pop(kwarg) + + # create Plotter object + if axis != 'multi': + plotter = LinesPlotter(data=linesData, kind='CSDPSD', axis=axis, **axisArgs, **kwargs) + else: + plotter = MultiPlotter(data=linesData, kind='CSDPSD', metaFig=None, **axisArgs, **kwargs) + ### NOTE: only kind='LFPPSD' exists for MultiPlotter in plotter.py --> ?? + + + metaFig = plotter.metafig + + # Set up the default legend settings + legendKwargs = {} + legendKwargs['title'] = 'Electrodes' + legendKwargs['loc'] = 'upper right' + legendKwargs['fontsize'] = 'small' + + # add the legend + if legend: + axisArgs['legend'] = legendKwargs + + # Generate the figure + PSDPlot = plotter.plot(**axisArgs, **kwargs) + + + # Default is to return the figure, but you can also return the plotter + if returnPlotter: + return metaFig + else: + return PSDPlot + + + + + + + + + + + From ddde8891b49523a9fbadd012e051c08791613b7f Mon Sep 17 00:00:00 2001 From: aranega Date: Fri, 24 Feb 2023 13:21:13 -0600 Subject: [PATCH 11/65] Add metadata for sub-cellular connectivity --- netpyne/metadata/metadata.py | 212 +++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) diff --git a/netpyne/metadata/metadata.py b/netpyne/metadata/metadata.py index 3e35c4e4f..e540fc4c1 100644 --- a/netpyne/metadata/metadata.py +++ b/netpyne/metadata/metadata.py @@ -787,6 +787,218 @@ }, }, # --------------------------------------------------------------------------------------------------------------------- + # netParams.subConnParams + # --------------------------------------------------------------------------------------------------------------------- + "subConnParams": { + "label": "Sub-Cellular Connectivity parameters", + "suggestions": "", + "help": "", + "hintText": "", + "children": { + "preConds": { + "label": "Conditions for the presynaptic cells", + "help": "Presynaptic cell conditions defined using attributes/tags and the required value e.g. {'cellType': 'PYR'}. Values can be lists, e.g. {'pop': ['Exc1', 'Exc2']}. For location properties, the list values correspond to the min and max values, e.g. {'ynorm': [0.1, 0.6]}.", + "suggestions": "", + "hintText": "", + "children": { + "pop": { + "label": "Population (multiple selection available)", + "suggestions": "", + "help": "Cells belonging to this population (or list of populations) will be connected pre-synaptically.", + "hintText": "", + }, + "cellType": { + "label": "Cell type (multiple selection available)", + "suggestions": "", + "help": "Ccells with this cell type attribute/tag will be connected pre-synaptically.", + "hintText": "", + }, + "cellModel": { + "label": "Cell model (multiple selection available)", + "suggestions": "", + "help": "Cells with this cell model attribute/tag will be connected pre-synaptically.", + "hintText": "", + }, + "x": { + "label": "Range of x-axis locations", + "suggestions": "", + "help": "Cells within these x-axis locations will be connected pre-synaptically.", + "hintText": "", + }, + "y": { + "label": "Range of y-axis locations", + "suggestions": "", + "help": "Cells within these y-axis locations will be connected pre-synaptically.", + "hintText": "", + }, + "z": { + "label": "Range of z-axis locations", + "suggestions": "", + "help": "Cells within these z-axis locations will be connected pre-synaptically..", + "hintText": "", + }, + "xnorm": { + "label": "Range of normalized x-axis locations", + "suggestions": "", + "help": "Cells within these normalized x-axis locations will be connected pre-synaptically.", + "hintText": "", + }, + "ynorm": { + "label": "Range of normalized y-axis locations", + "suggestions": "", + "help": "Cells within these normalized y-axis locations will be connected pre-synaptically.", + "hintText": "", + }, + "znorm": { + "label": "Range of normalized z-axis locations", + "suggestions": "", + "help": "Cells within these normalized z-axis locations will be connected pre-synaptically.", + "hintText": "", + }, + }, + }, + "postConds": { + "label": "Conditions for the postsynaptic cells", + "help": "Defined as a dictionary with the attributes/tags of the postsynaptic cell and the required values e.g. {'cellType': 'PYR'}. Values can be lists, e.g. {'pop': ['Exc1', 'Exc2']}. For location properties, the list values correspond to the min and max values, e.g. {'ynorm': [0.1, 0.6]}.", + "suggestions": "", + "hintText": "", + "children": { + "pop": { + "label": "Population (multiple selection available)", + "suggestions": "", + "help": "Cells belonging to this population (or list of populations) will be connected post-synaptically.", + "hintText": "", + }, + "cellType": { + "label": "Cell type (multiple selection available)", + "suggestions": "", + "help": "Ccells with this cell type attribute/tag will be connected post-synaptically.", + "hintText": "", + }, + "cellModel": { + "label": "Cell model (multiple selection available)", + "suggestions": "", + "help": "Cells with this cell model attribute/tag will be connected post-synaptically.", + "hintText": "", + }, + "x": { + "label": "Range of x-axis locations", + "suggestions": "", + "help": "Cells within these x-axis locations will be connected post-synaptically.", + "hintText": "", + }, + "y": { + "label": "Range of y-axis locations", + "suggestions": "", + "help": "Cells within these y-axis locations will be connected post-synaptically.", + "hintText": "", + }, + "z": { + "label": "Range of z-axis locations", + "suggestions": "", + "help": "Cells within these z-axis locations will be connected post-synaptically..", + "hintText": "", + }, + "xnorm": { + "label": "Range of normalized x-axis locations", + "suggestions": "", + "help": "Cells within these normalized x-axis locations will be connected post-synaptically.", + "hintText": "", + }, + "ynorm": { + "label": "Range of normalized y-axis locations", + "suggestions": "", + "help": "Cells within these normalized y-axis locations will be connected post-synaptically.", + "hintText": "", + }, + "znorm": { + "label": "Range of normalized z-axis locations", + "suggestions": "", + "help": "Cells within these normalized z-axis locations will be connected post-synaptically.", + "hintText": "", + }, + }, + }, + "groupSynMechs": { + "label": "Synaptic mechanism", + "help": " List of synaptic mechanisms grouped together when redistributing synapses. If omitted, post-synaptic locations of all connections (meeting preConds and postConds) are redistributed independently with a given profile (defined below). If a list is provided, synapses of common connections (same pre- and post-synaptic neurons for each mechanism) are relocated to the same location. For example, [‘AMPA’,’NMDA’].", + "suggestions": "", + "hintText": "", + }, + "sec": { + "label": "Redistributed Synapse Sections", + "help": "List of sections admitting redistributed synapses. If omitted, the section used to redistribute synapses is the soma or, if it does not exist, the first available section in the post-synaptic cell. For example, [‘Adend1’,’Adend2’, ‘Adend3’,’Bdend’].", + "suggestions": "", + "hintText": "soma", + "type": "list(str)", + }, + "density": { + "label": "Type of redistribution", + "help": "", + "suggestions": "", + "hintText": "", + "children": { + "gridY": { + "label": "Depth y-coordinate Positions", + "help": "List of positions in y-coordinate (depth).", + "suggestions": "", + "hintText": "", + "type": "list(int, float)", + }, + "gridX": { + "label": "Depth x-coordinate Positions", + "help": "List of positions in x-coordinate (or z).", + "suggestions": "", + "hintText": "", + "type": "list(int, float)", + }, + "fixedSomaY": { + "label": "Soma's y-coordinate", + "help": "Absolute position y-coordinate of the soma, used to shift gridY (also provided in absolute coordinates).", + "suggestions": "", + "hintText": "", + "type": "int,float", + }, + "gridValues": { + "label": "Synaptic Density", + "help": "One or two-dimensional list expressing the (relative) synaptic density in the coordinates defined by (gridX and) gridY.", + "suggestions": "", + "hintText": "", + "type": "list(int, float)", + }, + "ref_sec": { + "label": "Reference Section", + "help": "Section used as a reference from which distance is computed. If omitted, the section used to reference distances is the soma or, if it does not exist, anything starting with ‘som’ or, otherwise, the first available section in the post-synaptic cell.", + "suggestions": "", + "hintText": "", + "type": "str", + }, + "ref_seg": { + "label": "Reference Segment", + "help": "Segment within the section used to reference the distances. If omitted, it is used a default value (0.5).", + "suggestions": "", + "hintText": "", + "type": "int, float", + }, + "target_distance": { + "label": "Target Distance", + "help": "Target distance from the reference where synapses will be reallocated. If omitted, this value is set to 0. The chosen location will be the closest to this target, between the allowed sections.", + "suggestions": "", + "hintText": "", + "type": "int, float", + }, + "coord": { + "label": "Coordinate's System", + "help": "Coordinates’ system used to compute distance. If omitted, the distance is computed along the dendritic tree. Alternatively, it may be used ‘cartesian’ to calculate the distance in the euclidean space (distance from the reference to the target segment in the cartesian coordinate system).", + "suggestions": "", + "hintText": "", + "type": "int, float", + }, + } + } + }, + }, + # --------------------------------------------------------------------------------------------------------------------- # netParams.stimSourceParams # --------------------------------------------------------------------------------------------------------------------- "stimSourceParams": { From 7d5507bd20c0fcf5dccadd3ba3065aec01d53724 Mon Sep 17 00:00:00 2001 From: aranega Date: Mon, 27 Feb 2023 12:53:12 -0600 Subject: [PATCH 12/65] Change type of geom, mechs and synMechs to accept string functions expressions --- netpyne/metadata/metadata.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/netpyne/metadata/metadata.py b/netpyne/metadata/metadata.py index 3e35c4e4f..7607a45c3 100644 --- a/netpyne/metadata/metadata.py +++ b/netpyne/metadata/metadata.py @@ -363,7 +363,7 @@ "suggestions": "", "help": "", "hintText": "10", - "type": "float", + "type": "func", }, "L": { "label": "Length (um)", @@ -371,7 +371,7 @@ "suggestions": "", "help": "", "hintText": "50", - "type": "float", + "type": "func", }, "Ra": { "label": "Axial resistance, Ra (ohm-cm)", @@ -379,14 +379,14 @@ "suggestions": "", "help": "", "hintText": "100", - "type": "float", + "type": "func", }, "cm": { "label": "Membrane capacitance, cm (uF/cm2)", "suggestions": "", "help": "", "hintText": "1", - "type": "float", + "type": "func", }, "pt3d": { "label": "3D points", @@ -401,7 +401,7 @@ "suggestions": "", "help": "", "hintText": "1", - "type": "float", + "type": "func", }, }, "mechs": { @@ -409,7 +409,7 @@ "help": "Dictionary of density/distributed mechanisms, including the name of the mechanism (e.g. hh or pas) and a list of properties of the mechanism (e.g. {'g': 0.003, 'e': -70}).", "suggestions": "", "hintText": "", - "type": "float", + "type": "func", }, "ions": { "label": "Ions", @@ -539,28 +539,28 @@ "help": "Define the time constant for the first exponential.", "suggestions": "", "hintText": "1", - "type": "float", + "type": "func", }, "tau2": { "label": "Time constant for exponential 2 (ms)", "help": "Define the time constant for the second exponential.", "suggestions": "", "hintText": "5", - "type": "float", + "type": "func", }, "e": { "label": "Reversal potential (mV)", "help": "Reversal potential of the synaptic receptors.", "suggestions": "", "hintText": "0", - "type": "float", + "type": "func", }, "i": { "label": "synaptic current (nA)", "help": "Synaptic current in nA.", "suggestions": "", "hintText": "10", - "type": "float", + "type": "func", }, }, }, From 32c0db2e7a5a4eb6d0c7420105efb1f1b9d7ce29 Mon Sep 17 00:00:00 2001 From: vvbragin Date: Fri, 7 Apr 2023 13:19:37 +0200 Subject: [PATCH 13/65] optimized and re-factored sim.gatherData(), added new optional arguments for flexibility --- CHANGES.md | 10 ++ netpyne/sim/gather.py | 335 ++++++++++++++++++------------------------ 2 files changed, 154 insertions(+), 191 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 833854561..835debbd1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,13 @@ +# Version in development + +**New features** + +- Extended sim.gatherData() with more optional arguments for flexibility + +**Bug fixes** + +- Fixed errors in plotting with sim.cfg.gatherOnlySimData=True + # Version 1.0.4 **New features** diff --git a/netpyne/sim/gather.py b/netpyne/sim/gather.py index 7cc1f1e8a..ca81ea829 100644 --- a/netpyne/sim/gather.py +++ b/netpyne/sim/gather.py @@ -23,7 +23,7 @@ # ------------------------------------------------------------------------------ # Gather data from nodes # ------------------------------------------------------------------------------ -def gatherData(gatherLFP=True, gatherDipole=True): +def gatherData(gatherLFP=True, gatherDipole=True, gatherOnlySimData=None, includeSimDataEntries=None, analyze=True): """ Function for/to @@ -48,6 +48,10 @@ def gatherData(gatherLFP=True, gatherDipole=True): if sim.rank == 0: print('\nGathering data...') + # flag to avoid saving cell data and cell gids per populations (saves gather time and space; cannot inspect cells and pops) + if gatherOnlySimData is None: + gatherOnlySimData = sim.cfg.gatherOnlySimData + # flag to avoid saving sections data for each cell (saves gather time and space; cannot inspect cell secs or re-simulate) if not sim.cfg.saveCellSecs: for cell in sim.net.cells: @@ -89,183 +93,131 @@ def gatherData(gatherLFP=True, gatherDipole=True): simDataVecs.append('dipole') singleNodeVecs = ['t'] + if includeSimDataEntries: + singleNodeVecs = [v for v in singleNodeVecs if v in includeSimDataEntries] + if sim.nhosts > 1: # only gather if >1 nodes netPopsCellGids = {popLabel: list(pop.cellGids) for popLabel, pop in sim.net.pops.items()} - # gather only sim data - if getattr(sim.cfg, 'gatherOnlySimData', False): - nodeData = {'simData': sim.simData} - data = [None] * sim.nhosts - data[0] = {} - for k, v in nodeData.items(): - data[0][k] = v - gather = sim.pc.py_alltoall(data) - sim.pc.barrier() - - if sim.rank == 0: # simData - print(' Gathering only sim data...') - sim.allSimData = Dict() - for k in list(gather[0]['simData'].keys()): # initialize all keys of allSimData dict - if gatherLFP and k == 'LFP': - sim.allSimData[k] = np.zeros((gather[0]['simData']['LFP'].shape)) - elif gatherLFP and k == 'LFPPops': - sim.allSimData[k] = { - p: np.zeros(gather[0]['simData']['LFP'].shape) - for p in gather[0]['simData']['LFPPops'].keys() - } - elif gatherDipole and k == 'dipoleSum': - sim.allSimData[k] = np.zeros((gather[0]['simData']['dipoleSum'].shape)) - elif sim.cfg.recordDipolesHNN and k == 'dipole': - for dk in sim.cfg.recordDipolesHNN: - sim.allSimData[k][dk] = np.zeros(len(gather[0]['simData']['dipole'][dk])) - else: - sim.allSimData[k] = {} - - for key in singleNodeVecs: # store single node vectors (eg. 't') - sim.allSimData[key] = list(nodeData['simData'][key]) - - # fill in allSimData taking into account if data is dict of h.Vector (code needs improvement to be more generic) - for node in gather: # concatenate data from each node - for key, val in node['simData'].items(): # update simData dics of dics of h.Vector - if key in simDataVecs: # simData dicts that contain Vectors - if isinstance(val, dict): - for key2, val2 in val.items(): - if isinstance(val2, dict): - sim.allSimData[key].update(Dict({key2: Dict()})) - for stim, val3 in val2.items(): - sim.allSimData[key][key2].update( - {stim: list(val3)} - ) # udpate simData dicts which are dicts of dicts of Vectors (eg. ['stim']['cell_1']['backgrounsd']=h.Vector) - elif key == 'dipole': - sim.allSimData[key][key2] = np.add( - sim.allSimData[key][key2], val2.as_numpy() - ) # add together dipole values from each node - else: - sim.allSimData[key].update( - {key2: list(val2)} - ) # udpate simData dicts which are dicts of Vectors (eg. ['v']['cell_1']=h.Vector) - else: - sim.allSimData[key] = list(sim.allSimData[key]) + list( - val - ) # udpate simData dicts which are Vectors - elif gatherLFP and key == 'LFP': - sim.allSimData[key] += np.array(val) - elif gatherLFP and key == 'LFPPops': - for p in val: - sim.allSimData[key][p] += np.array(val[p]) - elif gatherDipole and key == 'dipoleSum': - sim.allSimData[key] += np.array(val) - elif key not in singleNodeVecs: - sim.allSimData[key].update(val) # update simData dicts which are not Vectors - - if len(sim.allSimData['spkt']) > 0: - sim.allSimData['spkt'], sim.allSimData['spkid'] = zip( - *sorted(zip(sim.allSimData['spkt'], sim.allSimData['spkid'])) - ) # sort spks - sim.allSimData['spkt'], sim.allSimData['spkid'] = list(sim.allSimData['spkt']), list( - sim.allSimData['spkid'] - ) - - sim.net.allPops = ODict() # pops - for popLabel, pop in sim.net.pops.items(): - sim.net.allPops[popLabel] = pop.__getstate__() # can't use dict comprehension for OrderedDict - - sim.net.allCells = [c.__dict__ for c in sim.net.cells] + if includeSimDataEntries: + simData = {key: sim.simData[key] for key in includeSimDataEntries} + else: + simData = sim.simData - # gather cells, pops and sim data + if gatherOnlySimData: + nodeData = { + 'simData': simData + } else: nodeData = { + 'simData': simData, 'netCells': [c.__getstate__() for c in sim.net.cells], 'netPopsCellGids': netPopsCellGids, - 'simData': sim.simData, } if gatherLFP and hasattr(sim.net, 'recXElectrode'): nodeData['xElectrodeTransferResistances'] = sim.net.recXElectrode.transferResistances - data = [None] * sim.nhosts - data[0] = {} - for k, v in nodeData.items(): - data[0][k] = v + data = [None] * sim.nhosts + data[0] = {} + for k, v in nodeData.items(): + data[0][k] = v + gather = sim.pc.py_alltoall(data) + sim.pc.barrier() + + if sim.rank == 0: + sim.allSimData = Dict() + + if gatherOnlySimData: + print(' Gathering only sim data...') - # print data - gather = sim.pc.py_alltoall(data) - sim.pc.barrier() - if sim.rank == 0: + # However, still need to ensure these list aren't empty to avoid errors during analyzing/plotting + # TODO: this should be improved by handling abovementioned errors in analyzing/plotting instead of adding this workaround here + allCells = getattr(sim.net, 'allCells', []) + if len(allCells) == 0: + sim.net.allCells = [c.__getstate__() for c in sim.net.cells] + + allPops = getattr(sim.net, 'allPops', ODict()) + if len(allPops) == 0: + sim.net.allPops = allPops + for popLabel, pop in sim.net.pops.items(): + sim.net.allPops[popLabel] = pop.__getstate__() # can't use dict comprehension for OrderedDict + else: allCells = [] allPops = ODict() for popLabel, pop in sim.net.pops.items(): allPops[popLabel] = pop.__getstate__() # can't use dict comprehension for OrderedDict allPopsCellGids = {popLabel: [] for popLabel in netPopsCellGids} - sim.allSimData = Dict() allResistances = {} + + for k in list(gather[0]['simData'].keys()): # initialize all keys of allSimData dict + if gatherLFP and k == 'LFP': + sim.allSimData[k] = np.zeros((gather[0]['simData']['LFP'].shape)) + elif gatherLFP and k == 'LFPPops': + sim.allSimData[k] = { + p: np.zeros(gather[0]['simData']['LFP'].shape) + for p in gather[0]['simData']['LFPPops'].keys() + } + elif gatherDipole and k == 'dipoleSum': + sim.allSimData[k] = np.zeros((gather[0]['simData']['dipoleSum'].shape)) + elif sim.cfg.recordDipolesHNN and k == 'dipole': + for dk in sim.cfg.recordDipolesHNN: + sim.allSimData[k][dk] = np.zeros(len(gather[0]['simData']['dipole'][dk])) + else: + sim.allSimData[k] = {} - for k in list(gather[0]['simData'].keys()): # initialize all keys of allSimData dict - if gatherLFP and k == 'LFP': - sim.allSimData[k] = np.zeros((gather[0]['simData']['LFP'].shape)) - elif gatherLFP and k == 'LFPPops': - sim.allSimData[k] = { - p: np.zeros(gather[0]['simData']['LFP'].shape) - for p in gather[0]['simData']['LFPPops'].keys() - } - elif gatherDipole and k == 'dipoleSum': - sim.allSimData[k] = np.zeros((gather[0]['simData']['dipoleSum'].shape)) - elif sim.cfg.recordDipolesHNN and k == 'dipole': - for dk in sim.cfg.recordDipolesHNN: - sim.allSimData[k][dk] = np.zeros(len(gather[0]['simData']['dipole'][dk])) - else: - sim.allSimData[k] = {} - - for key in singleNodeVecs: # store single node vectors (eg. 't') - sim.allSimData[key] = list(nodeData['simData'][key]) + for key in singleNodeVecs: # store single node vectors (eg. 't') + sim.allSimData[key] = list(nodeData['simData'][key]) - # fill in allSimData taking into account if data is dict of h.Vector (code needs improvement to be more generic) - for node in gather: # concatenate data from each node + # fill in allSimData taking into account if data is dict of h.Vector (code needs improvement to be more generic) + for node in gather: # concatenate data from each node + if not gatherOnlySimData: allCells.extend(node['netCells']) # extend allCells list for popLabel, popCellGids in node['netPopsCellGids'].items(): allPopsCellGids[popLabel].extend(popCellGids) - - for key, val in node['simData'].items(): # update simData dics of dics of h.Vector - if key in simDataVecs: # simData dicts that contain Vectors - if isinstance(val, dict): - for key2, val2 in val.items(): - if isinstance(val2, dict): - sim.allSimData[key].update(Dict({key2: Dict()})) - for stim, val3 in val2.items(): - sim.allSimData[key][key2].update( - {stim: list(val3)} - ) # udpate simData dicts which are dicts of dicts of Vectors (eg. ['stim']['cell_1']['backgrounsd']=h.Vector) - elif key == 'dipole': - sim.allSimData[key][key2] = np.add( - sim.allSimData[key][key2], val2.as_numpy() - ) # add together dipole values from each node - else: - sim.allSimData[key].update( - {key2: list(val2)} - ) # udpate simData dicts which are dicts of Vectors (eg. ['v']['cell_1']=h.Vector) - else: - sim.allSimData[key] = list(sim.allSimData[key]) + list( - val - ) # udpate simData dicts which are Vectors - elif gatherLFP and key == 'LFP': - sim.allSimData[key] += np.array(val) - elif gatherLFP and key == 'LFPPops': - for p in val: - sim.allSimData[key][p] += np.array(val[p]) - elif gatherDipole and key == 'dipoleSum': - sim.allSimData[key] += np.array(val) - elif key not in singleNodeVecs: - sim.allSimData[key].update(val) # update simData dicts which are not Vectors if 'xElectrodeTransferResistances' in node: allResistances.update(node['xElectrodeTransferResistances']) - if len(sim.allSimData['spkt']) > 0: - sim.allSimData['spkt'], sim.allSimData['spkid'] = zip( - *sorted(zip(sim.allSimData['spkt'], sim.allSimData['spkid'])) - ) # sort spks - sim.allSimData['spkt'], sim.allSimData['spkid'] = list(sim.allSimData['spkt']), list( - sim.allSimData['spkid'] - ) + for key, val in node['simData'].items(): # update simData dics of dics of h.Vector + if key in simDataVecs: # simData dicts that contain Vectors + if isinstance(val, dict): + for key2, val2 in val.items(): + if isinstance(val2, dict): + sim.allSimData[key].update(Dict({key2: Dict()})) + for stim, val3 in val2.items(): + sim.allSimData[key][key2].update( + {stim: list(val3)} + ) # udpate simData dicts which are dicts of dicts of Vectors (eg. ['stim']['cell_1']['backgrounsd']=h.Vector) + elif key == 'dipole': + sim.allSimData[key][key2] = np.add( + sim.allSimData[key][key2], val2.as_numpy() + ) # add together dipole values from each node + else: + sim.allSimData[key].update( + {key2: list(val2)} + ) # udpate simData dicts which are dicts of Vectors (eg. ['v']['cell_1']=h.Vector) + else: + sim.allSimData[key] = list(sim.allSimData[key]) + list( + val + ) # udpate simData dicts which are Vectors + elif gatherLFP and key == 'LFP': + sim.allSimData[key] += np.array(val) + elif gatherLFP and key == 'LFPPops': + for p in val: + sim.allSimData[key][p] += np.array(val[p]) + elif gatherDipole and key == 'dipoleSum': + sim.allSimData[key] += np.array(val) + elif key not in singleNodeVecs: + sim.allSimData[key].update(val) # update simData dicts which are not Vectors + + if len(sim.allSimData['spkt']) > 0: + sim.allSimData['spkt'], sim.allSimData['spkid'] = zip( + *sorted(zip(sim.allSimData['spkt'], sim.allSimData['spkid'])) + ) # sort spks + sim.allSimData['spkt'], sim.allSimData['spkid'] = list(sim.allSimData['spkt']), list( + sim.allSimData['spkid'] + ) + if not gatherOnlySimData: sim.net.allCells = sorted(allCells, key=lambda k: k['gid']) for popLabel, pop in allPops.items(): @@ -324,55 +276,56 @@ def gatherData(gatherLFP=True, gatherDipole=True): if sim.cfg.timing: print((' Done; gather time = %0.2f s.' % sim.timingData['gatherTime'])) - print('\nAnalyzing...') - sim.totalSpikes = len(sim.allSimData['spkt']) - sim.totalSynapses = sum([len(cell['conns']) for cell in sim.net.allCells]) - if sim.cfg.createPyStruct: - if sim.cfg.compactConnFormat: - preGidIndex = sim.cfg.compactConnFormat.index('preGid') if 'preGid' in sim.cfg.compactConnFormat else 0 - sim.totalConnections = sum( - [len(set([conn[preGidIndex] for conn in cell['conns']])) for cell in sim.net.allCells] - ) + if analyze: + print('\nAnalyzing...') + sim.totalSpikes = len(sim.allSimData['spkt']) + sim.totalSynapses = sum([len(cell['conns']) for cell in sim.net.allCells]) + if sim.cfg.createPyStruct: + if sim.cfg.compactConnFormat: + preGidIndex = sim.cfg.compactConnFormat.index('preGid') if 'preGid' in sim.cfg.compactConnFormat else 0 + sim.totalConnections = sum( + [len(set([conn[preGidIndex] for conn in cell['conns']])) for cell in sim.net.allCells] + ) + else: + sim.totalConnections = sum( + [len(set([conn['preGid'] for conn in cell['conns']])) for cell in sim.net.allCells] + ) else: - sim.totalConnections = sum( - [len(set([conn['preGid'] for conn in cell['conns']])) for cell in sim.net.allCells] - ) - else: - sim.totalConnections = sim.totalSynapses - sim.numCells = len(sim.net.allCells) + sim.totalConnections = sim.totalSynapses + sim.numCells = len(sim.net.allCells) - if sim.totalSpikes > 0: - sim.firingRate = float(sim.totalSpikes) / sim.numCells / sim.cfg.duration * 1e3 # Calculate firing rate - else: - sim.firingRate = 0 - if sim.numCells > 0: - sim.connsPerCell = sim.totalConnections / float( - sim.numCells - ) # Calculate the number of connections per cell - sim.synsPerCell = sim.totalSynapses / float(sim.numCells) # Calculate the number of connections per cell - else: - sim.connsPerCell = 0 - sim.synsPerCell = 0 + if sim.totalSpikes > 0: + sim.firingRate = float(sim.totalSpikes) / sim.numCells / sim.cfg.duration * 1e3 # Calculate firing rate + else: + sim.firingRate = 0 + if sim.numCells > 0: + sim.connsPerCell = sim.totalConnections / float( + sim.numCells + ) # Calculate the number of connections per cell + sim.synsPerCell = sim.totalSynapses / float(sim.numCells) # Calculate the number of connections per cell + else: + sim.connsPerCell = 0 + sim.synsPerCell = 0 - print((' Cells: %i' % (sim.numCells))) - print((' Connections: %i (%0.2f per cell)' % (sim.totalConnections, sim.connsPerCell))) - if sim.totalSynapses != sim.totalConnections: - print((' Synaptic contacts: %i (%0.2f per cell)' % (sim.totalSynapses, sim.synsPerCell))) + print((' Cells: %i' % (sim.numCells))) + print((' Connections: %i (%0.2f per cell)' % (sim.totalConnections, sim.connsPerCell))) + if sim.totalSynapses != sim.totalConnections: + print((' Synaptic contacts: %i (%0.2f per cell)' % (sim.totalSynapses, sim.synsPerCell))) - if 'runTime' in sim.timingData: - print((' Spikes: %i (%0.2f Hz)' % (sim.totalSpikes, sim.firingRate))) - print((' Simulated time: %0.1f s; %i workers' % (sim.cfg.duration / 1e3, sim.nhosts))) - print((' Run time: %0.2f s' % (sim.timingData['runTime']))) + if 'runTime' in sim.timingData: + print((' Spikes: %i (%0.2f Hz)' % (sim.totalSpikes, sim.firingRate))) + print((' Simulated time: %0.1f s; %i workers' % (sim.cfg.duration / 1e3, sim.nhosts))) + print((' Run time: %0.2f s' % (sim.timingData['runTime']))) - if sim.cfg.printPopAvgRates and not sim.cfg.gatherOnlySimData: + if sim.cfg.printPopAvgRates and not gatherOnlySimData: - trange = sim.cfg.printPopAvgRates if isinstance(sim.cfg.printPopAvgRates, list) else None - sim.allSimData['popRates'] = sim.analysis.popAvgRates(tranges=trange) + trange = sim.cfg.printPopAvgRates if isinstance(sim.cfg.printPopAvgRates, list) else None + sim.allSimData['popRates'] = sim.analysis.popAvgRates(tranges=trange) - if 'plotfI' in sim.cfg.analysis: - sim.analysis.calculatefI() # need to call here so data is saved to file + if 'plotfI' in sim.cfg.analysis: + sim.analysis.calculatefI() # need to call here so data is saved to file - sim.allSimData['avgRate'] = sim.firingRate # save firing rate + sim.allSimData['avgRate'] = sim.firingRate # save firing rate return sim.allSimData From 8ab187585551581cbc67e180d9d491b5f043c830 Mon Sep 17 00:00:00 2001 From: vvbragin Date: Wed, 19 Apr 2023 01:36:38 +0200 Subject: [PATCH 14/65] fixed broken links in LFP documentation and tutorial --- doc/source/tutorial.rst | 6 +++--- examples/LFPrecording/README.md | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index 01d1a4f61..949f7c84e 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -910,7 +910,7 @@ Notice how the rate initially increases as a function of connection weight, but Tutorial 9: Recording and plotting LFPs --------------------------------------- -Examples of how to record and analyze local field potentials (LFP) in single cells and networks are included in the \examples folder: `LFP recording example `_ . LFP recording also works with parallel simulations. +Examples of how to record and analyze local field potentials (LFP) in single cells and networks are included in the \examples folder: `LFP recording example `_ . LFP recording also works with parallel simulations. To record LFP just set the list of 3D locations of the LFP electrodes in the `simConfig` attribute `recordLFP` e.g. ``simConfig.recordLFP = e.g. [[50, 100, 50], [50, 200, 50]]`` (note the y coordinate represents depth, so will be represented as a negative value when plotted). The LFP signal in each electrode is obtained by summing the extracellular potential contributed by each segment of each neuron. Extracellular potentials are calculated using the "line source approximation" and assuming an Ohmic medium with conductivity sigma = 0.3 mS/mm. For more information on modeling LFPs see `Scholarpedia `_ or `this article `_ . @@ -918,13 +918,13 @@ The recorded LFP signal will be stored in ``sim.allSimData['LFP']`` as a 2D list To plot the LFP use the ``sim.analysis.plotLFP()`` method. This allows to plot for each electrode: 1) the time-resolved LFP signal ('timeSeries'), 2) the power spectral density ('PSD'), 3) the spectrogram / time-frequency profile ('spectrogram'), 4) and the 3D locations of the electrodes overlaid over the model neurons ('locations'). See :ref:`analysis_functions` for the full list of ``plotLFP()`` arguments. -The first example ( :download:`cell_lfp.py <../../examples/LFPrecording/cell_lfp.py>`) shows LFP recording for a single cell -- M1 corticospinal neuron -- with 700+ compartments (segments) and multiple somatic and dendritic ionic channels. The cell parameters are loaded from a .json file. The cell receives NetStim input to its soma via an excitatory synapse. Ten LFP electrodes are placed at both sides of the neuron at 5 different cortical depths. The soma voltage, LFP time-resolved signal and the 3D locations of the electrodes are plotted: +The `first example `_ shows LFP recording for a single cell -- M1 corticospinal neuron -- with 700+ compartments (segments) and multiple somatic and dendritic ionic channels. The cell parameters are loaded from a .json file. The cell receives NetStim input to its soma via an excitatory synapse. Ten LFP electrodes are placed at both sides of the neuron at 5 different cortical depths. The soma voltage, LFP time-resolved signal and the 3D locations of the electrodes are plotted: .. image:: figs/lfp_cell.png :width: 60% :align: center -The second example ( :download:`net_lfp.py <../../examples/LFPrecording/net_lfp.py>`) shows LFP recording for a network very similar to that shown in Tutorial 5. However, in this case, the cells have been replaced with a more realistic model: a 6-compartment M1 corticostriatal neuron with multiple ionic channels. The cell parameters are loaded from a .json file. Cell receive NetStim inputs and include excitatory and inhibitory connections. Four LFP electrodes are placed at different cortical depths. The raster plot and LFP time-resolved signal, PSD, spectrogram and 3D locations of the electrodes are plotted: +The `second example `_ shows LFP recording for a network very similar to that shown in Tutorial 5. However, in this case, the cells have been replaced with a more realistic model: a 6-compartment M1 corticostriatal neuron with multiple ionic channels. The cell parameters are loaded from a .json file. Cell receive NetStim inputs and include excitatory and inhibitory connections. Four LFP electrodes are placed at different cortical depths. The raster plot and LFP time-resolved signal, PSD, spectrogram and 3D locations of the electrodes are plotted: .. image:: figs/lfp_net.png :width: 90% diff --git a/examples/LFPrecording/README.md b/examples/LFPrecording/README.md index 6fbde8dcc..60550c0f8 100644 --- a/examples/LFPrecording/README.md +++ b/examples/LFPrecording/README.md @@ -4,35 +4,35 @@ Two examples of using LFP recording in NetPyNE: 1) LFP recording for a single cell -- M1 corticospinal neuron -- with 700+ compartments (segments) and multiple somatic and dendritic ionic channels. The cell parameters are loaded from a .json file. The cell receives NetStim input to its soma via an excitatory synapse. Ten LFP electrodes are placed at both sides of the neuron at 5 different cortical depths. The soma voltage, LFP time-resolved signal and the 3D locations of the electrodes are plotted: -![lfp_cell](https://github.com/Neurosim-lab/netpyne/raw/lfp/examples/LFPrecording/lfp_cell.png) +![lfp_cell](https://raw.githubusercontent.com/suny-downstate-medical-center/netpyne/development/doc/source/figs/lfp_cell.png) 2) LFP recording for a network very similar to that shown in Tutorial 5. However, in this case, the cells have been replaced with a more realistic model: a 6-compartment corticostriatal neuron with multiple ionic channels. The cell parameters are loaded from a .json file. Cell receive NetStim inputs and include excitatory and inhibitory connections. Four LFP electrodes are placed at different cortical depths. The raster plot and LFP time-resolved signal, PSD, spectrogram and 3D locations of the electrodes are plotted: -![lfp_net](https://github.com/Neurosim-lab/netpyne/raw/lfp/examples/LFPrecording/lfp_net.png) +![lfp_net](https://raw.githubusercontent.com/suny-downstate-medical-center/netpyne/development/doc/source/figs/lfp_net.png) To record LFP just set the list of 3D locations of the LFP electrodes in the `simConfig` attribute `recordLFP` e.g. ``simConfig.recordLFP = e.g. [[50, 100, 50], [50, 200, 50]]`` (note the y coordinate represents depth, so will be represented as a negative value whehn plotted). The LFP signal in each electrode is obtained by summing the extracellular potential contributed by each segment of each neuron. Extracellular potentials are calculated using the "line source approximation" and assuming an Ohmic medium with conductivity sigma = 0.3 mS/mm. For more information on modeling LFPs see http://www.scholarpedia.org/article/Local_field_potential or https://doi.org/10.3389/fncom.2016.00065 . To plot the LFP use the ``sim.analysis.plotLFP()`` method. This allows to plot for each electrode: 1) the time-resolved LFP signal ('timeSeries'), 2) the power spectral density ('PSD'), 3) the spectrogram / time-frequency profile ('spectrogram'), 4) and the 3D locations of the electrodes overlayed over the model neurons ('locations'). See :ref:`analysis_functions` for the full list of ``plotLFP()`` arguments. -For further details see Tutorial 9: http://neurosimlab.org/netpyne/tutorial.html#recording-and-plotting-lfps-tutorial-9 +For further details see Tutorial 9: http://netpyne.org/tutorial.html#tutorial-9-recording-and-plotting-lfps ## Setup and execution Requires NEURON with Python and MPI support. -1. Type `nrnivmodl mod` to create a directory called either i686 or x86_64, depending on your computer's architecture. -2. To run the single cell example type: `python lfp_cell.py` or `mpiexec -np [num_proc] nrniv -mpi lfp_cell.py` -2. To run the network cell example type: `python lfp_net.py` or `mpiexec -np [num_proc] nrniv -mpi lfp_net.py` +1. Type `nrnivmodl mod` to compile .mod files (it will create a directory called either i686 or x86_64, depending on your computer's architecture). +2. To run the single cell example type: `python src/cell/init.py` or `mpiexec -np [num_proc] nrniv -mpi src/cell/init.py` +2. To run the network cell example type: `python src/net/init.py` or `mpiexec -np [num_proc] nrniv -mpi src/net/init.py` ## Overview of file structure: -* /cell_lfp.py: Code to run example of LFP recording in single cell. +* src/cell/init.py: Code to run example of LFP recording in single cell. -* /net_lfp.py: Code to run example of LFP recording in network. +* src/net/init.py: Code to run example of LFP recording in network. -* /PT5B_reduced_cellParams.json: Parameters of cell used for single cell example (netParams.cellParams). +* cells/PT5B_reduced_cellParams.json: Parameters of cell used for single cell example (netParams.cellParams). -* /IT2_reduced_cellParams.json: Parameters of cell used for network example (netParams.cellParams). +* cells/IT2_reduced_cellParams.json: Parameters of cell used for network example (netParams.cellParams). * /mod/: Mod files required for single cell and network models. From acd8403cdb064f1b7b570c5ec007c697b9ff5a08 Mon Sep 17 00:00:00 2001 From: vvbragin Date: Wed, 19 Apr 2023 23:10:14 +0200 Subject: [PATCH 15/65] updated some links in installation instructions --- doc/source/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/install.rst b/doc/source/install.rst index cf7a1920d..7e2c3ce09 100644 --- a/doc/source/install.rst +++ b/doc/source/install.rst @@ -41,7 +41,7 @@ Install the development version of NetPyNE via GitHub and pip The NetPyNE package source files, as well as example models, are available via GitHub at: https://github.com/Neurosim-lab/netpyne. The following instructions will install the version in the GitHub "development" branch -- it will include some of the latest enhancements and bug fixes, but could also include temporary bugs: -1) ``git clone https://github.com/Neurosim-lab/netpyne.git`` +1) ``git clone https://github.com/suny-downstate-medical-center/netpyne.git`` 2) ``cd netpyne`` 3) ``git checkout development`` 4) ``pip install -e .`` @@ -55,7 +55,7 @@ This version can also be used by developers interested in extending the package. Use a browser-based online version of NetPyNE GUI (beta version) ------------------------------------------------------ -The NetPyNE GUI is available online at: `gui.netpyne.org `_. There is a maximum number of simultaneous users for this online version, so if you can't log in, please try again later. +The NetPyNE GUI is available online at: `v2.opensourcebrain.org `_. There is a maximum number of simultaneous users for this online version, so if you can't log in, please try again later. Note: the GUI also includes an interactive Python Jupyter Notebook (click "Python" icon at bottom-left) that you can use to directly run NetPyNE code/models (i.e. without using the actual graphical interface). From dcf50eb7324b8fc32d8f7469ae6e93af76961a6b Mon Sep 17 00:00:00 2001 From: vvbragin Date: Thu, 20 Apr 2023 06:49:54 +0200 Subject: [PATCH 16/65] made 'tests' folder a package to fix importing issue --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb From 577a3e53babcd940bb384711fc968e64c05ec9de Mon Sep 17 00:00:00 2001 From: vvbragin Date: Tue, 25 Apr 2023 14:43:08 +0200 Subject: [PATCH 17/65] support saving and loading data in .mat and .hdf5 formats --- CHANGES.md | 28 ++----- netpyne/analysis/dipole.py | 4 + netpyne/sim/load.py | 12 ++- netpyne/sim/save.py | 17 ++-- netpyne/sim/utils.py | 163 +++++++++++++++++++++---------------- 5 files changed, 120 insertions(+), 104 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fd15f67a8..f05bf3317 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +# Version in development + +**New features** + +- Support saving and loading data in .mat and .hdf5 formats + # Version 1.0.4.1 **New features** @@ -52,28 +58,6 @@ - Added ability to load PointCell from saved network -- Added MultiPlotter class to allow plotting line data on multiple axes - -- Added option to use separate axis for each PSD trace (set axis='multi') - -- Added new Batch method named SBI (Simulation Based Inference) with example folder (sbiOptim) - -- Added support for string functions in properties of cell mechanism, in cell geometry (in netParams.cellParams) - -- Added support for string functions in synMech parameters - -- Massive update of schemas (validator.py and setup.py) - -- More control over POINTER variables through synMechParams (e.g. for gap junctions) - -- Introduced cell variables in cellParams - -- Added plotRateSpectrogram to utils.py - -- Functions prepareCSD() and plotCSD() are now available - -- Updated documentation on install and about - **Bug fixes** - Fixed bug with loading CompartCell with custom mechanisms from saved network diff --git a/netpyne/analysis/dipole.py b/netpyne/analysis/dipole.py index 88f52d355..165467d4b 100644 --- a/netpyne/analysis/dipole.py +++ b/netpyne/analysis/dipole.py @@ -40,6 +40,10 @@ def plotDipole(showCell=None, showPop=None, timeRange=None, dpi=300, figSize=(6, p = sim.allSimData['dipolePops'][showPop] else: p = sim.allSimData['dipoleSum'] + + # if list (as a result of side-effect of some of save-load operations), make sure to convert to np.array + if isinstance(p, list): + p = np.array(p) p = p / 1000.0 # convert from nA to uA except: diff --git a/netpyne/sim/load.py b/netpyne/sim/load.py index bd07ffaef..f742c0deb 100644 --- a/netpyne/sim/load.py +++ b/netpyne/sim/load.py @@ -92,17 +92,15 @@ def _byteify(data, ignore_dicts=False): print(('Loading file %s ... ' % (filename))) dataraw = loadmat(filename, struct_as_record=False, squeeze_me=True) data = utils._mat2dict(dataraw) - # savemat(sim.cfg.filename+'.mat', replaceNoneObj(dataSave)) # replace None and {} with [] so can save in .mat format - print('Finished saving!') + data = utils._restoreFromMat(data) # load HDF5 file (uses very inefficient hdf5storage module which supports dicts) - elif ext == 'saveHDF5': - # dataSaveUTF8 = _dict2utf8(replaceNoneObj(dataSave)) # replace None and {} with [], and convert to utf + elif ext in ['hdf5', 'h5']: import hdf5storage print(('Loading file %s ... ' % (filename))) - # hdf5storage.writes(dataSaveUTF8, filename=sim.cfg.filename+'.hdf5') - print('NOT IMPLEMENTED!') + keys = hdf5storage.read('__np_keys__', filename=filename) + data = {k: hdf5storage.read(k, filename=filename) for k in keys} # load CSV file (currently only saves spikes) elif ext == 'csv': @@ -283,7 +281,7 @@ def loadNet(filename, data=None, instantiate=True, compactConnFormat=False): def sort(popKeyValue): # the assumption while sorting is that populations order corresponds to cell gids in this population - cellGids = popKeyValue[1]['cellGids'] + cellGids = popKeyValue[1].get('cellGids', []) if len(cellGids) > 0: return cellGids[0] else: diff --git a/netpyne/sim/save.py b/netpyne/sim/save.py index 37d226360..5b7575e56 100644 --- a/netpyne/sim/save.py +++ b/netpyne/sim/save.py @@ -232,22 +232,25 @@ def saveData(include=None, filename=None, saveLFP=True): path = filePath + '.mat' print(f'Saving output as {path} ... ') - savemat( - path, utils.tupleToList(utils.replaceNoneObj(dataSave)) - ) # replace None and {} with [] so can save in .mat format + dataSave = utils.replaceDictODict(dataSave) + dataSave = utils._ensureMatCompatible(dataSave) + savemat(path, utils.tupleToList(dataSave), long_field_names=True) savedFiles.append(path) print('Finished saving!') # Save to HDF5 file (uses very inefficient hdf5storage module which supports dicts) if sim.cfg.saveHDF5: - dataSaveUTF8 = utils._dict2utf8( - utils.replaceNoneObj(dataSave) - ) # replace None and {} with [], and convert to utf + recXElectrode = dataSave['net'].get('recXElectrode') + if recXElectrode: + dataSave['net']['recXElectrode'] = recXElectrode.toJSON() + dataSaveUTF8 = utils._ensureHDF5Compatible(dataSave) + keys = list(dataSaveUTF8.keys()) + dataSaveUTF8['__np_keys__'] = keys import hdf5storage path = filePath + '.hdf5' print(f'Saving output as {path} ... ') - hdf5storage.writes(dataSaveUTF8, filename=path) + hdf5storage.writes(dataSaveUTF8, filename=path, truncate_existing=True) savedFiles.append(path) print('Finished saving!') diff --git a/netpyne/sim/utils.py b/netpyne/sim/utils.py index cef5ceead..93bfef475 100644 --- a/netpyne/sim/utils.py +++ b/netpyne/sim/utils.py @@ -592,39 +592,7 @@ def replaceFuncObj(obj): return obj -# ------------------------------------------------------------------------------ -# Replace None from dict or list with [](so can be saved to .mat) -# ------------------------------------------------------------------------------ -def replaceNoneObj(obj): - """ - Function for/to - - Parameters - ---------- - obj : - - **Default:** *required* - - - """ - - if type(obj) == list: # or type(obj) == tuple: - for item in obj: - if isinstance(item, (list, dict, Dict, ODict)): - replaceNoneObj(item) - - elif isinstance(obj, (dict, Dict, ODict)): - for key, val in obj.items(): - if isinstance(val, (list, dict, Dict, ODict)): - replaceNoneObj(val) - if val == None: - obj[key] = [] - elif val == {}: - obj[key] = [] # also replace empty dicts with empty list - return obj - - -# ------------------------------------------------------------------------------ +#------------------------------------------------------------------------------ # Replace Dict with dict and Odict with OrderedDict # ------------------------------------------------------------------------------ def replaceDictODict(obj): @@ -641,11 +609,12 @@ def replaceDictODict(obj): """ if type(obj) == list: - for item in obj: + for ind, item in enumerate(obj): if type(item) == Dict: item = item.todict() elif type(item) == ODict: item = item.toOrderedDict() + obj[ind] = item if type(item) in [list, dict, OrderedDict]: replaceDictODict(item) @@ -799,47 +768,106 @@ def clearObj(obj): # ------------------------------------------------------------------------------ # Support funcs to load from mat -# ------------------------------------------------------------------------------ -def _mat2dict(obj): +#------------------------------------------------------------------------------ + +def _ensureMatCompatible(obj): + """ + Function for/to + + Parameters + ---------- + obj : + + **Default:** *required* + + + """ + + + if type(obj) == list:# or type(obj) == tuple: + for item in obj: + if isinstance(item, (list, dict, Dict, ODict)): + _ensureMatCompatible(item) + + elif isinstance(obj, (dict, Dict, ODict)): + for key,val in obj.items(): + if isinstance(val, (list, dict, Dict, ODict)): + _ensureMatCompatible(val) + if val is None: + obj[key] = '__np_none__' + elif isinstance(val, list) and len(val) > 0 and all(isinstance(s, str) for s in val): + # string arrays get corrupted while saving to mat, so converting it to csv beforehand + elements = ','.join(val) + obj[key] = f'[{elements}]' + return obj + +def _restoreFromMat(obj): + """ + Function for/to + + Parameters + ---------- + obj : + + **Default:** *required* + + + """ + + + if type(obj) == list: + for item in obj: + if isinstance(item, (list, dict, Dict, ODict)): + _restoreFromMat(item) + + elif isinstance(obj, (dict, Dict, ODict)): + for key,val in obj.items(): + if isinstance(val, (list, dict, Dict, ODict)): + _restoreFromMat(val) + if val == '__np_none__': + obj[key] = None + elif isinstance(val, str) and val.startswith('[') and val.endswith(']'): + obj[key] = val[1:-1].split(',') + return obj + +def _mat2dict(obj, parentKey=None): """ A recursive function which constructs from matobjects nested dictionaries Enforce lists for conns, synMechs and stims even if 1 element (matlab converts to dict otherwise) """ - import scipy.io as spio + from scipy.io.matlab.mio5_params import mat_struct import numpy as np - if isinstance(obj, dict): + isMatStruct = isinstance(obj, mat_struct) + if isinstance(obj, (dict, mat_struct)): out = {} - for key in obj: - if isinstance(obj[key], spio.matlab.mio5_params.mat_struct): - if key in ['conns', 'stims', 'synMechs']: - out[key] = [_mat2dict(obj[key])] # convert to 1-element list + if isMatStruct: fieldNames = obj._fieldnames + else: fieldNames = obj + + keysOfLists = ['conns', 'stims', 'synMechs', 'cells', 'cellGids', 'include'] + if parentKey == 'tags': + keysOfLists.append('label') + for key in fieldNames: + if isMatStruct: val = obj.__dict__[key] + else: val = obj[key] + + if isinstance(val, mat_struct): + if key in keysOfLists: + out[key] = [_mat2dict(val, parentKey=key)] # convert to 1-element list else: - out[key] = _mat2dict(obj[key]) - elif isinstance(obj[key], np.ndarray): - out[key] = _mat2dict(obj[key]) - else: - out[key] = obj[key] - - elif isinstance(obj, spio.matlab.mio5_params.mat_struct): - out = {} - for key in obj._fieldnames: - val = obj.__dict__[key] - if isinstance(val, spio.matlab.mio5_params.mat_struct): - if key in ['conns', 'stims', 'synMechs']: - out[key] = [_mat2dict(val)] # convert to 1-element list - else: - out[key] = _mat2dict(val) + out[key] = _mat2dict(val, parentKey=key) elif isinstance(val, np.ndarray): - out[key] = _mat2dict(val) + out[key] = _mat2dict(val, parentKey=key) + elif key in keysOfLists: # isinstance(val, Number) and + out[key] = [val] # convert to 1-element list else: out[key] = val elif isinstance(obj, np.ndarray): out = [] for item in obj: - if isinstance(item, spio.matlab.mio5_params.mat_struct) or isinstance(item, np.ndarray): + if isinstance(item, mat_struct) or isinstance(item, np.ndarray): out.append(_mat2dict(item)) else: out.append(item) @@ -851,23 +879,22 @@ def _mat2dict(obj): # ------------------------------------------------------------------------------ -# Convert dict strings to utf8 so can be saved in HDF5 format +# Support funcs to save in HDF5 format # ------------------------------------------------------------------------------ -def _dict2utf8(obj): - # unidict = {k.decode('utf8'): v.decode('utf8') for k, v in strdict.items()} - # print obj +def _ensureHDF5Compatible(obj): import collections if isinstance(obj, basestring): - return obj.decode('utf8') + return obj elif isinstance(obj, collections.Mapping): for key in list(obj.keys()): if isinstance(key, Number): - obj[str(key).decode('utf8')] = obj[key] - obj.pop(key) - return dict(list(map(_dict2utf8, iter(obj.items())))) + obj[str(key)] = obj.pop(key) + return dict(list(map(_ensureHDF5Compatible, iter(obj.items())))) + elif isinstance(obj, np.ndarray): + return obj.tolist() elif isinstance(obj, collections.Iterable): - return type(obj)(list(map(_dict2utf8, obj))) + return type(obj)(list(map(_ensureHDF5Compatible, obj))) else: return obj From 428508d319ce068d5baefd6f6f0794bf051c14a6 Mon Sep 17 00:00:00 2001 From: vvbragin Date: Wed, 3 May 2023 22:38:50 +0200 Subject: [PATCH 18/65] Bug fixes in plotRatePSD. Allow to specify scale (linear/log) --- CHANGES.md | 4 ++++ netpyne/analysis/spikes_legacy.py | 31 ++++++++++++++++++++----------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8ae8bc4b8..139797462 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,12 +4,16 @@ - Extended sim.gatherData() with more optional arguments for flexibility +- Specify linear/log scale in `plotRatePSD()` + **Bug fixes** - Fixed errors in plotting with sim.cfg.gatherOnlySimData=True - Support saving and loading data in .mat and .hdf5 formats +- Multiple fixes in `plotRatePSD()` - popColors, fft, minFreq + # Version 1.0.4.1 **New features** diff --git a/netpyne/analysis/spikes_legacy.py b/netpyne/analysis/spikes_legacy.py index 335ebd921..fbbf7e483 100755 --- a/netpyne/analysis/spikes_legacy.py +++ b/netpyne/analysis/spikes_legacy.py @@ -1795,7 +1795,8 @@ def plotRatePSD( smooth=0, norm=False, overlay=True, - popColors=None, + popColors={}, + yLogScale = True, ylim=None, figSize=(10, 8), fontSize=12, @@ -1884,6 +1885,11 @@ def plotRatePSD( **Default:** ``None`` uses standard colors **Options:** ``