From 3be3d22ddeb93af45cd829c37ab50f69d4800590 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 25 Sep 2024 09:36:18 +0100 Subject: [PATCH 01/40] get_protocol_period: relax assertion for spacer_times --- ibllib/io/extractors/ephys_fpga.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibllib/io/extractors/ephys_fpga.py b/ibllib/io/extractors/ephys_fpga.py index 3810abf29..eb6c5141d 100644 --- a/ibllib/io/extractors/ephys_fpga.py +++ b/ibllib/io/extractors/ephys_fpga.py @@ -569,7 +569,8 @@ def get_protocol_period(session_path, protocol_number, bpod_sync): # Ensure that the number of detected spacers matched the number of expected tasks if acquisition_description := session_params.read_params(session_path): n_tasks = len(acquisition_description.get('tasks', [])) - assert n_tasks == len(spacer_times), f'expected {n_tasks} spacers, found {len(spacer_times)}' + assert len(spacer_times) >= protocol_number, (f'expected {n_tasks} spacers, found only {len(spacer_times)} - ' + f'can not return protocol number {protocol_number}.') assert n_tasks > protocol_number >= 0, f'protocol number must be between 0 and {n_tasks}' else: assert protocol_number < len(spacer_times) From a607f241d9f0d8fda5e1b71c150993a036e8cee8 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 25 Sep 2024 09:40:55 +0100 Subject: [PATCH 02/40] lower alpha for event markers --- ibllib/qc/task_qc_viewer/task_qc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py index 89a8d172f..86767328a 100644 --- a/ibllib/qc/task_qc_viewer/task_qc.py +++ b/ibllib/qc/task_qc_viewer/task_qc.py @@ -140,7 +140,8 @@ def create_plots(self, axes, 'ymin': 0, 'ymax': 4, 'linewidth': 2, - 'ax': axes + 'ax': axes, + 'alpha': 0.5, } bnc1 = self.qc.extractor.frame_ttls From 68beb147cc244fc525992d11d0512b9a10c51496 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 25 Sep 2024 12:34:17 +0100 Subject: [PATCH 03/40] some cleaning-up --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 97 ++++++++++++++----------- 1 file changed, 53 insertions(+), 44 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 48155b270..3f3717d54 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -1,7 +1,8 @@ """An interactive PyQT QC data frame.""" import logging -from PyQt5 import QtCore, QtWidgets +from PyQt5 import QtWidgets +from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, pyqtSlot from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT import pandas as pd @@ -12,13 +13,13 @@ _logger = logging.getLogger(__name__) -class DataFrameModel(QtCore.QAbstractTableModel): - DtypeRole = QtCore.Qt.UserRole + 1000 - ValueRole = QtCore.Qt.UserRole + 1001 +class DataFrameTableModel(QAbstractTableModel): + DtypeRole = Qt.UserRole + 1000 + ValueRole = Qt.UserRole + 1001 - def __init__(self, df=pd.DataFrame(), parent=None): - super(DataFrameModel, self).__init__(parent) - self._dataframe = df + def __init__(self, parent=None, dataFrame: pd.DataFrame = pd.DataFrame()): + super(DataFrameTableModel, self).__init__(parent) + self._dataframe = dataFrame def setDataFrame(self, dataframe): self.beginResetModel() @@ -28,50 +29,50 @@ def setDataFrame(self, dataframe): def dataFrame(self): return self._dataframe - dataFrame = QtCore.pyqtProperty(pd.DataFrame, fget=dataFrame, fset=setDataFrame) + dataFrame = pyqtProperty(pd.DataFrame, fget=dataFrame, fset=setDataFrame) - @QtCore.pyqtSlot(int, QtCore.Qt.Orientation, result=str) - def headerData(self, section: int, orientation: QtCore.Qt.Orientation, - role: int = QtCore.Qt.DisplayRole): - if role == QtCore.Qt.DisplayRole: - if orientation == QtCore.Qt.Horizontal: + @pyqtSlot(int, Qt.Orientation, result=str) + def headerData(self, section: int, orientation: Qt.Orientation, + role: int = Qt.DisplayRole): + if role == Qt.DisplayRole: + if orientation == Qt.Horizontal: return self._dataframe.columns[section] else: return str(self._dataframe.index[section]) - return QtCore.QVariant() + return QVariant() - def rowCount(self, parent=QtCore.QModelIndex()): + def rowCount(self, parent=QModelIndex()): if parent.isValid(): return 0 return len(self._dataframe.index) - def columnCount(self, parent=QtCore.QModelIndex()): + def columnCount(self, parent=QModelIndex()): if parent.isValid(): return 0 return self._dataframe.columns.size - def data(self, index, role=QtCore.Qt.DisplayRole): + def data(self, index, role=Qt.DisplayRole): if (not index.isValid() or not (0 <= index.row() < self.rowCount() and 0 <= index.column() < self.columnCount())): - return QtCore.QVariant() + return QVariant() row = self._dataframe.index[index.row()] col = self._dataframe.columns[index.column()] dt = self._dataframe[col].dtype val = self._dataframe.iloc[row][col] - if role == QtCore.Qt.DisplayRole: + if role == Qt.DisplayRole: return str(val) - elif role == DataFrameModel.ValueRole: + elif role == DataFrameTableModel.ValueRole: return val - if role == DataFrameModel.DtypeRole: + if role == DataFrameTableModel.DtypeRole: return dt - return QtCore.QVariant() + return QVariant() def roleNames(self): roles = { - QtCore.Qt.DisplayRole: b'display', - DataFrameModel.DtypeRole: b'dtype', - DataFrameModel.ValueRole: b'value' + Qt.DisplayRole: b'display', + DataFrameTableModel.DtypeRole: b'dtype', + DataFrameTableModel.ValueRole: b'value' } return roles @@ -83,6 +84,8 @@ def sort(self, col, order): :param order: the order to be sorted, 0 is descending; 1, ascending :return: """ + if self._dataframe.empty: + return self.layoutAboutToBeChanged.emit() col_name = self._dataframe.columns.values[col] # print('sorting by ' + col_name) @@ -125,37 +128,43 @@ def __init__(self, parent=None, wheel=None): class GraphWindow(QtWidgets.QWidget): def __init__(self, parent=None, wheel=None): QtWidgets.QWidget.__init__(self, parent=parent) + self.lineEditPath = QtWidgets.QLineEdit(self) + + self.pushButtonLoad = QtWidgets.QPushButton("Select File", self) + self.pushButtonLoad.clicked.connect(self.loadFile) + + self.tableModel = DataFrameTableModel(self) + self.tableView = QtWidgets.QTableView(self) + self.tableView.setModel(self.tableModel) + self.tableView.setSortingEnabled(True) + self.tableView.doubleClicked.connect(self.tv_double_clicked) + vLayout = QtWidgets.QVBoxLayout(self) hLayout = QtWidgets.QHBoxLayout() - self.pathLE = QtWidgets.QLineEdit(self) - hLayout.addWidget(self.pathLE) - self.loadBtn = QtWidgets.QPushButton("Select File", self) - hLayout.addWidget(self.loadBtn) + hLayout.addWidget(self.lineEditPath) + hLayout.addWidget(self.pushButtonLoad) vLayout.addLayout(hLayout) - self.pandasTv = QtWidgets.QTableView(self) - vLayout.addWidget(self.pandasTv) - self.loadBtn.clicked.connect(self.load_file) - self.pandasTv.setSortingEnabled(True) - self.pandasTv.doubleClicked.connect(self.tv_double_clicked) + vLayout.addWidget(self.tableView) + self.wplot = PlotWindow(wheel=wheel) self.wplot.show() + self.tableModel.dataChanged.connect(self.wplot.canvas.draw) + self.wheel = wheel - def load_file(self): + def loadFile(self): fileName, _ = QtWidgets.QFileDialog.getOpenFileName( self, "Open File", "", "CSV Files (*.csv)") - self.pathLE.setText(fileName) + self.lineEditPath.setText(fileName) df = pd.read_csv(fileName) - self.update_df(df) + self.updateDataframe(df) - def update_df(self, df): - model = DataFrameModel(df) - self.pandasTv.setModel(model) - self.wplot.canvas.draw() + def updateDataframe(self, dataFrame: pd.DataFrame): + self.tableModel.setDataFrame(dataFrame) def tv_double_clicked(self): - df = self.pandasTv.model()._dataframe - ind = self.pandasTv.currentIndex() + df = self.tableView.model()._dataframe + ind = self.tableView.currentIndex() start = df.loc[ind.row()]['intervals_0'] finish = df.loc[ind.row()]['intervals_1'] dt = finish - start @@ -179,6 +188,6 @@ def viewqc(qc=None, title=None, wheel=None): qcw = GraphWindow(wheel=wheel) qcw.setWindowTitle(title) if qc is not None: - qcw.update_df(qc) + qcw.updateDataframe(qc) qcw.show() return qcw From 9f99ce68e7737d8114d8a0e1614388cd8ceaf678 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 25 Sep 2024 13:58:21 +0100 Subject: [PATCH 04/40] remove unused rolenames from DataFrameTableModel, various small fixes --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 74 +++++++++++-------------- 1 file changed, 32 insertions(+), 42 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 3f3717d54..507099767 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -1,4 +1,5 @@ """An interactive PyQT QC data frame.""" + import logging from PyQt5 import QtWidgets @@ -14,26 +15,24 @@ class DataFrameTableModel(QAbstractTableModel): - DtypeRole = Qt.UserRole + 1000 - ValueRole = Qt.UserRole + 1001 - def __init__(self, parent=None, dataFrame: pd.DataFrame = pd.DataFrame()): super(DataFrameTableModel, self).__init__(parent) self._dataframe = dataFrame - def setDataFrame(self, dataframe): + def setDataFrame(self, dataFrame: pd.DataFrame): self.beginResetModel() - self._dataframe = dataframe.copy() + self._dataframe = dataFrame.copy() self.endResetModel() - def dataFrame(self): + def dataFrame(self) -> pd.DataFrame: return self._dataframe dataFrame = pyqtProperty(pd.DataFrame, fget=dataFrame, fset=setDataFrame) @pyqtSlot(int, Qt.Orientation, result=str) - def headerData(self, section: int, orientation: Qt.Orientation, - role: int = Qt.DisplayRole): + def headerData( + self, section: int, orientation: Qt.Orientation, role: int = Qt.DisplayRole + ): if role == Qt.DisplayRole: if orientation == Qt.Horizontal: return self._dataframe.columns[section] @@ -52,30 +51,19 @@ def columnCount(self, parent=QModelIndex()): return self._dataframe.columns.size def data(self, index, role=Qt.DisplayRole): - if (not index.isValid() or not (0 <= index.row() < self.rowCount() and - 0 <= index.column() < self.columnCount())): + if not index.isValid(): return QVariant() row = self._dataframe.index[index.row()] col = self._dataframe.columns[index.column()] - dt = self._dataframe[col].dtype - val = self._dataframe.iloc[row][col] if role == Qt.DisplayRole: + if isinstance(val, np.generic): + return val.item() return str(val) - elif role == DataFrameTableModel.ValueRole: - return val - if role == DataFrameTableModel.DtypeRole: - return dt + # elif role == Qt.BackgroundRole: + # return QBrush(Qt.red) return QVariant() - def roleNames(self): - roles = { - Qt.DisplayRole: b'display', - DataFrameTableModel.DtypeRole: b'dtype', - DataFrameTableModel.ValueRole: b'value' - } - return roles - def sort(self, col, order): """ Sort table by given column number. @@ -84,7 +72,7 @@ def sort(self, col, order): :param order: the order to be sorted, 0 is descending; 1, ascending :return: """ - if self._dataframe.empty: + if self.columnCount() == 0: return self.layoutAboutToBeChanged.emit() col_name = self._dataframe.columns.values[col] @@ -95,7 +83,6 @@ def sort(self, col, order): class PlotCanvas(FigureCanvasQTAgg): - def __init__(self, parent=None, width=5, height=4, dpi=100, wheel=None): fig = Figure(figsize=(width, height), dpi=dpi) @@ -103,13 +90,13 @@ def __init__(self, parent=None, width=5, height=4, dpi=100, wheel=None): self.setParent(parent) FigureCanvasQTAgg.setSizePolicy( - self, - QtWidgets.QSizePolicy.Expanding, - QtWidgets.QSizePolicy.Expanding) + self, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding + ) FigureCanvasQTAgg.updateGeometry(self) if wheel: self.ax, self.ax2 = fig.subplots( - 2, 1, gridspec_kw={'height_ratios': [2, 1]}, sharex=True) + 2, 1, gridspec_kw={"height_ratios": [2, 1]}, sharex=True + ) else: self.ax = fig.add_subplot(111) self.draw() @@ -119,7 +106,7 @@ class PlotWindow(QtWidgets.QWidget): def __init__(self, parent=None, wheel=None): QtWidgets.QWidget.__init__(self, parent=None) self.canvas = PlotCanvas(wheel=wheel) - self.vbl = QtWidgets.QVBoxLayout() # Set box for plotting + self.vbl = QtWidgets.QVBoxLayout() # Set box for plotting self.vbl.addWidget(self.canvas) self.setLayout(self.vbl) self.vbl.addWidget(NavigationToolbar2QT(self.canvas, self)) @@ -137,6 +124,7 @@ def __init__(self, parent=None, wheel=None): self.tableView = QtWidgets.QTableView(self) self.tableView.setModel(self.tableModel) self.tableView.setSortingEnabled(True) + self.tableView.horizontalHeader().setDefaultAlignment(Qt.AlignLeft | Qt.AlignVCenter) self.tableView.doubleClicked.connect(self.tv_double_clicked) vLayout = QtWidgets.QVBoxLayout(self) @@ -154,7 +142,10 @@ def __init__(self, parent=None, wheel=None): def loadFile(self): fileName, _ = QtWidgets.QFileDialog.getOpenFileName( - self, "Open File", "", "CSV Files (*.csv)") + self, "Open File", "", "CSV Files (*.csv)" + ) + if len(fileName) == 0: + return self.lineEditPath.setText(fileName) df = pd.read_csv(fileName) self.updateDataframe(df) @@ -163,22 +154,21 @@ def updateDataframe(self, dataFrame: pd.DataFrame): self.tableModel.setDataFrame(dataFrame) def tv_double_clicked(self): - df = self.tableView.model()._dataframe ind = self.tableView.currentIndex() - start = df.loc[ind.row()]['intervals_0'] - finish = df.loc[ind.row()]['intervals_1'] - dt = finish - start + data = self.tableModel.dataFrame.loc[ind.row()] + t0 = data["intervals_0"] + t1 = data["intervals_1"] + dt = t1 - t0 if self.wheel: - idx = np.searchsorted( - self.wheel['re_ts'], np.array([start - dt / 10, finish + dt / 10])) - period = self.wheel['re_pos'][idx[0]:idx[1]] + idx = np.searchsorted(self.wheel["re_ts"], np.array([t0 - dt / 10, t1 + dt / 10])) + period = self.wheel["re_pos"][idx[0] : idx[1]] if period.size == 0: - _logger.warning('No wheel data during trial #%i', ind.row()) + _logger.warning("No wheel data during trial #%i", ind.row()) else: min_val, max_val = np.min(period), np.max(period) self.wplot.canvas.ax2.set_ylim(min_val - 1, max_val + 1) - self.wplot.canvas.ax2.set_xlim(start - dt / 10, finish + dt / 10) - self.wplot.canvas.ax.set_xlim(start - dt / 10, finish + dt / 10) + self.wplot.canvas.ax2.set_xlim(t0 - dt / 10, t1 + dt / 10) + self.wplot.canvas.ax.set_xlim(t0 - dt / 10, t1 + dt / 10) self.wplot.canvas.draw() From bfbe1007c119de768ac6c20fe1239f8fa3bd9c93 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 25 Sep 2024 15:21:10 +0100 Subject: [PATCH 05/40] Update task_qc.py --- ibllib/qc/task_qc_viewer/task_qc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py index 86767328a..b8fc1749f 100644 --- a/ibllib/qc/task_qc_viewer/task_qc.py +++ b/ibllib/qc/task_qc_viewer/task_qc.py @@ -285,8 +285,11 @@ def show_session_task_qc(qc_or_session=None, bpod_only=False, local=False, one=N trial_events=list(events), color_map=cm, linestyle=ls) + # Update table and callbacks - w.update_df(qc.frame) + n_trials = qc.frame.shape[0] + df_trials = pd.DataFrame({k: v for k, v in task_qc.extractor.data.items() if v.size == n_trials}) + w.updateDataframe(df_trials.merge(qc.frame, left_index=True, right_index=True)) qt.run_app() return qc From 6ae91f6656648a004f90901159fa6996011a2e8e Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 25 Sep 2024 20:19:50 +0100 Subject: [PATCH 06/40] add ColoredDataFrameTableModel --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 81 +++++++++++++++++-------- 1 file changed, 55 insertions(+), 26 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 507099767..cbde17809 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -3,7 +3,9 @@ import logging from PyQt5 import QtWidgets -from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, pyqtSlot +from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, QObject +from PyQt5.QtGui import QBrush, QColor +import matplotlib.pyplot as plt from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT import pandas as pd @@ -15,8 +17,8 @@ class DataFrameTableModel(QAbstractTableModel): - def __init__(self, parent=None, dataFrame: pd.DataFrame = pd.DataFrame()): - super(DataFrameTableModel, self).__init__(parent) + def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame = pd.DataFrame()): + super().__init__(parent) self._dataframe = dataFrame def setDataFrame(self, dataFrame: pd.DataFrame): @@ -29,10 +31,7 @@ def dataFrame(self) -> pd.DataFrame: dataFrame = pyqtProperty(pd.DataFrame, fget=dataFrame, fset=setDataFrame) - @pyqtSlot(int, Qt.Orientation, result=str) - def headerData( - self, section: int, orientation: Qt.Orientation, role: int = Qt.DisplayRole - ): + def headerData(self, section: int, orientation: Qt.Orientation, role: int = ...): if role == Qt.DisplayRole: if orientation == Qt.Horizontal: return self._dataframe.columns[section] @@ -40,17 +39,17 @@ def headerData( return str(self._dataframe.index[section]) return QVariant() - def rowCount(self, parent=QModelIndex()): - if parent.isValid(): + def rowCount(self, parent: QModelIndex = ...): + if isinstance(parent, QModelIndex) and parent.isValid(): return 0 return len(self._dataframe.index) - def columnCount(self, parent=QModelIndex()): - if parent.isValid(): + def columnCount(self, parent: QModelIndex = ...): + if isinstance(parent, QModelIndex) and parent.isValid(): return 0 return self._dataframe.columns.size - def data(self, index, role=Qt.DisplayRole): + def data(self, index: QModelIndex, role: int = ...) -> QVariant: if not index.isValid(): return QVariant() row = self._dataframe.index[index.row()] @@ -59,29 +58,59 @@ def data(self, index, role=Qt.DisplayRole): if role == Qt.DisplayRole: if isinstance(val, np.generic): return val.item() - return str(val) - # elif role == Qt.BackgroundRole: - # return QBrush(Qt.red) + return QVariant(str(val)) return QVariant() - def sort(self, col, order): - """ - Sort table by given column number. - - :param col: the column number selected (between 0 and self._dataframe.columns.size) - :param order: the order to be sorted, 0 is descending; 1, ascending - :return: - """ + def sort(self, column: int, order: Qt.SortOrder = ...): if self.columnCount() == 0: return self.layoutAboutToBeChanged.emit() - col_name = self._dataframe.columns.values[col] - # print('sorting by ' + col_name) + col_name = self._dataframe.columns.values[column] self._dataframe.sort_values(by=col_name, ascending=not order, inplace=True) self._dataframe.reset_index(inplace=True, drop=True) self.layoutChanged.emit() +class ColoredDataFrameTableModel(DataFrameTableModel): + _colors: pd.DataFrame + _cmap = plt.get_cmap('plasma') + _alpha = 0.5 + + def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame = pd.DataFrame()): + super().__init__(parent=parent, dataFrame=dataFrame) + self._setColors() + self.modelReset.connect(self._setColors) + self.dataChanged.connect(self._setColors) + self.layoutChanged.connect(self._setColors) + + def _setColors(self): + df = self._dataframe.copy() + df = df.replace([np.inf, -np.inf], np.nan) + for col in df.select_dtypes(include=['bool']): + df[col] = df[col].astype(float) + for col in df.select_dtypes(exclude=['bool']): + df[col] = pd.to_numeric(df[col], errors='coerce').astype(float) + if df[col].nunique() == 1: + df[col] = QColor.fromRgb(*self._cmap(0, self._alpha, True)) + else: + df[col] = (df[col] - df[col].min()) / (df[col].max() - df[col].min()) + df[col] = [QColor.fromRgb(*x) for x in self._cmap(df[col], self._alpha, True).tolist()] + self._colors = df + + def data(self, index, role=...): + if not index.isValid(): + return QVariant() + if role == Qt.BackgroundRole: + row = self._dataframe.index[index.row()] + col = self._dataframe.columns[index.column()] + val = self._dataframe.iloc[row][col] + if isinstance(val, (np.bool_, np.number)) and not np.isnan(val): + return self._colors.iloc[row][col] + else: + return QBrush(Qt.white) + return super().data(index, role) + + class PlotCanvas(FigureCanvasQTAgg): def __init__(self, parent=None, width=5, height=4, dpi=100, wheel=None): fig = Figure(figsize=(width, height), dpi=dpi) @@ -120,7 +149,7 @@ def __init__(self, parent=None, wheel=None): self.pushButtonLoad = QtWidgets.QPushButton("Select File", self) self.pushButtonLoad.clicked.connect(self.loadFile) - self.tableModel = DataFrameTableModel(self) + self.tableModel = ColoredDataFrameTableModel(self) self.tableView = QtWidgets.QTableView(self) self.tableView.setModel(self.tableModel) self.tableView.setSortingEnabled(True) From 5644b352f8037b2a15a5fe99f8821b732319aa0d Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 25 Sep 2024 22:12:28 +0100 Subject: [PATCH 07/40] Update ViewEphysQC.py --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 47 ++++++++++++++++--------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index cbde17809..a1b46f174 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -10,6 +10,7 @@ from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT import pandas as pd import numpy as np +from sklearn.preprocessing import MinMaxScaler from ibllib.misc import qt @@ -84,18 +85,34 @@ def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame = pd.DataFrame self.layoutChanged.connect(self._setColors) def _setColors(self): - df = self._dataframe.copy() - df = df.replace([np.inf, -np.inf], np.nan) - for col in df.select_dtypes(include=['bool']): - df[col] = df[col].astype(float) - for col in df.select_dtypes(exclude=['bool']): - df[col] = pd.to_numeric(df[col], errors='coerce').astype(float) - if df[col].nunique() == 1: - df[col] = QColor.fromRgb(*self._cmap(0, self._alpha, True)) - else: - df[col] = (df[col] - df[col].min()) / (df[col].max() - df[col].min()) - df[col] = [QColor.fromRgb(*x) for x in self._cmap(df[col], self._alpha, True).tolist()] - self._colors = df + vals = self._dataframe.copy() + if vals.empty: + self._colors = vals + return + + # coerce non-bool / non-numeric values to numeric + for col in vals.select_dtypes(exclude=['bool', 'number']): + vals[col] = vals[col].to_numeric(errors='coerce') + + # normalize numeric values + cols = vals.select_dtypes(include=['number']).columns + vals.replace([np.inf, -np.inf], np.nan, inplace=True) + vals[cols] = MinMaxScaler().fit_transform(vals[cols]) + + # convert boolean values + cols = vals.select_dtypes(include=['bool']).columns + vals[cols] = vals[cols].astype(float) + + # assign QColors + colors = vals.astype(object) + for col in vals.columns: + colors[col] = [QColor.fromRgb(*x) for x in self._cmap(vals[col], self._alpha, True)] + + # NaNs should be white + nans = vals.isna() + colors[nans] = QColor('white') + + self._colors = colors def data(self, index, role=...): if not index.isValid(): @@ -103,11 +120,7 @@ def data(self, index, role=...): if role == Qt.BackgroundRole: row = self._dataframe.index[index.row()] col = self._dataframe.columns[index.column()] - val = self._dataframe.iloc[row][col] - if isinstance(val, (np.bool_, np.number)) and not np.isnan(val): - return self._colors.iloc[row][col] - else: - return QBrush(Qt.white) + return self._colors.iloc[row][col] return super().data(index, role) From 074b01a09f79ca119a83788fd4b3b3b888f786f1 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Thu, 26 Sep 2024 00:38:39 +0100 Subject: [PATCH 08/40] Update ViewEphysQC.py --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 71 ++++++++++++------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index a1b46f174..4e1f31c5c 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -6,11 +6,11 @@ from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, QObject from PyQt5.QtGui import QBrush, QColor import matplotlib.pyplot as plt +from matplotlib.colors import ListedColormap from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT import pandas as pd import numpy as np -from sklearn.preprocessing import MinMaxScaler from ibllib.misc import qt @@ -18,9 +18,9 @@ class DataFrameTableModel(QAbstractTableModel): - def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame = pd.DataFrame()): + def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None): super().__init__(parent) - self._dataframe = dataFrame + self._dataframe = pd.DataFrame() if dataFrame is None else dataFrame def setDataFrame(self, dataFrame: pd.DataFrame): self.beginResetModel() @@ -73,54 +73,53 @@ def sort(self, column: int, order: Qt.SortOrder = ...): class ColoredDataFrameTableModel(DataFrameTableModel): - _colors: pd.DataFrame - _cmap = plt.get_cmap('plasma') - _alpha = 0.5 + _rgba: np.ndarray + _cmap: ListedColormap - def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame = pd.DataFrame()): + def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, + colorMap: ListedColormap | None = None, alpha: float = 0.5): super().__init__(parent=parent, dataFrame=dataFrame) - self._setColors() - self.modelReset.connect(self._setColors) - self.dataChanged.connect(self._setColors) - self.layoutChanged.connect(self._setColors) - - def _setColors(self): - vals = self._dataframe.copy() - if vals.empty: - self._colors = vals + + self._alpha = alpha + if colorMap is None: + self._cmap = plt.get_cmap('plasma') + self._cmap.set_bad(color='w') + else: + self._cmap = colorMap + + self._setRgba() + self.modelReset.connect(self._setRgba) + self.dataChanged.connect(self._setRgba) + self.layoutChanged.connect(self._setRgba) + + def _setRgba(self): + values = self._dataframe.copy() + if values.empty: + self._rgba = values return # coerce non-bool / non-numeric values to numeric - for col in vals.select_dtypes(exclude=['bool', 'number']): - vals[col] = vals[col].to_numeric(errors='coerce') + cols = values.select_dtypes(exclude=['bool', 'number']).columns + values[cols] = values[cols].apply(pd.to_numeric, errors='coerce') # normalize numeric values - cols = vals.select_dtypes(include=['number']).columns - vals.replace([np.inf, -np.inf], np.nan, inplace=True) - vals[cols] = MinMaxScaler().fit_transform(vals[cols]) + cols = values.select_dtypes(include=['number']).columns + values.replace([np.inf, -np.inf], np.nan, inplace=True) + values[cols] -= values[cols].min() + values[cols] /= values[cols].max() # convert boolean values - cols = vals.select_dtypes(include=['bool']).columns - vals[cols] = vals[cols].astype(float) - - # assign QColors - colors = vals.astype(object) - for col in vals.columns: - colors[col] = [QColor.fromRgb(*x) for x in self._cmap(vals[col], self._alpha, True)] - - # NaNs should be white - nans = vals.isna() - colors[nans] = QColor('white') + cols = values.select_dtypes(include=['bool']).columns + values[cols] = values[cols].astype(float) - self._colors = colors + # store color values to ndarray + self._rgba = self._cmap(values, self._alpha, True) def data(self, index, role=...): if not index.isValid(): return QVariant() if role == Qt.BackgroundRole: - row = self._dataframe.index[index.row()] - col = self._dataframe.columns[index.column()] - return self._colors.iloc[row][col] + return QColor.fromRgb(*self._rgba[index.row(), index.column()]) return super().data(index, role) From 9203f04436b77fc88ec7e1460518d86fab25ef6f Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Thu, 26 Sep 2024 00:58:23 +0100 Subject: [PATCH 09/40] Update ViewEphysQC.py --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 4e1f31c5c..f101435e5 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -103,10 +103,11 @@ def _setRgba(self): values[cols] = values[cols].apply(pd.to_numeric, errors='coerce') # normalize numeric values - cols = values.select_dtypes(include=['number']).columns values.replace([np.inf, -np.inf], np.nan, inplace=True) + cols = values.select_dtypes(include=['number']).columns + values[cols].astype(float) values[cols] -= values[cols].min() - values[cols] /= values[cols].max() + values[cols] = values[cols].div(values[cols].max()).replace(np.inf, 0, inplace=True) # convert boolean values cols = values.select_dtypes(include=['bool']).columns From 2f8a5c9ee380d2844b6287865114fe7715ff1b8b Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Thu, 26 Sep 2024 01:43:34 +0100 Subject: [PATCH 10/40] Update ViewEphysQC.py --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 32 +++++++++++++------------ 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index f101435e5..5836275cf 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -93,28 +93,30 @@ def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, self.layoutChanged.connect(self._setRgba) def _setRgba(self): - values = self._dataframe.copy() - if values.empty: - self._rgba = values + df = self._dataframe.copy() + if df.empty: + self._rgba = df return # coerce non-bool / non-numeric values to numeric - cols = values.select_dtypes(exclude=['bool', 'number']).columns - values[cols] = values[cols].apply(pd.to_numeric, errors='coerce') - - # normalize numeric values - values.replace([np.inf, -np.inf], np.nan, inplace=True) - cols = values.select_dtypes(include=['number']).columns - values[cols].astype(float) - values[cols] -= values[cols].min() - values[cols] = values[cols].div(values[cols].max()).replace(np.inf, 0, inplace=True) + cols = df.select_dtypes(exclude=['bool', 'number']).columns + df[cols] = df[cols].apply(pd.to_numeric, errors='coerce') + + # normalize numeric values, avoiding inf values and division by zero + num_cols = df.select_dtypes(include=['number']).columns + df[num_cols].replace([np.inf, -np.inf], np.nan) + mask = df[num_cols].nunique(dropna=True) == 1 + cols = num_cols[mask] + df[cols] = df[cols].where(df[cols].isna(), other=0.0) + cols = num_cols[~mask] + df[cols] = (df[cols] - df[cols].min()) / (df[cols].max() - df[cols].min()) # convert boolean values - cols = values.select_dtypes(include=['bool']).columns - values[cols] = values[cols].astype(float) + cols = df.select_dtypes(include=['bool']).columns + df[cols] = df[cols].astype(float) # store color values to ndarray - self._rgba = self._cmap(values, self._alpha, True) + self._rgba = self._cmap(df, self._alpha, True) def data(self, index, role=...): if not index.isValid(): From 4b4b063cba242d82c69cffb3c33b4ec44bfb86c3 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Thu, 26 Sep 2024 09:59:58 +0100 Subject: [PATCH 11/40] moveable sections, tooltips for header --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 5836275cf..ed6847aff 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -6,6 +6,7 @@ from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, QObject from PyQt5.QtGui import QBrush, QColor import matplotlib.pyplot as plt +from PyQt5.QtWidgets import QStyledItemDelegate from matplotlib.colors import ListedColormap from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT @@ -33,7 +34,7 @@ def dataFrame(self) -> pd.DataFrame: dataFrame = pyqtProperty(pd.DataFrame, fget=dataFrame, fset=setDataFrame) def headerData(self, section: int, orientation: Qt.Orientation, role: int = ...): - if role == Qt.DisplayRole: + if role in (Qt.DisplayRole, Qt.ToolTipRole): if orientation == Qt.Horizontal: return self._dataframe.columns[section] else: @@ -103,12 +104,11 @@ def _setRgba(self): df[cols] = df[cols].apply(pd.to_numeric, errors='coerce') # normalize numeric values, avoiding inf values and division by zero - num_cols = df.select_dtypes(include=['number']).columns - df[num_cols].replace([np.inf, -np.inf], np.nan) - mask = df[num_cols].nunique(dropna=True) == 1 - cols = num_cols[mask] - df[cols] = df[cols].where(df[cols].isna(), other=0.0) - cols = num_cols[~mask] + cols = df.select_dtypes(include=['number']).columns + df[cols].replace([np.inf, -np.inf], np.nan) + m = df[cols].nunique() <= 1 # boolean mask for columns with only 1 unique value + df[cols[m]] = df[cols[m]].where(df[cols[m]].isna(), other=0.0) + cols = cols[~m] df[cols] = (df[cols] - df[cols].min()) / (df[cols].max() - df[cols].min()) # convert boolean values @@ -169,6 +169,8 @@ def __init__(self, parent=None, wheel=None): self.tableView.setModel(self.tableModel) self.tableView.setSortingEnabled(True) self.tableView.horizontalHeader().setDefaultAlignment(Qt.AlignLeft | Qt.AlignVCenter) + self.tableView.horizontalHeader().setSectionsMovable(True) + self.tableView.verticalHeader().hide() self.tableView.doubleClicked.connect(self.tv_double_clicked) vLayout = QtWidgets.QVBoxLayout(self) From ddbad74022d4a5aec46df12c4a65a8a579a05aed Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Thu, 26 Sep 2024 11:25:58 +0100 Subject: [PATCH 12/40] speed up sort and color handling --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index ed6847aff..2df0b91c3 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -54,10 +54,8 @@ def columnCount(self, parent: QModelIndex = ...): def data(self, index: QModelIndex, role: int = ...) -> QVariant: if not index.isValid(): return QVariant() - row = self._dataframe.index[index.row()] - col = self._dataframe.columns[index.column()] - val = self._dataframe.iloc[row][col] if role == Qt.DisplayRole: + val = self._dataframe.iloc[index.row()][index.column()] if isinstance(val, np.generic): return val.item() return QVariant(str(val)) @@ -66,10 +64,9 @@ def data(self, index: QModelIndex, role: int = ...) -> QVariant: def sort(self, column: int, order: Qt.SortOrder = ...): if self.columnCount() == 0: return + columnName = self._dataframe.columns.values[column] self.layoutAboutToBeChanged.emit() - col_name = self._dataframe.columns.values[column] - self._dataframe.sort_values(by=col_name, ascending=not order, inplace=True) - self._dataframe.reset_index(inplace=True, drop=True) + self._dataframe.sort_values(by=columnName, ascending=not order, inplace=True) self.layoutChanged.emit() @@ -91,7 +88,6 @@ def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, self._setRgba() self.modelReset.connect(self._setRgba) self.dataChanged.connect(self._setRgba) - self.layoutChanged.connect(self._setRgba) def _setRgba(self): df = self._dataframe.copy() @@ -122,7 +118,8 @@ def data(self, index, role=...): if not index.isValid(): return QVariant() if role == Qt.BackgroundRole: - return QColor.fromRgb(*self._rgba[index.row(), index.column()]) + row = self._dataframe.index[index.row()] + return QColor.fromRgb(*self._rgba[row][index.column()]) return super().data(index, role) @@ -170,7 +167,7 @@ def __init__(self, parent=None, wheel=None): self.tableView.setSortingEnabled(True) self.tableView.horizontalHeader().setDefaultAlignment(Qt.AlignLeft | Qt.AlignVCenter) self.tableView.horizontalHeader().setSectionsMovable(True) - self.tableView.verticalHeader().hide() + # self.tableView.verticalHeader().hide() self.tableView.doubleClicked.connect(self.tv_double_clicked) vLayout = QtWidgets.QVBoxLayout(self) From 2fbc5c681c3c5cddaf0d06d7167a96a06e712867 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Thu, 26 Sep 2024 12:25:34 +0100 Subject: [PATCH 13/40] add filter for column names --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 2df0b91c3..62ca7cb94 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -6,7 +6,6 @@ from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, QObject from PyQt5.QtGui import QBrush, QColor import matplotlib.pyplot as plt -from PyQt5.QtWidgets import QStyledItemDelegate from matplotlib.colors import ListedColormap from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT @@ -64,9 +63,9 @@ def data(self, index: QModelIndex, role: int = ...) -> QVariant: def sort(self, column: int, order: Qt.SortOrder = ...): if self.columnCount() == 0: return - columnName = self._dataframe.columns.values[column] + column = self._dataframe.columns[column] self.layoutAboutToBeChanged.emit() - self._dataframe.sort_values(by=columnName, ascending=not order, inplace=True) + self._dataframe.sort_values(by=column, ascending=not order, inplace=True) self.layoutChanged.emit() @@ -156,7 +155,10 @@ def __init__(self, parent=None, wheel=None): class GraphWindow(QtWidgets.QWidget): def __init__(self, parent=None, wheel=None): QtWidgets.QWidget.__init__(self, parent=parent) - self.lineEditPath = QtWidgets.QLineEdit(self) + + self.lineEditFilter = QtWidgets.QLineEdit(self) + self.lineEditFilter.setPlaceholderText('Filter columns by name') + self.lineEditFilter.textChanged.connect(self.changeFilter) self.pushButtonLoad = QtWidgets.QPushButton("Select File", self) self.pushButtonLoad.clicked.connect(self.loadFile) @@ -167,12 +169,12 @@ def __init__(self, parent=None, wheel=None): self.tableView.setSortingEnabled(True) self.tableView.horizontalHeader().setDefaultAlignment(Qt.AlignLeft | Qt.AlignVCenter) self.tableView.horizontalHeader().setSectionsMovable(True) - # self.tableView.verticalHeader().hide() + self.tableView.verticalHeader().hide() self.tableView.doubleClicked.connect(self.tv_double_clicked) vLayout = QtWidgets.QVBoxLayout(self) hLayout = QtWidgets.QHBoxLayout() - hLayout.addWidget(self.lineEditPath) + hLayout.addWidget(self.lineEditFilter) hLayout.addWidget(self.pushButtonLoad) vLayout.addLayout(hLayout) vLayout.addWidget(self.tableView) @@ -183,13 +185,18 @@ def __init__(self, parent=None, wheel=None): self.wheel = wheel + def changeFilter(self, string: str): + headers = [self.tableModel.headerData(x, Qt.Horizontal, Qt.DisplayRole) + for x in range(self.tableModel.columnCount())] + for idx, column in enumerate(headers): + self.tableView.setColumnHidden(idx, string.lower() not in column.lower()) + def loadFile(self): fileName, _ = QtWidgets.QFileDialog.getOpenFileName( self, "Open File", "", "CSV Files (*.csv)" ) if len(fileName) == 0: return - self.lineEditPath.setText(fileName) df = pd.read_csv(fileName) self.updateDataframe(df) From 7bfcf723ca4e2a3303228e871a2634abec0457fe Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Thu, 26 Sep 2024 13:11:09 +0100 Subject: [PATCH 14/40] Update ViewEphysQC.py --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 62ca7cb94..ea2688b3f 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -6,6 +6,7 @@ from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, QObject from PyQt5.QtGui import QBrush, QColor import matplotlib.pyplot as plt +from PyQt5.QtWidgets import QTableView from matplotlib.colors import ListedColormap from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT @@ -17,6 +18,8 @@ _logger = logging.getLogger(__name__) +class FreezeTableView(QTableView) + class DataFrameTableModel(QAbstractTableModel): def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None): super().__init__(parent) @@ -203,9 +206,8 @@ def loadFile(self): def updateDataframe(self, dataFrame: pd.DataFrame): self.tableModel.setDataFrame(dataFrame) - def tv_double_clicked(self): - ind = self.tableView.currentIndex() - data = self.tableModel.dataFrame.loc[ind.row()] + def tv_double_clicked(self, index: QModelIndex): + data = self.tableModel.dataFrame.iloc[index.row()] t0 = data["intervals_0"] t1 = data["intervals_1"] dt = t1 - t0 @@ -213,13 +215,13 @@ def tv_double_clicked(self): idx = np.searchsorted(self.wheel["re_ts"], np.array([t0 - dt / 10, t1 + dt / 10])) period = self.wheel["re_pos"][idx[0] : idx[1]] if period.size == 0: - _logger.warning("No wheel data during trial #%i", ind.row()) + _logger.warning("No wheel data during trial #%i", index.row()) else: min_val, max_val = np.min(period), np.max(period) self.wplot.canvas.ax2.set_ylim(min_val - 1, max_val + 1) self.wplot.canvas.ax2.set_xlim(t0 - dt / 10, t1 + dt / 10) self.wplot.canvas.ax.set_xlim(t0 - dt / 10, t1 + dt / 10) - + self.wplot.setWindowTitle(f"Trial {data.get('trial_no', '?')}") self.wplot.canvas.draw() From f227b9a7e3acc496d9a148214ccb9b53486285f4 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Thu, 26 Sep 2024 13:12:31 +0100 Subject: [PATCH 15/40] Update ViewEphysQC.py --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index ea2688b3f..22fbcef06 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -4,9 +4,8 @@ from PyQt5 import QtWidgets from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, QObject -from PyQt5.QtGui import QBrush, QColor +from PyQt5.QtGui import QColor import matplotlib.pyplot as plt -from PyQt5.QtWidgets import QTableView from matplotlib.colors import ListedColormap from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT @@ -18,8 +17,6 @@ _logger = logging.getLogger(__name__) -class FreezeTableView(QTableView) - class DataFrameTableModel(QAbstractTableModel): def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None): super().__init__(parent) From fd30945c87524703a561df217d8ce87407e0df63 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Thu, 26 Sep 2024 14:59:54 +0100 Subject: [PATCH 16/40] allow pinning of columns to that they won't be filtered --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 38 +++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 22fbcef06..a46711af1 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -3,9 +3,11 @@ import logging from PyQt5 import QtWidgets -from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, QObject +from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, \ + QObject, QPoint, pyqtSignal, pyqtSlot from PyQt5.QtGui import QColor import matplotlib.pyplot as plt +from PyQt5.QtWidgets import QMenu, QAction, QHeaderView from matplotlib.colors import ListedColormap from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT @@ -153,9 +155,13 @@ def __init__(self, parent=None, wheel=None): class GraphWindow(QtWidgets.QWidget): + _pinnedColumns = [] + def __init__(self, parent=None, wheel=None): QtWidgets.QWidget.__init__(self, parent=parent) + self.columnPinned = pyqtSignal(int, bool) + self.lineEditFilter = QtWidgets.QLineEdit(self) self.lineEditFilter.setPlaceholderText('Filter columns by name') self.lineEditFilter.textChanged.connect(self.changeFilter) @@ -169,9 +175,15 @@ def __init__(self, parent=None, wheel=None): self.tableView.setSortingEnabled(True) self.tableView.horizontalHeader().setDefaultAlignment(Qt.AlignLeft | Qt.AlignVCenter) self.tableView.horizontalHeader().setSectionsMovable(True) - self.tableView.verticalHeader().hide() + self.tableView.horizontalHeader().setContextMenuPolicy(Qt.CustomContextMenu) + self.tableView.horizontalHeader().customContextMenuRequested.connect( + self.contextMenu) self.tableView.doubleClicked.connect(self.tv_double_clicked) + self.pinAction = QAction('Pin column', self) + self.pinAction.setCheckable(True) + self.pinAction.toggled.connect(self.pinColumn) + vLayout = QtWidgets.QVBoxLayout(self) hLayout = QtWidgets.QHBoxLayout() hLayout.addWidget(self.lineEditFilter) @@ -185,11 +197,31 @@ def __init__(self, parent=None, wheel=None): self.wheel = wheel + def contextMenu(self, pos: QPoint): + idx = self.sender().logicalIndexAt(pos) + action = self.pinAction + action.setData(idx) + action.setChecked(idx in self._pinnedColumns) + menu = QMenu(self) + menu.addAction(action) + menu.exec(self.mapToParent(pos)) + + @pyqtSlot(bool) + @pyqtSlot(bool, int) + def pinColumn(self, pin: bool, idx: int | None = None): + idx = idx if idx is not None else self.sender().data() + if not pin and idx in self._pinnedColumns: + self._pinnedColumns.remove(idx) + if pin and idx not in self._pinnedColumns: + self._pinnedColumns.append(idx) + self.changeFilter(self.lineEditFilter.text()) + def changeFilter(self, string: str): headers = [self.tableModel.headerData(x, Qt.Horizontal, Qt.DisplayRole) for x in range(self.tableModel.columnCount())] for idx, column in enumerate(headers): - self.tableView.setColumnHidden(idx, string.lower() not in column.lower()) + self.tableView.setColumnHidden(idx, string.lower() not in column.lower() + and idx not in self._pinnedColumns) def loadFile(self): fileName, _ = QtWidgets.QFileDialog.getOpenFileName( From 520441c00b207c450c9b2aaf4df2e18181db8bd9 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Thu, 26 Sep 2024 15:43:47 +0100 Subject: [PATCH 17/40] correct location of context menu popup --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index a46711af1..ef563bd84 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -204,7 +204,7 @@ def contextMenu(self, pos: QPoint): action.setChecked(idx in self._pinnedColumns) menu = QMenu(self) menu.addAction(action) - menu.exec(self.mapToParent(pos)) + menu.exec(self.sender().mapToGlobal(pos)) @pyqtSlot(bool) @pyqtSlot(bool, int) From 54b0df0cbbe61d41ece2788e24153796ee1dfd45 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Fri, 27 Sep 2024 10:18:18 +0100 Subject: [PATCH 18/40] happy colors --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index ef563bd84..c46763db4 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -76,12 +76,12 @@ class ColoredDataFrameTableModel(DataFrameTableModel): _cmap: ListedColormap def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, - colorMap: ListedColormap | None = None, alpha: float = 0.5): + colorMap: ListedColormap | None = None, alpha: float = 1): super().__init__(parent=parent, dataFrame=dataFrame) self._alpha = alpha if colorMap is None: - self._cmap = plt.get_cmap('plasma') + self._cmap = plt.get_cmap('spring') self._cmap.set_bad(color='w') else: self._cmap = colorMap From cdbbed51e95438f6043b04239ebb3fe3af932fb5 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Fri, 27 Sep 2024 12:37:43 +0100 Subject: [PATCH 19/40] add signals & slots for ColoredDataFrameTableModel, alpha slider --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 91 ++++++++++++++++++------- 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index c46763db4..9a89d7303 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -5,10 +5,10 @@ from PyQt5 import QtWidgets from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, \ QObject, QPoint, pyqtSignal, pyqtSlot -from PyQt5.QtGui import QColor +from PyQt5.QtGui import QColor, QPalette import matplotlib.pyplot as plt from PyQt5.QtWidgets import QMenu, QAction, QHeaderView -from matplotlib.colors import ListedColormap +from matplotlib.colors import Colormap from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT import pandas as pd @@ -56,7 +56,7 @@ def data(self, index: QModelIndex, role: int = ...) -> QVariant: if not index.isValid(): return QVariant() if role == Qt.DisplayRole: - val = self._dataframe.iloc[index.row()][index.column()] + val = self._dataframe.iloc[index.row(), index.column()] if isinstance(val, np.generic): return val.item() return QVariant(str(val)) @@ -72,24 +72,47 @@ def sort(self, column: int, order: Qt.SortOrder = ...): class ColoredDataFrameTableModel(DataFrameTableModel): + colormapChanged = pyqtSignal(Colormap) + alphaChanged = pyqtSignal(float) _rgba: np.ndarray - _cmap: ListedColormap + _cmap: Colormap + _alpha: int def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, - colorMap: ListedColormap | None = None, alpha: float = 1): + colormap: Colormap | None = None, alpha: int = 255): super().__init__(parent=parent, dataFrame=dataFrame) - self._alpha = alpha - if colorMap is None: - self._cmap = plt.get_cmap('spring') - self._cmap.set_bad(color='w') - else: - self._cmap = colorMap - - self._setRgba() + self.colormapChanged.connect(self._setRgba) self.modelReset.connect(self._setRgba) self.dataChanged.connect(self._setRgba) + if colormap is None: + colormap = plt.get_cmap('spring') + colormap.set_bad(color='w') + self.setColormap(colormap) + self.setAlpha(alpha) + + @pyqtSlot(Colormap) + def setColormap(self, colormap: Colormap): + self._cmap = colormap + self.colormapChanged.emit(colormap) + + def getColormap(self) -> Colormap: + return self._cmap + + colormap = pyqtProperty(Colormap, fget=getColormap, fset=setColormap) + + @pyqtSlot(int) + def setAlpha(self, alpha: int = 255): + _, self._alpha, _ = sorted([0, alpha, 255]) + self.alphaChanged.emit(self._alpha) + self.layoutChanged.emit() + + def getAlpha(self) -> int: + return self._alpha + + alpha = pyqtProperty(int, fget=getAlpha, fset=setAlpha) + def _setRgba(self): df = self._dataframe.copy() if df.empty: @@ -112,15 +135,16 @@ def _setRgba(self): cols = df.select_dtypes(include=['bool']).columns df[cols] = df[cols].astype(float) - # store color values to ndarray - self._rgba = self._cmap(df, self._alpha, True) + # store color values to ndarray & emit signal + self._rgba = self._cmap(df, alpha=None, bytes=True) + self.layoutChanged.emit() def data(self, index, role=...): if not index.isValid(): return QVariant() if role == Qt.BackgroundRole: row = self._dataframe.index[index.row()] - return QColor.fromRgb(*self._rgba[row][index.column()]) + return QColor.fromRgb(*self._rgba[row][index.column()][:3], self._alpha) return super().data(index, role) @@ -162,13 +186,10 @@ def __init__(self, parent=None, wheel=None): self.columnPinned = pyqtSignal(int, bool) - self.lineEditFilter = QtWidgets.QLineEdit(self) - self.lineEditFilter.setPlaceholderText('Filter columns by name') - self.lineEditFilter.textChanged.connect(self.changeFilter) - self.pushButtonLoad = QtWidgets.QPushButton("Select File", self) self.pushButtonLoad.clicked.connect(self.loadFile) + # define table model & view self.tableModel = ColoredDataFrameTableModel(self) self.tableView = QtWidgets.QTableView(self) self.tableView.setModel(self.tableModel) @@ -176,18 +197,42 @@ def __init__(self, parent=None, wheel=None): self.tableView.horizontalHeader().setDefaultAlignment(Qt.AlignLeft | Qt.AlignVCenter) self.tableView.horizontalHeader().setSectionsMovable(True) self.tableView.horizontalHeader().setContextMenuPolicy(Qt.CustomContextMenu) - self.tableView.horizontalHeader().customContextMenuRequested.connect( - self.contextMenu) + self.tableView.horizontalHeader().customContextMenuRequested.connect(self.contextMenu) self.tableView.doubleClicked.connect(self.tv_double_clicked) + # define colors for highlighted cells + p = self.tableView.palette() + p.setColor(QPalette.Highlight, Qt.black) + p.setColor(QPalette.HighlightedText, Qt.white) + self.tableView.setPalette(p) + + # QAction for pinning columns self.pinAction = QAction('Pin column', self) self.pinAction.setCheckable(True) self.pinAction.toggled.connect(self.pinColumn) - vLayout = QtWidgets.QVBoxLayout(self) + # Filter columns by name + self.lineEditFilter = QtWidgets.QLineEdit(self) + self.lineEditFilter.setPlaceholderText('Filter columns') + self.lineEditFilter.textChanged.connect(self.changeFilter) + + # slider for alpha values + self.sliderAlpha = QtWidgets.QSlider(Qt.Horizontal, self) + self.sliderAlpha.setMinimum(0) + self.sliderAlpha.setMaximum(255) + self.sliderAlpha.setValue(self.tableModel.alpha) + self.sliderAlpha.valueChanged.connect(self.tableModel.setAlpha) + + # Horizontal layout hLayout = QtWidgets.QHBoxLayout() hLayout.addWidget(self.lineEditFilter) + hLayout.addWidget(QtWidgets.QLabel('Alpha', self)) + hLayout.addWidget(self.sliderAlpha) + hLayout.addStretch(1) hLayout.addWidget(self.pushButtonLoad) + + # Vertical layout + vLayout = QtWidgets.QVBoxLayout(self) vLayout.addLayout(hLayout) vLayout.addWidget(self.tableView) From 57ce07827098e8acf74e78ba94db9378b3cf814c Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Fri, 27 Sep 2024 12:53:13 +0100 Subject: [PATCH 20/40] add picker for colormap --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 9a89d7303..dccd4368e 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -92,6 +92,12 @@ def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, self.setColormap(colormap) self.setAlpha(alpha) + @pyqtSlot(str) + def setColormapByName(self, colormapName: str): + colormap = plt.get_cmap(colormapName) + colormap.set_bad(color='w') + self.setColormap(colormap) + @pyqtSlot(Colormap) def setColormap(self, colormap: Colormap): self._cmap = colormap @@ -216,6 +222,11 @@ def __init__(self, parent=None, wheel=None): self.lineEditFilter.setPlaceholderText('Filter columns') self.lineEditFilter.textChanged.connect(self.changeFilter) + # colormap picker + self.comboboxColormap = QtWidgets.QComboBox(self) + self.comboboxColormap.addItems(['plasma', 'spring', 'summer', 'autumn', 'winter']) + self.comboboxColormap.currentTextChanged.connect(self.tableModel.setColormapByName) + # slider for alpha values self.sliderAlpha = QtWidgets.QSlider(Qt.Horizontal, self) self.sliderAlpha.setMinimum(0) @@ -226,6 +237,8 @@ def __init__(self, parent=None, wheel=None): # Horizontal layout hLayout = QtWidgets.QHBoxLayout() hLayout.addWidget(self.lineEditFilter) + hLayout.addWidget(QtWidgets.QLabel('Colormap', self)) + hLayout.addWidget(self.comboboxColormap) hLayout.addWidget(QtWidgets.QLabel('Alpha', self)) hLayout.addWidget(self.sliderAlpha) hLayout.addStretch(1) From 61fbb6c25b5bf397a722649d82f1b6d22f4c14d9 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Fri, 27 Sep 2024 12:58:28 +0100 Subject: [PATCH 21/40] Update ViewEphysQC.py --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index dccd4368e..e4bfb2cc4 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -224,7 +224,9 @@ def __init__(self, parent=None, wheel=None): # colormap picker self.comboboxColormap = QtWidgets.QComboBox(self) - self.comboboxColormap.addItems(['plasma', 'spring', 'summer', 'autumn', 'winter']) + colormaps = {self.tableModel.colormap.name, 'plasma', 'spring', 'summer', 'autumn', 'winter'} + self.comboboxColormap.addItems(colormaps) + self.comboboxColormap.setCurrentText(self.tableModel.colormap.name) self.comboboxColormap.currentTextChanged.connect(self.tableModel.setColormapByName) # slider for alpha values From da50ac9c291b506b199716f8da18185813b8e40b Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Sat, 28 Sep 2024 08:37:45 +0100 Subject: [PATCH 22/40] separate normalization from rgba calculation --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index e4bfb2cc4..164c47de1 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -74,6 +74,7 @@ def sort(self, column: int, order: Qt.SortOrder = ...): class ColoredDataFrameTableModel(DataFrameTableModel): colormapChanged = pyqtSignal(Colormap) alphaChanged = pyqtSignal(float) + _normalizedData = pd.DataFrame _rgba: np.ndarray _cmap: Colormap _alpha: int @@ -82,9 +83,9 @@ def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, colormap: Colormap | None = None, alpha: int = 255): super().__init__(parent=parent, dataFrame=dataFrame) + self.modelReset.connect(self._normalizeData) + self.dataChanged.connect(self._normalizeData) self.colormapChanged.connect(self._setRgba) - self.modelReset.connect(self._setRgba) - self.dataChanged.connect(self._setRgba) if colormap is None: colormap = plt.get_cmap('spring') @@ -119,7 +120,7 @@ def getAlpha(self) -> int: alpha = pyqtProperty(int, fget=getAlpha, fset=setAlpha) - def _setRgba(self): + def _normalizeData(self): df = self._dataframe.copy() if df.empty: self._rgba = df @@ -141,8 +142,15 @@ def _setRgba(self): cols = df.select_dtypes(include=['bool']).columns df[cols] = df[cols].astype(float) - # store color values to ndarray & emit signal - self._rgba = self._cmap(df, alpha=None, bytes=True) + # store as property & call _setRgba() + self._normalizedData = df + self._setRgba() + + def _setRgba(self): + if self._normalizedData.empty: + self._rgba = np.ndarray([]) + else: + self._rgba = self._cmap(self._normalizedData, alpha=None, bytes=True) self.layoutChanged.emit() def data(self, index, role=...): From 027c2ea16268020fa3544b5ff7f05b786c3ccf5b Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Sat, 28 Sep 2024 16:43:39 +0100 Subject: [PATCH 23/40] dynamic handling of text color --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 44 ++++++++++++++++--------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 164c47de1..1fbaa3034 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -4,10 +4,10 @@ from PyQt5 import QtWidgets from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, \ - QObject, QPoint, pyqtSignal, pyqtSlot + QObject, QPoint, pyqtSignal, pyqtSlot, QCoreApplication, QSettings from PyQt5.QtGui import QColor, QPalette import matplotlib.pyplot as plt -from PyQt5.QtWidgets import QMenu, QAction, QHeaderView +from PyQt5.QtWidgets import QMenu, QAction from matplotlib.colors import Colormap from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT @@ -74,10 +74,11 @@ def sort(self, column: int, order: Qt.SortOrder = ...): class ColoredDataFrameTableModel(DataFrameTableModel): colormapChanged = pyqtSignal(Colormap) alphaChanged = pyqtSignal(float) - _normalizedData = pd.DataFrame - _rgba: np.ndarray + _normData = pd.DataFrame + _background: np.ndarray _cmap: Colormap _alpha: int + _foreground: np.ndarray def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, colormap: Colormap | None = None, alpha: int = 255): @@ -85,10 +86,10 @@ def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, self.modelReset.connect(self._normalizeData) self.dataChanged.connect(self._normalizeData) - self.colormapChanged.connect(self._setRgba) + self.colormapChanged.connect(self._defineColors) if colormap is None: - colormap = plt.get_cmap('spring') + colormap = plt.get_cmap('plasma') colormap.set_bad(color='w') self.setColormap(colormap) self.setAlpha(alpha) @@ -123,7 +124,7 @@ def getAlpha(self) -> int: def _normalizeData(self): df = self._dataframe.copy() if df.empty: - self._rgba = df + self._background = df return # coerce non-bool / non-numeric values to numeric @@ -143,14 +144,17 @@ def _normalizeData(self): df[cols] = df[cols].astype(float) # store as property & call _setRgba() - self._normalizedData = df - self._setRgba() + self._normData = df + self._defineColors() - def _setRgba(self): - if self._normalizedData.empty: - self._rgba = np.ndarray([]) + def _defineColors(self): + if self._normData.empty: + self._background = np.ndarray([]) + self._foreground = np.ndarray([]) else: - self._rgba = self._cmap(self._normalizedData, alpha=None, bytes=True) + self._background = self._cmap(self._normData, alpha=None, bytes=True)[:, :, :3] + brightness = (self._background * np.array([[[0.21, 0.72, 0.07]]])).sum(axis=2) + self._foreground = 255 - brightness.astype(int) self.layoutChanged.emit() def data(self, index, role=...): @@ -158,7 +162,11 @@ def data(self, index, role=...): return QVariant() if role == Qt.BackgroundRole: row = self._dataframe.index[index.row()] - return QColor.fromRgb(*self._rgba[row][index.column()][:3], self._alpha) + return QColor.fromRgb(*self._background[row][index.column()], self._alpha) + if role == Qt.ForegroundRole: + row = self._dataframe.index[index.row()] + val = self._foreground[row][index.column()] * self._alpha + return QColor('black') if val < 32512 else QColor('white') return super().data(index, role) @@ -198,6 +206,9 @@ class GraphWindow(QtWidgets.QWidget): def __init__(self, parent=None, wheel=None): QtWidgets.QWidget.__init__(self, parent=parent) + # Store layout changes to QSettings + self.settings = QSettings() + self.columnPinned = pyqtSignal(int, bool) self.pushButtonLoad = QtWidgets.QPushButton("Select File", self) @@ -232,7 +243,7 @@ def __init__(self, parent=None, wheel=None): # colormap picker self.comboboxColormap = QtWidgets.QComboBox(self) - colormaps = {self.tableModel.colormap.name, 'plasma', 'spring', 'summer', 'autumn', 'winter'} + colormaps = sorted(list({self.tableModel.colormap.name, 'cividis', 'inferno', 'magma', 'plasma', 'viridis'})) self.comboboxColormap.addItems(colormaps) self.comboboxColormap.setCurrentText(self.tableModel.colormap.name) self.comboboxColormap.currentTextChanged.connect(self.tableModel.setColormapByName) @@ -323,6 +334,9 @@ def tv_double_clicked(self, index: QModelIndex): def viewqc(qc=None, title=None, wheel=None): + QCoreApplication.setOrganizationName('International Brain Laboratory') + QCoreApplication.setOrganizationDomain('internationalbrainlab.org') + QCoreApplication.setApplicationName('QC Viewer') qt.create_app() qcw = GraphWindow(wheel=wheel) qcw.setWindowTitle(title) From 7f6dbb625bcac5fb8c83c6e96cd0fb8d70d8feb9 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Sat, 28 Sep 2024 22:58:07 +0100 Subject: [PATCH 24/40] switch to using pyqtgraph's colormaps --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 62 ++++++++++++------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 1fbaa3034..731d05bb2 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -5,14 +5,13 @@ from PyQt5 import QtWidgets from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, \ QObject, QPoint, pyqtSignal, pyqtSlot, QCoreApplication, QSettings -from PyQt5.QtGui import QColor, QPalette -import matplotlib.pyplot as plt +from PyQt5.QtGui import QColor, QPalette, QShowEvent from PyQt5.QtWidgets import QMenu, QAction -from matplotlib.colors import Colormap from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT import pandas as pd import numpy as np +from pyqtgraph import colormap, ColorMap from ibllib.misc import qt @@ -72,43 +71,36 @@ def sort(self, column: int, order: Qt.SortOrder = ...): class ColoredDataFrameTableModel(DataFrameTableModel): - colormapChanged = pyqtSignal(Colormap) - alphaChanged = pyqtSignal(float) + colormapChanged = pyqtSignal(str) + alphaChanged = pyqtSignal(int) _normData = pd.DataFrame _background: np.ndarray - _cmap: Colormap + _cmap: ColorMap _alpha: int _foreground: np.ndarray def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, - colormap: Colormap | None = None, alpha: int = 255): + colormap: str = 'plasma', alpha: int = 255): super().__init__(parent=parent, dataFrame=dataFrame) - self.modelReset.connect(self._normalizeData) self.dataChanged.connect(self._normalizeData) self.colormapChanged.connect(self._defineColors) - - if colormap is None: - colormap = plt.get_cmap('plasma') - colormap.set_bad(color='w') self.setColormap(colormap) self.setAlpha(alpha) @pyqtSlot(str) - def setColormapByName(self, colormapName: str): - colormap = plt.get_cmap(colormapName) - colormap.set_bad(color='w') - self.setColormap(colormap) + def setColormap(self, name: str): + for source in [None, 'matplotlib', 'colorcet']: + if name in colormap.listMaps(source): + self._cmap = colormap.get(name, source) + self.colormapChanged.emit(name) + return + _logger.warning(f'No such colormap: "{name}"') - @pyqtSlot(Colormap) - def setColormap(self, colormap: Colormap): - self._cmap = colormap - self.colormapChanged.emit(colormap) + def getColormap(self) -> str: + return self._cmap.name - def getColormap(self) -> Colormap: - return self._cmap - - colormap = pyqtProperty(Colormap, fget=getColormap, fset=setColormap) + colormap = pyqtProperty(str, fget=getColormap, fset=setColormap) @pyqtSlot(int) def setAlpha(self, alpha: int = 255): @@ -152,9 +144,10 @@ def _defineColors(self): self._background = np.ndarray([]) self._foreground = np.ndarray([]) else: - self._background = self._cmap(self._normData, alpha=None, bytes=True)[:, :, :3] - brightness = (self._background * np.array([[[0.21, 0.72, 0.07]]])).sum(axis=2) - self._foreground = 255 - brightness.astype(int) + m = np.isfinite(self._normData) # binary mask for finite values + self._background = np.ones((*self._normData.shape, 3), dtype=int) * 255 + self._background[m] = self._cmap.mapToByte(self._normData.values[m])[:, :3] + self._foreground = 255 - (self._background * np.array([[[0.21, 0.72, 0.07]]])).sum(axis=2).astype(int) self.layoutChanged.emit() def data(self, index, role=...): @@ -162,7 +155,8 @@ def data(self, index, role=...): return QVariant() if role == Qt.BackgroundRole: row = self._dataframe.index[index.row()] - return QColor.fromRgb(*self._background[row][index.column()], self._alpha) + val = self._background[row][index.column()] + return QColor.fromRgb(*val, self._alpha) if role == Qt.ForegroundRole: row = self._dataframe.index[index.row()] val = self._foreground[row][index.column()] * self._alpha @@ -243,10 +237,10 @@ def __init__(self, parent=None, wheel=None): # colormap picker self.comboboxColormap = QtWidgets.QComboBox(self) - colormaps = sorted(list({self.tableModel.colormap.name, 'cividis', 'inferno', 'magma', 'plasma', 'viridis'})) - self.comboboxColormap.addItems(colormaps) - self.comboboxColormap.setCurrentText(self.tableModel.colormap.name) - self.comboboxColormap.currentTextChanged.connect(self.tableModel.setColormapByName) + colormaps = {self.tableModel.colormap, 'inferno', 'magma', 'plasma'} + self.comboboxColormap.addItems(sorted(list(colormaps))) + self.comboboxColormap.setCurrentText(self.tableModel.colormap) + self.comboboxColormap.currentTextChanged.connect(self.tableModel.setColormap) # slider for alpha values self.sliderAlpha = QtWidgets.QSlider(Qt.Horizontal, self) @@ -276,6 +270,10 @@ def __init__(self, parent=None, wheel=None): self.wheel = wheel + def showEvent(self, a0: QShowEvent) -> None: + super().showEvent(a0) + self.activateWindow() + def contextMenu(self, pos: QPoint): idx = self.sender().logicalIndexAt(pos) action = self.pinAction From 5608ec1c8bc3ceeace16b262a4949e4665a889ee Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Sun, 29 Sep 2024 19:34:32 +0100 Subject: [PATCH 25/40] filter by tokens --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 49 ++++++++++++++++--------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 731d05bb2..34672530b 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -73,11 +73,11 @@ def sort(self, column: int, order: Qt.SortOrder = ...): class ColoredDataFrameTableModel(DataFrameTableModel): colormapChanged = pyqtSignal(str) alphaChanged = pyqtSignal(int) - _normData = pd.DataFrame + _normData = pd.DataFrame() _background: np.ndarray + _foreground: np.ndarray _cmap: ColorMap _alpha: int - _foreground: np.ndarray def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, colormap: str = 'plasma', alpha: int = 255): @@ -115,9 +115,6 @@ def getAlpha(self) -> int: def _normalizeData(self): df = self._dataframe.copy() - if df.empty: - self._background = df - return # coerce non-bool / non-numeric values to numeric cols = df.select_dtypes(exclude=['bool', 'number']).columns @@ -135,14 +132,23 @@ def _normalizeData(self): cols = df.select_dtypes(include=['bool']).columns df[cols] = df[cols].astype(float) - # store as property & call _setRgba() + # store as property & call _defineColors() self._normData = df self._defineColors() def _defineColors(self): + """ + Define the background and foreground colors according to the table's data. + + The background color is set to the colormap-mapped values of the normalized + data, and the foreground color is set to the inverse of the background's + approximated luminosity. + + The `layoutChanged` signal is emitted after the colors are defined. + """ if self._normData.empty: - self._background = np.ndarray([]) - self._foreground = np.ndarray([]) + self._background = np.zeros((0, 0, 3), dtype=int) + self._foreground = np.zeros((0, 0), dtype=int) else: m = np.isfinite(self._normData) # binary mask for finite values self._background = np.ones((*self._normData.shape, 3), dtype=int) * 255 @@ -153,14 +159,15 @@ def _defineColors(self): def data(self, index, role=...): if not index.isValid(): return QVariant() - if role == Qt.BackgroundRole: - row = self._dataframe.index[index.row()] - val = self._background[row][index.column()] - return QColor.fromRgb(*val, self._alpha) - if role == Qt.ForegroundRole: + if role in (Qt.BackgroundRole, Qt.ForegroundRole): row = self._dataframe.index[index.row()] - val = self._foreground[row][index.column()] * self._alpha - return QColor('black') if val < 32512 else QColor('white') + col = index.column() + if role == Qt.BackgroundRole: + val = self._background[row][col] + return QColor.fromRgb(*val, self._alpha) + if role == Qt.ForegroundRole: + val = self._foreground[row][col] + return QColor('black' if (val * self._alpha) < 32512 else 'white') return super().data(index, role) @@ -234,6 +241,7 @@ def __init__(self, parent=None, wheel=None): self.lineEditFilter = QtWidgets.QLineEdit(self) self.lineEditFilter.setPlaceholderText('Filter columns') self.lineEditFilter.textChanged.connect(self.changeFilter) + self.lineEditFilter.setMinimumWidth(200) # colormap picker self.comboboxColormap = QtWidgets.QComboBox(self) @@ -252,6 +260,7 @@ def __init__(self, parent=None, wheel=None): # Horizontal layout hLayout = QtWidgets.QHBoxLayout() hLayout.addWidget(self.lineEditFilter) + hLayout.addSpacing(50) hLayout.addWidget(QtWidgets.QLabel('Colormap', self)) hLayout.addWidget(self.comboboxColormap) hLayout.addWidget(QtWidgets.QLabel('Alpha', self)) @@ -264,6 +273,8 @@ def __init__(self, parent=None, wheel=None): vLayout.addLayout(hLayout) vLayout.addWidget(self.tableView) + self.setMinimumSize(500, 400) + self.wplot = PlotWindow(wheel=wheel) self.wplot.show() self.tableModel.dataChanged.connect(self.wplot.canvas.draw) @@ -294,11 +305,13 @@ def pinColumn(self, pin: bool, idx: int | None = None): self.changeFilter(self.lineEditFilter.text()) def changeFilter(self, string: str): - headers = [self.tableModel.headerData(x, Qt.Horizontal, Qt.DisplayRole) + headers = [self.tableModel.headerData(x, Qt.Horizontal, Qt.DisplayRole).lower() for x in range(self.tableModel.columnCount())] + tokens = [y.lower() for y in (x.strip() for x in string.split(',')) if len(y)] + showAll = len(tokens) == 0 for idx, column in enumerate(headers): - self.tableView.setColumnHidden(idx, string.lower() not in column.lower() - and idx not in self._pinnedColumns) + show = showAll or any((t in column for t in tokens)) or idx in self._pinnedColumns + self.tableView.setColumnHidden(idx, not show) def loadFile(self): fileName, _ = QtWidgets.QFileDialog.getOpenFileName( From 02e9a2cd268ca613d74473baac26b69ea72046cb Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Tue, 1 Oct 2024 11:23:00 +0100 Subject: [PATCH 26/40] move models to iblqt --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 164 +----------------------- requirements.txt | 1 + 2 files changed, 6 insertions(+), 159 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 34672530b..420c5ea9a 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -3,174 +3,20 @@ import logging from PyQt5 import QtWidgets -from PyQt5.QtCore import pyqtProperty, Qt, QVariant, QAbstractTableModel, QModelIndex, \ - QObject, QPoint, pyqtSignal, pyqtSlot, QCoreApplication, QSettings -from PyQt5.QtGui import QColor, QPalette, QShowEvent +from PyQt5.QtCore import Qt, QModelIndex, QPoint, pyqtSignal, pyqtSlot, QCoreApplication, QSettings +from PyQt5.QtGui import QPalette, QShowEvent from PyQt5.QtWidgets import QMenu, QAction +from iblqt.core import ColoredDataFrameTableModel from matplotlib.figure import Figure from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg, NavigationToolbar2QT import pandas as pd import numpy as np -from pyqtgraph import colormap, ColorMap from ibllib.misc import qt _logger = logging.getLogger(__name__) -class DataFrameTableModel(QAbstractTableModel): - def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None): - super().__init__(parent) - self._dataframe = pd.DataFrame() if dataFrame is None else dataFrame - - def setDataFrame(self, dataFrame: pd.DataFrame): - self.beginResetModel() - self._dataframe = dataFrame.copy() - self.endResetModel() - - def dataFrame(self) -> pd.DataFrame: - return self._dataframe - - dataFrame = pyqtProperty(pd.DataFrame, fget=dataFrame, fset=setDataFrame) - - def headerData(self, section: int, orientation: Qt.Orientation, role: int = ...): - if role in (Qt.DisplayRole, Qt.ToolTipRole): - if orientation == Qt.Horizontal: - return self._dataframe.columns[section] - else: - return str(self._dataframe.index[section]) - return QVariant() - - def rowCount(self, parent: QModelIndex = ...): - if isinstance(parent, QModelIndex) and parent.isValid(): - return 0 - return len(self._dataframe.index) - - def columnCount(self, parent: QModelIndex = ...): - if isinstance(parent, QModelIndex) and parent.isValid(): - return 0 - return self._dataframe.columns.size - - def data(self, index: QModelIndex, role: int = ...) -> QVariant: - if not index.isValid(): - return QVariant() - if role == Qt.DisplayRole: - val = self._dataframe.iloc[index.row(), index.column()] - if isinstance(val, np.generic): - return val.item() - return QVariant(str(val)) - return QVariant() - - def sort(self, column: int, order: Qt.SortOrder = ...): - if self.columnCount() == 0: - return - column = self._dataframe.columns[column] - self.layoutAboutToBeChanged.emit() - self._dataframe.sort_values(by=column, ascending=not order, inplace=True) - self.layoutChanged.emit() - - -class ColoredDataFrameTableModel(DataFrameTableModel): - colormapChanged = pyqtSignal(str) - alphaChanged = pyqtSignal(int) - _normData = pd.DataFrame() - _background: np.ndarray - _foreground: np.ndarray - _cmap: ColorMap - _alpha: int - - def __init__(self, parent: QObject = ..., dataFrame: pd.DataFrame | None = None, - colormap: str = 'plasma', alpha: int = 255): - super().__init__(parent=parent, dataFrame=dataFrame) - self.modelReset.connect(self._normalizeData) - self.dataChanged.connect(self._normalizeData) - self.colormapChanged.connect(self._defineColors) - self.setColormap(colormap) - self.setAlpha(alpha) - - @pyqtSlot(str) - def setColormap(self, name: str): - for source in [None, 'matplotlib', 'colorcet']: - if name in colormap.listMaps(source): - self._cmap = colormap.get(name, source) - self.colormapChanged.emit(name) - return - _logger.warning(f'No such colormap: "{name}"') - - def getColormap(self) -> str: - return self._cmap.name - - colormap = pyqtProperty(str, fget=getColormap, fset=setColormap) - - @pyqtSlot(int) - def setAlpha(self, alpha: int = 255): - _, self._alpha, _ = sorted([0, alpha, 255]) - self.alphaChanged.emit(self._alpha) - self.layoutChanged.emit() - - def getAlpha(self) -> int: - return self._alpha - - alpha = pyqtProperty(int, fget=getAlpha, fset=setAlpha) - - def _normalizeData(self): - df = self._dataframe.copy() - - # coerce non-bool / non-numeric values to numeric - cols = df.select_dtypes(exclude=['bool', 'number']).columns - df[cols] = df[cols].apply(pd.to_numeric, errors='coerce') - - # normalize numeric values, avoiding inf values and division by zero - cols = df.select_dtypes(include=['number']).columns - df[cols].replace([np.inf, -np.inf], np.nan) - m = df[cols].nunique() <= 1 # boolean mask for columns with only 1 unique value - df[cols[m]] = df[cols[m]].where(df[cols[m]].isna(), other=0.0) - cols = cols[~m] - df[cols] = (df[cols] - df[cols].min()) / (df[cols].max() - df[cols].min()) - - # convert boolean values - cols = df.select_dtypes(include=['bool']).columns - df[cols] = df[cols].astype(float) - - # store as property & call _defineColors() - self._normData = df - self._defineColors() - - def _defineColors(self): - """ - Define the background and foreground colors according to the table's data. - - The background color is set to the colormap-mapped values of the normalized - data, and the foreground color is set to the inverse of the background's - approximated luminosity. - - The `layoutChanged` signal is emitted after the colors are defined. - """ - if self._normData.empty: - self._background = np.zeros((0, 0, 3), dtype=int) - self._foreground = np.zeros((0, 0), dtype=int) - else: - m = np.isfinite(self._normData) # binary mask for finite values - self._background = np.ones((*self._normData.shape, 3), dtype=int) * 255 - self._background[m] = self._cmap.mapToByte(self._normData.values[m])[:, :3] - self._foreground = 255 - (self._background * np.array([[[0.21, 0.72, 0.07]]])).sum(axis=2).astype(int) - self.layoutChanged.emit() - - def data(self, index, role=...): - if not index.isValid(): - return QVariant() - if role in (Qt.BackgroundRole, Qt.ForegroundRole): - row = self._dataframe.index[index.row()] - col = index.column() - if role == Qt.BackgroundRole: - val = self._background[row][col] - return QColor.fromRgb(*val, self._alpha) - if role == Qt.ForegroundRole: - val = self._foreground[row][col] - return QColor('black' if (val * self._alpha) < 32512 else 'white') - return super().data(index, role) - - class PlotCanvas(FigureCanvasQTAgg): def __init__(self, parent=None, width=5, height=4, dpi=100, wheel=None): fig = Figure(figsize=(width, height), dpi=dpi) @@ -245,7 +91,7 @@ def __init__(self, parent=None, wheel=None): # colormap picker self.comboboxColormap = QtWidgets.QComboBox(self) - colormaps = {self.tableModel.colormap, 'inferno', 'magma', 'plasma'} + colormaps = {self.tableModel.colormap, 'inferno', 'magma', 'plasma', 'summer'} self.comboboxColormap.addItems(sorted(list(colormaps))) self.comboboxColormap.setCurrentText(self.tableModel.colormap) self.comboboxColormap.currentTextChanged.connect(self.tableModel.setColormap) @@ -305,7 +151,7 @@ def pinColumn(self, pin: bool, idx: int | None = None): self.changeFilter(self.lineEditFilter.text()) def changeFilter(self, string: str): - headers = [self.tableModel.headerData(x, Qt.Horizontal, Qt.DisplayRole).lower() + headers = [self.tableModel.headerData(x, Qt.Horizontal, Qt.DisplayRole).value().lower() for x in range(self.tableModel.columnCount())] tokens = [y.lower() for y in (x.strip() for x in string.split(',')) if len(y)] showAll = len(tokens) == 0 diff --git a/requirements.txt b/requirements.txt index 7524c22f3..005e43b5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,7 @@ tqdm>=4.32.1 iblatlas>=0.5.3 ibl-neuropixel>=1.0.1 iblutil>=1.11.0 +iblqt>=0.1.0 mtscomp>=1.0.1 ONE-api~=2.9.rc0 phylib>=2.6.0 From af0abe3080a9ce1990ed8f0129ea4aa55687e7cf Mon Sep 17 00:00:00 2001 From: owinter Date: Wed, 2 Oct 2024 18:41:54 +0100 Subject: [PATCH 27/40] fix stim freeze indexing issue in ephys_fpga extraction --- ibllib/io/extractors/ephys_fpga.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ibllib/io/extractors/ephys_fpga.py b/ibllib/io/extractors/ephys_fpga.py index eb6c5141d..e7ae93220 100644 --- a/ibllib/io/extractors/ephys_fpga.py +++ b/ibllib/io/extractors/ephys_fpga.py @@ -936,6 +936,7 @@ def build_trials(self, sync, chmap, display=False, **kwargs): t_trial_start = np.sort(np.r_[fpga_events['intervals_0'][:, 0], missing_bpod]) else: t_trial_start = fpga_events['intervals_0'] + t_trial_start = t_trial_start[ifpga] out = alfio.AlfBunch() # Add the Bpod trial events, converting the timestamp fields to FPGA time. @@ -960,9 +961,9 @@ def build_trials(self, sync, chmap, display=False, **kwargs): # f2ttl times are unreliable owing to calibration and Bonsai sync square update issues. # Take the first event after the FPGA aligned stimulus trigger time. - fpga_trials['stimOn_times'][ibpod] = _assign_events_to_trial( + fpga_trials['stimOn_times'] = _assign_events_to_trial( out['stimOnTrigger_times'], f2ttl_t, take='first', t_trial_end=out['stimOffTrigger_times']) - fpga_trials['stimOff_times'][ibpod] = _assign_events_to_trial( + fpga_trials['stimOff_times'] = _assign_events_to_trial( out['stimOffTrigger_times'], f2ttl_t, take='first', t_trial_end=out['intervals'][:, 1]) # For stim freeze we take the last event before the stim off trigger time. # To avoid assigning early events (e.g. for sessions where there are few flips due to @@ -981,13 +982,12 @@ def build_trials(self, sync, chmap, display=False, **kwargs): # take last event after freeze/stim on trigger, before stim off trigger stim_freeze = _assign_events_to_trial(lims, f2ttl_t, take='last', t_trial_end=out['stimOffTrigger_times']) fpga_trials['stimFreeze_times'][go_trials] = stim_freeze[go_trials] - # Feedback times are valve open on correct trials and error tone in on incorrect trials fpga_trials['feedback_times'] = np.copy(fpga_trials['valveOpen_times']) ind_err = np.isnan(fpga_trials['valveOpen_times']) fpga_trials['feedback_times'][ind_err] = fpga_trials['errorCue_times'][ind_err] - out.update({k: fpga_trials[k][ifpga] for k in fpga_trials.keys()}) + out.update({k: fpga_trials[k] for k in fpga_trials.keys()}) if display: # pragma: no cover width = 0.5 From 4a3627fc3e22ab69fcdd0402a41d5b04009183e7 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Thu, 3 Oct 2024 10:21:12 +0100 Subject: [PATCH 28/40] change ITI constants from 1s to 500ms --- ibllib/qc/task_metrics.py | 2 +- ibllib/tests/qc/test_task_metrics.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ibllib/qc/task_metrics.py b/ibllib/qc/task_metrics.py index c510d271c..2a7a4934f 100644 --- a/ibllib/qc/task_metrics.py +++ b/ibllib/qc/task_metrics.py @@ -679,7 +679,7 @@ def check_iti_delays(data, subtract_pauses=False, **_): An array of boolean values, 1 per trial, where True means trial passes QC threshold. """ # Initialize array the length of completed trials - ITI = 1. + ITI = .5 metric = np.full(data['intervals'].shape[0], np.nan) passed = metric.copy() pauses = (data['pause_duration'] if subtract_pauses else np.zeros_like(metric))[:-1] diff --git a/ibllib/tests/qc/test_task_metrics.py b/ibllib/tests/qc/test_task_metrics.py index a0433332a..6e4a206b4 100644 --- a/ibllib/tests/qc/test_task_metrics.py +++ b/ibllib/tests/qc/test_task_metrics.py @@ -159,7 +159,7 @@ def load_fake_bpod_data(n=5): # add a 5s pause on 3rd trial pauses[2] = 5. quiescence_length = 0.2 + np.random.standard_exponential(size=(n,)) - iti_length = 1 # inter-trial interval + iti_length = .5 # inter-trial interval # trial lengths include quiescence period, a couple small trigger delays and iti trial_lengths = quiescence_length + resp_feeback_delay + (trigg_delay * 4) + iti_length # add on 60s for nogos + feedback time (1 or 2s) + ~0.5s for other responses From 04ef5bc1a5101e17164e81ea2d1d31be0c5e7c84 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Fri, 4 Oct 2024 11:11:33 +0100 Subject: [PATCH 29/40] sort dataframe, store UI settings --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 100 ++++++++++++++++++------ ruff.toml | 4 + 2 files changed, 78 insertions(+), 26 deletions(-) create mode 100644 ruff.toml diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 420c5ea9a..d4178260b 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -3,7 +3,16 @@ import logging from PyQt5 import QtWidgets -from PyQt5.QtCore import Qt, QModelIndex, QPoint, pyqtSignal, pyqtSlot, QCoreApplication, QSettings +from PyQt5.QtCore import ( + Qt, + QModelIndex, + pyqtSignal, + pyqtSlot, + QCoreApplication, + QSettings, + QSize, + QPoint, +) from PyQt5.QtGui import QPalette, QShowEvent from PyQt5.QtWidgets import QMenu, QAction from iblqt.core import ColoredDataFrameTableModel @@ -24,14 +33,10 @@ def __init__(self, parent=None, width=5, height=4, dpi=100, wheel=None): FigureCanvasQTAgg.__init__(self, fig) self.setParent(parent) - FigureCanvasQTAgg.setSizePolicy( - self, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding - ) + FigureCanvasQTAgg.setSizePolicy(self, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding) FigureCanvasQTAgg.updateGeometry(self) if wheel: - self.ax, self.ax2 = fig.subplots( - 2, 1, gridspec_kw={"height_ratios": [2, 1]}, sharex=True - ) + self.ax, self.ax2 = fig.subplots(2, 1, gridspec_kw={'height_ratios': [2, 1]}, sharex=True) else: self.ax = fig.add_subplot(111) self.draw() @@ -53,12 +58,10 @@ class GraphWindow(QtWidgets.QWidget): def __init__(self, parent=None, wheel=None): QtWidgets.QWidget.__init__(self, parent=parent) - # Store layout changes to QSettings - self.settings = QSettings() - self.columnPinned = pyqtSignal(int, bool) - self.pushButtonLoad = QtWidgets.QPushButton("Select File", self) + # load button + self.pushButtonLoad = QtWidgets.QPushButton('Select File', self) self.pushButtonLoad.clicked.connect(self.loadFile) # define table model & view @@ -70,6 +73,7 @@ def __init__(self, parent=None, wheel=None): self.tableView.horizontalHeader().setSectionsMovable(True) self.tableView.horizontalHeader().setContextMenuPolicy(Qt.CustomContextMenu) self.tableView.horizontalHeader().customContextMenuRequested.connect(self.contextMenu) + self.tableView.verticalHeader().hide() self.tableView.doubleClicked.connect(self.tv_double_clicked) # define colors for highlighted cells @@ -98,6 +102,7 @@ def __init__(self, parent=None, wheel=None): # slider for alpha values self.sliderAlpha = QtWidgets.QSlider(Qt.Horizontal, self) + self.sliderAlpha.setMaximumWidth(100) self.sliderAlpha.setMinimum(0) self.sliderAlpha.setMaximum(255) self.sliderAlpha.setValue(self.tableModel.alpha) @@ -111,7 +116,7 @@ def __init__(self, parent=None, wheel=None): hLayout.addWidget(self.comboboxColormap) hLayout.addWidget(QtWidgets.QLabel('Alpha', self)) hLayout.addWidget(self.sliderAlpha) - hLayout.addStretch(1) + hLayout.addSpacing(50) hLayout.addWidget(self.pushButtonLoad) # Vertical layout @@ -119,7 +124,13 @@ def __init__(self, parent=None, wheel=None): vLayout.addLayout(hLayout) vLayout.addWidget(self.tableView) - self.setMinimumSize(500, 400) + # Recover layout from QSettings + self.settings = QSettings() + self.settings.beginGroup('MainWindow') + self.resize(self.settings.value('size', QSize(800, 600), QSize)) + self.comboboxColormap.setCurrentText(self.settings.value('colormap', 'plasma', str)) + self.sliderAlpha.setValue(self.settings.value('alpha', 255, int)) + self.settings.endGroup() self.wplot = PlotWindow(wheel=wheel) self.wplot.show() @@ -127,6 +138,14 @@ def __init__(self, parent=None, wheel=None): self.wheel = wheel + def closeEvent(self, _) -> bool: + self.settings.beginGroup('MainWindow') + self.settings.setValue('size', self.size()) + self.settings.setValue('colormap', self.tableModel.colormap) + self.settings.setValue('alpha', self.tableModel.alpha) + self.settings.endGroup() + self.wplot.close() + def showEvent(self, a0: QShowEvent) -> None: super().showEvent(a0) self.activateWindow() @@ -151,8 +170,10 @@ def pinColumn(self, pin: bool, idx: int | None = None): self.changeFilter(self.lineEditFilter.text()) def changeFilter(self, string: str): - headers = [self.tableModel.headerData(x, Qt.Horizontal, Qt.DisplayRole).value().lower() - for x in range(self.tableModel.columnCount())] + headers = [ + self.tableModel.headerData(x, Qt.Horizontal, Qt.DisplayRole).value().lower() + for x in range(self.tableModel.columnCount()) + ] tokens = [y.lower() for y in (x.strip() for x in string.split(',')) if len(y)] showAll = len(tokens) == 0 for idx, column in enumerate(headers): @@ -160,27 +181,53 @@ def changeFilter(self, string: str): self.tableView.setColumnHidden(idx, not show) def loadFile(self): - fileName, _ = QtWidgets.QFileDialog.getOpenFileName( - self, "Open File", "", "CSV Files (*.csv)" - ) + fileName, _ = QtWidgets.QFileDialog.getOpenFileName(self, 'Open File', '', 'CSV Files (*.csv)') if len(fileName) == 0: return df = pd.read_csv(fileName) self.updateDataframe(df) - def updateDataframe(self, dataFrame: pd.DataFrame): - self.tableModel.setDataFrame(dataFrame) + def updateDataframe(self, df: pd.DataFrame): + # clear pinned columns + self._pinnedColumns = [] + + # try to identify and sort columns containing timestamps + col_names = df.columns + df_interp = df.replace([-np.inf, np.inf], np.nan) + df_interp = df_interp.interpolate(limit_direction='both') + cols_mono = col_names[[df_interp[c].is_monotonic_increasing for c in col_names]] + cols_mono = [c for c in cols_mono if df[c].nunique() > 1] + cols_mono = df_interp[cols_mono].mean().sort_values().keys() + for idx, col_name in enumerate(cols_mono): + df.insert(idx, col_name, df.pop(col_name)) + + # columns containing boolean values are sorted to the end + cols_bool = list(df.select_dtypes('bool').columns) + cols_pass = [cols_bool.pop(i) for i, c in enumerate(cols_bool) if 'pass' in c] + cols_bool += cols_pass + for col_name in cols_bool: + df = df.join(df.pop(col_name)) + + # trial_no should always be the first column + if 'trial_no' in col_names: + df.insert(0, 'trial_no', df.pop('trial_no')) + + # define columns that should be pinned by default + for col in ['trial_no']: + self._pinnedColumns.append(df.columns.get_loc(col)) + + self.tableModel.setDataFrame(df) def tv_double_clicked(self, index: QModelIndex): data = self.tableModel.dataFrame.iloc[index.row()] - t0 = data["intervals_0"] - t1 = data["intervals_1"] + t0 = data['intervals_0'] + t1 = data['intervals_1'] dt = t1 - t0 if self.wheel: - idx = np.searchsorted(self.wheel["re_ts"], np.array([t0 - dt / 10, t1 + dt / 10])) - period = self.wheel["re_pos"][idx[0] : idx[1]] + idx = np.searchsorted(self.wheel['re_ts'], np.array([t0 - dt / 10, t1 + dt / 10])) + period = self.wheel['re_pos'][idx[0] : idx[1]] if period.size == 0: - _logger.warning("No wheel data during trial #%i", index.row()) + _logger.warning('No wheel data during trial #%i', index.row()) else: min_val, max_val = np.min(period), np.max(period) self.wplot.canvas.ax2.set_ylim(min_val - 1, max_val + 1) @@ -191,10 +238,11 @@ def tv_double_clicked(self, index: QModelIndex): def viewqc(qc=None, title=None, wheel=None): + app = qt.create_app() + app.setStyle('Fusion') QCoreApplication.setOrganizationName('International Brain Laboratory') QCoreApplication.setOrganizationDomain('internationalbrainlab.org') QCoreApplication.setApplicationName('QC Viewer') - qt.create_app() qcw = GraphWindow(wheel=wheel) qcw.setWindowTitle(title) if qc is not None: diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..253516e9f --- /dev/null +++ b/ruff.toml @@ -0,0 +1,4 @@ +line-length = 130 + +[format] +quote-style = "single" From 72aec0ffbbada6302d44f74d43c9592e1735a843 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Fri, 4 Oct 2024 11:12:37 +0100 Subject: [PATCH 30/40] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 005e43b5e..bf2ec84a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ tqdm>=4.32.1 iblatlas>=0.5.3 ibl-neuropixel>=1.0.1 iblutil>=1.11.0 -iblqt>=0.1.0 +iblqt>=0.1.2 mtscomp>=1.0.1 ONE-api~=2.9.rc0 phylib>=2.6.0 From c6d480b66986177f94561bb4f74a20c8374f3967 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Fri, 4 Oct 2024 18:43:18 +0100 Subject: [PATCH 31/40] add passing status of individual tests --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 12 ++++++++---- ibllib/qc/task_qc_viewer/task_qc.py | 7 ++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index d4178260b..8a7679aac 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -192,8 +192,8 @@ def updateDataframe(self, df: pd.DataFrame): self._pinnedColumns = [] # try to identify and sort columns containing timestamps - col_names = df.columns - df_interp = df.replace([-np.inf, np.inf], np.nan) + col_names = df.select_dtypes('number').columns + df_interp = df[col_names].replace([-np.inf, np.inf], np.nan) df_interp = df_interp.interpolate(limit_direction='both') cols_mono = col_names[[df_interp[c].is_monotonic_increasing for c in col_names]] cols_mono = [c for c in cols_mono if df[c].nunique() > 1] @@ -202,8 +202,12 @@ def updateDataframe(self, df: pd.DataFrame): df.insert(idx, col_name, df.pop(col_name)) # columns containing boolean values are sorted to the end - cols_bool = list(df.select_dtypes('bool').columns) - cols_pass = [cols_bool.pop(i) for i, c in enumerate(cols_bool) if 'pass' in c] + # of those, columns containing 'pass' in their title will be sorted by number of False values + col_names = df.columns + cols_bool = list(df.select_dtypes(['bool', 'boolean']).columns) + cols_pass = [c for c in cols_bool if 'pass' in c] + cols_bool = [c for c in cols_bool if c not in cols_pass] # I know. Friday evening, brain is fried ... sorry. + cols_pass = list((~df[cols_pass]).sum().sort_values().keys()) cols_bool += cols_pass for col_name in cols_bool: df = df.join(df.pop(col_name)) diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py index b8fc1749f..8d0c034c9 100644 --- a/ibllib/qc/task_qc_viewer/task_qc.py +++ b/ibllib/qc/task_qc_viewer/task_qc.py @@ -289,7 +289,12 @@ def show_session_task_qc(qc_or_session=None, bpod_only=False, local=False, one=N # Update table and callbacks n_trials = qc.frame.shape[0] df_trials = pd.DataFrame({k: v for k, v in task_qc.extractor.data.items() if v.size == n_trials}) - w.updateDataframe(df_trials.merge(qc.frame, left_index=True, right_index=True)) + df_pass = pd.DataFrame({k: v for k, v in qc.qc.passed.items() if isinstance(v, np.ndarray) and v.size == n_trials}) + df_pass.drop('_task_passed_trial_checks', axis=1, errors='ignore', inplace=True) + df_pass.rename(columns=lambda x: x.replace('_task', 'passed'), inplace=True) + df = df_trials.merge(qc.frame, left_index=True, right_index=True) + df = df.merge(df_pass.astype('boolean'), left_index=True, right_index=True) + w.updateDataframe(df) qt.run_app() return qc From 4c58d00345ded61c2ca94ae2aa65f6aefac7c9ad Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 9 Oct 2024 14:12:47 +0100 Subject: [PATCH 32/40] require iblqt >= 0.2.0 --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 8a7679aac..4e6684313 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -171,7 +171,7 @@ def pinColumn(self, pin: bool, idx: int | None = None): def changeFilter(self, string: str): headers = [ - self.tableModel.headerData(x, Qt.Horizontal, Qt.DisplayRole).value().lower() + self.tableModel.headerData(x, Qt.Horizontal, Qt.DisplayRole).lower() for x in range(self.tableModel.columnCount()) ] tokens = [y.lower() for y in (x.strip() for x in string.split(',')) if len(y)] diff --git a/requirements.txt b/requirements.txt index bf2ec84a1..b0f9fd71a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ tqdm>=4.32.1 iblatlas>=0.5.3 ibl-neuropixel>=1.0.1 iblutil>=1.11.0 -iblqt>=0.1.2 +iblqt>=0.2.0 mtscomp>=1.0.1 ONE-api~=2.9.rc0 phylib>=2.6.0 From af96df2131bb870735fe5408c82588385bcae7dc Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Tue, 15 Oct 2024 14:53:02 +0300 Subject: [PATCH 33/40] Resolves #853 --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 2 +- ibllib/qc/task_qc_viewer/task_qc.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 4e6684313..cae7431c2 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -229,7 +229,7 @@ def tv_double_clicked(self, index: QModelIndex): dt = t1 - t0 if self.wheel: idx = np.searchsorted(self.wheel['re_ts'], np.array([t0 - dt / 10, t1 + dt / 10])) - period = self.wheel['re_pos'][idx[0] : idx[1]] + period = self.wheel['re_pos'][idx[0]:idx[1]] if period.size == 0: _logger.warning('No wheel data during trial #%i', index.row()) else: diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py index 8d0c034c9..a49c703eb 100644 --- a/ibllib/qc/task_qc_viewer/task_qc.py +++ b/ibllib/qc/task_qc_viewer/task_qc.py @@ -288,7 +288,10 @@ def show_session_task_qc(qc_or_session=None, bpod_only=False, local=False, one=N # Update table and callbacks n_trials = qc.frame.shape[0] - df_trials = pd.DataFrame({k: v for k, v in task_qc.extractor.data.items() if v.size == n_trials}) + df_trials = pd.DataFrame({ + k: v for k, v in task_qc.extractor.data.items() + if v.size == n_trials and not k.startswith('wheel') + }) df_pass = pd.DataFrame({k: v for k, v in qc.qc.passed.items() if isinstance(v, np.ndarray) and v.size == n_trials}) df_pass.drop('_task_passed_trial_checks', axis=1, errors='ignore', inplace=True) df_pass.rename(columns=lambda x: x.replace('_task', 'passed'), inplace=True) From bbf915ddd937c506c6ac0e597fcebe5ff72891b9 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 16 Oct 2024 12:46:19 +0100 Subject: [PATCH 34/40] Revert "Resolves #853" This reverts commit af96df2131bb870735fe5408c82588385bcae7dc. --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 2 +- ibllib/qc/task_qc_viewer/task_qc.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index cae7431c2..4e6684313 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -229,7 +229,7 @@ def tv_double_clicked(self, index: QModelIndex): dt = t1 - t0 if self.wheel: idx = np.searchsorted(self.wheel['re_ts'], np.array([t0 - dt / 10, t1 + dt / 10])) - period = self.wheel['re_pos'][idx[0]:idx[1]] + period = self.wheel['re_pos'][idx[0] : idx[1]] if period.size == 0: _logger.warning('No wheel data during trial #%i', index.row()) else: diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py index a49c703eb..8d0c034c9 100644 --- a/ibllib/qc/task_qc_viewer/task_qc.py +++ b/ibllib/qc/task_qc_viewer/task_qc.py @@ -288,10 +288,7 @@ def show_session_task_qc(qc_or_session=None, bpod_only=False, local=False, one=N # Update table and callbacks n_trials = qc.frame.shape[0] - df_trials = pd.DataFrame({ - k: v for k, v in task_qc.extractor.data.items() - if v.size == n_trials and not k.startswith('wheel') - }) + df_trials = pd.DataFrame({k: v for k, v in task_qc.extractor.data.items() if v.size == n_trials}) df_pass = pd.DataFrame({k: v for k, v in qc.qc.passed.items() if isinstance(v, np.ndarray) and v.size == n_trials}) df_pass.drop('_task_passed_trial_checks', axis=1, errors='ignore', inplace=True) df_pass.rename(columns=lambda x: x.replace('_task', 'passed'), inplace=True) From 4cb2ee3a33b5a493521c7c5b58ffc67a73de5a1b Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 16 Oct 2024 12:56:05 +0100 Subject: [PATCH 35/40] Remove 'peakVelocity_times` from QC trials table --- ibllib/qc/task_qc_viewer/task_qc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py index 8d0c034c9..4e3be3ca9 100644 --- a/ibllib/qc/task_qc_viewer/task_qc.py +++ b/ibllib/qc/task_qc_viewer/task_qc.py @@ -288,7 +288,10 @@ def show_session_task_qc(qc_or_session=None, bpod_only=False, local=False, one=N # Update table and callbacks n_trials = qc.frame.shape[0] - df_trials = pd.DataFrame({k: v for k, v in task_qc.extractor.data.items() if v.size == n_trials}) + df_trials = pd.DataFrame({ + k: v for k, v in task_qc.extractor.data.items() + if v.size == n_trials and not k.startswith('peakVelocity') + }) df_pass = pd.DataFrame({k: v for k, v in qc.qc.passed.items() if isinstance(v, np.ndarray) and v.size == n_trials}) df_pass.drop('_task_passed_trial_checks', axis=1, errors='ignore', inplace=True) df_pass.rename(columns=lambda x: x.replace('_task', 'passed'), inplace=True) From f32005d700c2fd1655097c177794a649134d642a Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 16 Oct 2024 13:04:30 +0100 Subject: [PATCH 36/40] Revert "Remove 'peakVelocity_times` from QC trials table" This reverts commit 4cb2ee3a33b5a493521c7c5b58ffc67a73de5a1b. --- ibllib/qc/task_qc_viewer/task_qc.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py index 4e3be3ca9..8d0c034c9 100644 --- a/ibllib/qc/task_qc_viewer/task_qc.py +++ b/ibllib/qc/task_qc_viewer/task_qc.py @@ -288,10 +288,7 @@ def show_session_task_qc(qc_or_session=None, bpod_only=False, local=False, one=N # Update table and callbacks n_trials = qc.frame.shape[0] - df_trials = pd.DataFrame({ - k: v for k, v in task_qc.extractor.data.items() - if v.size == n_trials and not k.startswith('peakVelocity') - }) + df_trials = pd.DataFrame({k: v for k, v in task_qc.extractor.data.items() if v.size == n_trials}) df_pass = pd.DataFrame({k: v for k, v in qc.qc.passed.items() if isinstance(v, np.ndarray) and v.size == n_trials}) df_pass.drop('_task_passed_trial_checks', axis=1, errors='ignore', inplace=True) df_pass.rename(columns=lambda x: x.replace('_task', 'passed'), inplace=True) From 5e54694279a380ecf95021c4d964df4159b0c988 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Wed, 16 Oct 2024 13:04:34 +0100 Subject: [PATCH 37/40] Reapply "Resolves #853" This reverts commit bbf915ddd937c506c6ac0e597fcebe5ff72891b9. --- ibllib/qc/task_qc_viewer/ViewEphysQC.py | 2 +- ibllib/qc/task_qc_viewer/task_qc.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/ViewEphysQC.py b/ibllib/qc/task_qc_viewer/ViewEphysQC.py index 4e6684313..cae7431c2 100644 --- a/ibllib/qc/task_qc_viewer/ViewEphysQC.py +++ b/ibllib/qc/task_qc_viewer/ViewEphysQC.py @@ -229,7 +229,7 @@ def tv_double_clicked(self, index: QModelIndex): dt = t1 - t0 if self.wheel: idx = np.searchsorted(self.wheel['re_ts'], np.array([t0 - dt / 10, t1 + dt / 10])) - period = self.wheel['re_pos'][idx[0] : idx[1]] + period = self.wheel['re_pos'][idx[0]:idx[1]] if period.size == 0: _logger.warning('No wheel data during trial #%i', index.row()) else: diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py index 8d0c034c9..a49c703eb 100644 --- a/ibllib/qc/task_qc_viewer/task_qc.py +++ b/ibllib/qc/task_qc_viewer/task_qc.py @@ -288,7 +288,10 @@ def show_session_task_qc(qc_or_session=None, bpod_only=False, local=False, one=N # Update table and callbacks n_trials = qc.frame.shape[0] - df_trials = pd.DataFrame({k: v for k, v in task_qc.extractor.data.items() if v.size == n_trials}) + df_trials = pd.DataFrame({ + k: v for k, v in task_qc.extractor.data.items() + if v.size == n_trials and not k.startswith('wheel') + }) df_pass = pd.DataFrame({k: v for k, v in qc.qc.passed.items() if isinstance(v, np.ndarray) and v.size == n_trials}) df_pass.drop('_task_passed_trial_checks', axis=1, errors='ignore', inplace=True) df_pass.rename(columns=lambda x: x.replace('_task', 'passed'), inplace=True) From 8bfa7e365cbc8a6f76c83fc2200495b665eb7356 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Tue, 17 Dec 2024 15:05:56 +0000 Subject: [PATCH 38/40] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5f2fc9c35..b890b3e5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ tqdm>=4.32.1 iblatlas>=0.5.3 ibl-neuropixel>=1.5.0 iblutil>=1.13.0 -iblqt>=0.2.0 +iblqt>=0.3.2 mtscomp>=1.0.1 ONE-api>=2.11 phylib>=2.6.0 From fb3ea3188299c6666356ae704fde5c79bd3bca18 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Tue, 17 Dec 2024 15:47:56 +0000 Subject: [PATCH 39/40] fix CI --- ibllib/qc/task_qc_viewer/task_qc.py | 13 ++++++++----- ibllib/tests/qc/test_task_qc_viewer.py | 1 + requirements.txt | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py index a49c703eb..7b75589a4 100644 --- a/ibllib/qc/task_qc_viewer/task_qc.py +++ b/ibllib/qc/task_qc_viewer/task_qc.py @@ -288,14 +288,17 @@ def show_session_task_qc(qc_or_session=None, bpod_only=False, local=False, one=N # Update table and callbacks n_trials = qc.frame.shape[0] - df_trials = pd.DataFrame({ - k: v for k, v in task_qc.extractor.data.items() - if v.size == n_trials and not k.startswith('wheel') - }) + if 'task_qc' in locals(): + df_trials = pd.DataFrame({ + k: v for k, v in task_qc.extractor.data.items() + if v.size == n_trials and not k.startswith('wheel') + }) + df = df_trials.merge(qc.frame, left_index=True, right_index=True) + else: + df = qc.frame df_pass = pd.DataFrame({k: v for k, v in qc.qc.passed.items() if isinstance(v, np.ndarray) and v.size == n_trials}) df_pass.drop('_task_passed_trial_checks', axis=1, errors='ignore', inplace=True) df_pass.rename(columns=lambda x: x.replace('_task', 'passed'), inplace=True) - df = df_trials.merge(qc.frame, left_index=True, right_index=True) df = df.merge(df_pass.astype('boolean'), left_index=True, right_index=True) w.updateDataframe(df) qt.run_app() diff --git a/ibllib/tests/qc/test_task_qc_viewer.py b/ibllib/tests/qc/test_task_qc_viewer.py index 6db045f91..7115f371f 100644 --- a/ibllib/tests/qc/test_task_qc_viewer.py +++ b/ibllib/tests/qc/test_task_qc_viewer.py @@ -66,6 +66,7 @@ def test_show_session_task_qc(self, trials_tasks_mock, run_app_mock): qc_mock.compute_session_status.return_value = ('Fail', qc_mock.metrics, {'foo': 'FAIL'}) qc_mock.extractor.data = {'intervals': np.array([[0, 1]])} qc_mock.extractor.frame_ttls = qc_mock.extractor.audio_ttls = qc_mock.extractor.bpod_ttls = mock.MagicMock() + qc_mock.passed = dict() active_task = mock.Mock(spec=ChoiceWorldTrialsNidq, unsafe=True) active_task.run_qc.return_value = qc_mock diff --git a/requirements.txt b/requirements.txt index 5f2fc9c35..b890b3e5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ tqdm>=4.32.1 iblatlas>=0.5.3 ibl-neuropixel>=1.5.0 iblutil>=1.13.0 -iblqt>=0.2.0 +iblqt>=0.3.2 mtscomp>=1.0.1 ONE-api>=2.11 phylib>=2.6.0 From ad682fea463a6d1e0d6aaf9b785ff61bfc66b0da Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Tue, 17 Dec 2024 15:51:27 +0000 Subject: [PATCH 40/40] Update task_qc.py --- ibllib/qc/task_qc_viewer/task_qc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py index 7b75589a4..b9f212a5c 100644 --- a/ibllib/qc/task_qc_viewer/task_qc.py +++ b/ibllib/qc/task_qc_viewer/task_qc.py @@ -241,7 +241,8 @@ def show_session_task_qc(qc_or_session=None, bpod_only=False, local=False, one=N if isinstance(qc_or_session, QcFrame): qc = qc_or_session elif isinstance(qc_or_session, TaskQC): - qc = QcFrame(qc_or_session) + task_qc = qc_or_session + qc = QcFrame(task_qc) else: # assumed to be eid or session path one = one or ONE(mode='local' if local else 'auto') if not is_session_path(Path(qc_or_session)):