From 8ac33bbe3b670b214abae763ca6aa5dde57e9ae3 Mon Sep 17 00:00:00 2001 From: Marko Toplak Date: Mon, 4 Sep 2023 12:57:45 +0200 Subject: [PATCH 1/3] Spectra: compute individual display in own thread --- .../spectroscopy/tests/test_owspectra.py | 2 +- .../spectroscopy/widgets/owspectra.py | 154 +++++++++++------- 2 files changed, 100 insertions(+), 56 deletions(-) diff --git a/orangecontrib/spectroscopy/tests/test_owspectra.py b/orangecontrib/spectroscopy/tests/test_owspectra.py index 4ba6a269c..7b12ff712 100644 --- a/orangecontrib/spectroscopy/tests/test_owspectra.py +++ b/orangecontrib/spectroscopy/tests/test_owspectra.py @@ -32,7 +32,7 @@ def wait_for_graph(widget, timeout=5000): concurrent = widget.curveplot.show_average_thread if concurrent.task is not None: - spy = QSignalSpy(concurrent.average_shown) + spy = QSignalSpy(concurrent.shown) assert spy.wait(timeout), "Failed to update graph in the specified timeout" diff --git a/orangecontrib/spectroscopy/widgets/owspectra.py b/orangecontrib/spectroscopy/widgets/owspectra.py index 9ced32d75..b6e510c31 100644 --- a/orangecontrib/spectroscopy/widgets/owspectra.py +++ b/orangecontrib/spectroscopy/widgets/owspectra.py @@ -278,21 +278,21 @@ class InterruptException(Exception): class ShowAverage(QObject, ConcurrentMixin): - average_shown = pyqtSignal() + shown = pyqtSignal() def __init__(self, master): super().__init__(parent=master) ConcurrentMixin.__init__(self) self.master = master - def show_average(self): + def show(self): master = self.master master.clear_graph() # calls cancel master.view_average_menu.setChecked(True) master.set_pen_colors() master.viewtype = AVERAGE if not master.data: - self.average_shown.emit() + self.shown.emit() else: color_var = master.feature_color self.start(self.compute_averages, master.data, color_var, master.subset_indices, @@ -382,7 +382,93 @@ def on_done(self, res): master.curves_cont.update() master.plot.vb.set_mode_panning() - self.average_shown.emit() + self.shown.emit() + + def on_partial_result(self, result): + pass + + def on_exception(self, ex: Exception): + if isinstance(ex, InterruptException): + return + + raise ex + + +class ShowIndividual(QObject, ConcurrentMixin): + + shown = pyqtSignal() + + def __init__(self, master): + super().__init__(parent=master) + ConcurrentMixin.__init__(self) + self.master = master + + def show(self): + master = self.master + master.clear_graph() # calls cancel + master.view_average_menu.setChecked(False) + master.set_pen_colors() + master.viewtype = INDIVIDUAL + if not master.data: + return + sampled_indices = master._compute_sample(master.data.X) + self.start(self.compute_curves, master.data_x, master.data.X, + master.data_xsind, sampled_indices) + + @staticmethod + def compute_curves(x, ys, data_xsind, sampled_indices, state: TaskState): + def progress_interrupt(i: float): + if state.is_interruption_requested(): + raise InterruptException + + progress_interrupt(0) + ys = np.asarray(ys[sampled_indices][:, data_xsind]) + ys[np.isinf(ys)] = np.nan # remove infs that could ruin display + progress_interrupt(0) + return x, ys, sampled_indices + + def on_done(self, res): + x, ys, sampled_indices = res + + master = self.master + + if master.waterfall: + waterfall_constant = 0.1 + miny = bottleneck.nanmin(ys) + maxy = bottleneck.nanmax(ys) + space = (maxy - miny) * waterfall_constant + mul = (np.arange(len(ys))*space + 1).reshape(-1, 1) + ys = ys * mul + + # shuffle the data before drawing because classes often appear sequentially + # and the last class would then seem the most prevalent if colored + indices = list(range(len(sampled_indices))) + random.Random(master.sample_seed).shuffle(indices) + sampled_indices = [sampled_indices[i] for i in indices] + master.sampled_indices = sampled_indices + master.sampled_indices_inverse = {s: i for i, s in enumerate(master.sampled_indices)} + master.new_sampling.emit(len(master.sampled_indices)) + ys = ys[indices] # ys was already subsampled + + master.curves.append((x, ys)) + + # add curves efficiently + for y in ys: + master.add_curve(x, y, ignore_bounds=True) + + if x.size and ys.size: + bounding_rect = QGraphicsRectItem(QRectF( + QPointF(bottleneck.nanmin(x), bottleneck.nanmin(ys)), + QPointF(bottleneck.nanmax(x), bottleneck.nanmax(ys)))) + bounding_rect.setPen(QPen(Qt.NoPen)) # prevents border of 1 + master.curves_cont.add_bounds(bounding_rect) + + master.curves_plotted.append((x, ys)) + master.set_curve_pens() + master.curves_cont.update() + master.plot.vb.set_mode_panning() + + self.shown.emit() def on_partial_result(self, result): pass @@ -798,7 +884,10 @@ def __init__(self, parent: OWWidget, select=SELECTNONE): SelectionGroupMixin.__init__(self) self.show_average_thread = ShowAverage(self) - self.show_average_thread.average_shown.connect(self.rescale) + self.show_average_thread.shown.connect(self.rescale) + + self.show_individual_thread = ShowIndividual(self) + self.show_individual_thread.shown.connect(self.rescale) self.parent = parent @@ -1148,6 +1237,7 @@ def clear_data(self): def clear_graph(self): self.show_average_thread.cancel() + self.show_individual_thread.cancel() self.highlighted = None # reset caching. if not, it is not cleared when view changing when zoomed self.curves_cont.setCacheMode(QGraphicsItem.NoCache) @@ -1171,7 +1261,6 @@ def clear_graph(self): self.sampled_indices = [] self.sampled_indices_inverse = {} - self.sampling = None self.new_sampling.emit(None) for m in self.markings: @@ -1405,8 +1494,7 @@ def clear_connect_views(self): self.plot.scene().removeItem(v) self.connected_views = [] - def add_curves(self, x, ys, addc=True): - """ Add multiple curves with the same x domain. """ + def _compute_sample(self, ys): if len(ys) > MAX_INSTANCES_DRAWN: sample_selection = \ random.Random(self.sample_seed).sample(range(len(ys)), MAX_INSTANCES_DRAWN) @@ -1419,44 +1507,9 @@ def add_curves(self, x, ys, addc=True): subset_to_show = \ random.Random(self.sample_seed).sample(subset_to_show, subset_additional) sampled_indices = sorted(sample_selection + list(subset_to_show)) - self.sampling = True else: sampled_indices = list(range(len(ys))) - ys = np.asarray(self.data.X[sampled_indices][:, self.data_xsind]) - ys[np.isinf(ys)] = np.nan # remove infs that could ruin display - - if self.waterfall: - waterfall_constant = 0.1 - miny = bottleneck.nanmin(ys) - maxy = bottleneck.nanmax(ys) - space = (maxy - miny) * waterfall_constant - mul = (np.arange(len(ys))*space + 1).reshape(-1, 1) - ys = ys * mul - - # shuffle the data before drawing because classes often appear sequentially - # and the last class would then seem the most prevalent if colored - indices = list(range(len(sampled_indices))) - random.Random(self.sample_seed).shuffle(indices) - sampled_indices = [sampled_indices[i] for i in indices] - self.sampled_indices = sampled_indices - self.sampled_indices_inverse = {s: i for i, s in enumerate(self.sampled_indices)} - self.new_sampling.emit(len(self.sampled_indices)) - ys = ys[indices] # ys was already subsampled - - self.curves.append((x, ys)) - - # add curves efficiently - for y in ys: - self.add_curve(x, y, ignore_bounds=True) - - if x.size and ys.size: - bounding_rect = QGraphicsRectItem(QRectF( - QPointF(bottleneck.nanmin(x), bottleneck.nanmin(ys)), - QPointF(bottleneck.nanmax(x), bottleneck.nanmax(ys)))) - bounding_rect.setPen(QPen(Qt.NoPen)) # prevents border of 1 - self.curves_cont.add_bounds(bounding_rect) - - self.curves_plotted.append((x, ys)) + return sampled_indices def add_curve(self, x, y, pen=None, ignore_bounds=False): c = FinitePlotCurveItem(x=x, y=y, pen=pen if pen else self.pen_normal[None]) @@ -1518,16 +1571,7 @@ def set_pen_colors(self): self.legend.setVisible(bool(self.legend.items)) def show_individual(self): - self.clear_graph() - self.view_average_menu.setChecked(False) - self.set_pen_colors() - self.viewtype = INDIVIDUAL - if not self.data: - return - self.add_curves(self.data_x, self.data.X) - self.set_curve_pens() - self.curves_cont.update() - self.plot.vb.set_mode_panning() + self.show_individual_thread.show() def resample_curves(self, seed): self.sample_seed = seed @@ -1572,7 +1616,7 @@ def waterfall_apply(self): self.view_waterfall_menu.setChecked(self.waterfall) def show_average(self): - self.show_average_thread.show_average() + self.show_average_thread.show() def update_view(self): if self.viewtype == INDIVIDUAL: From cef7f0b850101c33c39404d2ea6e09042c05986f Mon Sep 17 00:00:00 2001 From: Marko Toplak Date: Tue, 5 Sep 2023 08:08:53 +0200 Subject: [PATCH 2/3] Individual display in thread: tests --- .../spectroscopy/tests/test_owpreprocess.py | 3 ++ .../spectroscopy/tests/test_owspectra.py | 37 +++++++++++++++++-- .../spectroscopy/widgets/owspectra.py | 3 ++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/orangecontrib/spectroscopy/tests/test_owpreprocess.py b/orangecontrib/spectroscopy/tests/test_owpreprocess.py index 3ead2d46e..7039bbf1c 100644 --- a/orangecontrib/spectroscopy/tests/test_owpreprocess.py +++ b/orangecontrib/spectroscopy/tests/test_owpreprocess.py @@ -7,6 +7,7 @@ from orangecontrib.spectroscopy.data import getx from orangecontrib.spectroscopy.tests import spectral_preprocess from orangecontrib.spectroscopy.tests.spectral_preprocess import pack_editor, wait_for_preview +from orangecontrib.spectroscopy.tests.test_owspectra import wait_for_graph from orangecontrib.spectroscopy.widgets.owpreprocess import OWPreprocess from orangecontrib.spectroscopy.widgets.preprocessors.misc import \ CutEditor, SavitzkyGolayFilteringEditor @@ -100,6 +101,8 @@ def test_transfer_highlight(self): data = SMALL_COLLAGEN self.send_signal("Data", data) wait_for_preview(self.widget) + wait_for_graph(self.widget.curveplot) + wait_for_graph(self.widget.curveplot_after) self.widget.curveplot.highlight(1) self.assertEqual(self.widget.curveplot_after.highlighted, 1) self.widget.curveplot.highlight(None) diff --git a/orangecontrib/spectroscopy/tests/test_owspectra.py b/orangecontrib/spectroscopy/tests/test_owspectra.py index 7b12ff712..3b9939768 100644 --- a/orangecontrib/spectroscopy/tests/test_owspectra.py +++ b/orangecontrib/spectroscopy/tests/test_owspectra.py @@ -18,7 +18,7 @@ from Orange.data import Table, Domain, ContinuousVariable, DiscreteVariable from orangecontrib.spectroscopy.widgets.owspectra import OWSpectra, MAX_INSTANCES_DRAWN, \ - PlotCurvesItem, NoSuchCurve, MAX_THICK_SELECTED + PlotCurvesItem, NoSuchCurve, MAX_THICK_SELECTED, CurvePlot from orangecontrib.spectroscopy.data import getx from orangecontrib.spectroscopy.widgets.line_geometry import intersect_curves, \ distance_line_segment @@ -30,9 +30,12 @@ def wait_for_graph(widget, timeout=5000): - concurrent = widget.curveplot.show_average_thread - if concurrent.task is not None: - spy = QSignalSpy(concurrent.shown) + if not isinstance(widget, CurvePlot): + widget = widget.curveplot + av = widget.show_average_thread + ind = widget.show_individual_thread + if av.task is not None or ind.task is not None: + spy = QSignalSpy(widget.graph_shown) assert spy.wait(timeout), "Failed to update graph in the specified timeout" @@ -88,6 +91,7 @@ def test_is_last_instance_force_sampling_and_permutation(self): mi = "orangecontrib.spectroscopy.widgets.owspectra.MAX_INSTANCES_DRAWN" with patch(mi, 100): self.send_signal("Data", self.unknown_last_instance) + wait_for_graph(self.widget) self.assertTrue(np.all(np.isnan(self.unknown_last_instance[self.widget.curveplot.sampled_indices].X[-1]))) def do_mousemove(self): @@ -123,6 +127,7 @@ def test_empty(self): def test_mouse_move(self): for data in self.normal_data + self.strange_data: self.send_signal("Data", data) + wait_for_graph(self.widget) self.do_mousemove() def test_rescale_y(self): @@ -150,6 +155,7 @@ def select_diagonal(self): def test_select_line(self): for data in self.normal_data: self.send_signal("Data", data) + wait_for_graph(self.widget) out = self.get_output("Selection") self.assertIsNone(out, None) out = self.get_output(self.widget.Outputs.annotated_data) @@ -199,10 +205,13 @@ def test_warning_no_x(self): def test_information(self): assert len(self.titanic) > MAX_INSTANCES_DRAWN self.send_signal("Data", self.titanic[:MAX_INSTANCES_DRAWN]) + wait_for_graph(self.widget) self.assertFalse(self.widget.Information.showing_sample.is_shown()) self.send_signal("Data", self.titanic) + wait_for_graph(self.widget) self.assertTrue(self.widget.Information.showing_sample.is_shown()) self.send_signal("Data", self.titanic[:MAX_INSTANCES_DRAWN]) + wait_for_graph(self.widget) self.assertFalse(self.widget.Information.showing_sample.is_shown()) def test_information_average_mode(self): @@ -215,12 +224,14 @@ def test_information_average_mode(self): def test_handle_floatname(self): self.send_signal("Data", self.collagen) + wait_for_graph(self.widget) x, _ = self.widget.curveplot.curves[0] fs = sorted([float(f.name) for f in self.collagen.domain.attributes]) np.testing.assert_equal(x, fs) def test_handle_nofloatname(self): self.send_signal("Data", self.iris) + wait_for_graph(self.widget) x, _ = self.widget.curveplot.curves[0] np.testing.assert_equal(x, range(len(self.iris.domain.attributes))) @@ -230,12 +241,14 @@ def numcurves(curves): self.send_signal("Data", self.iris) curves_plotted = self.widget.curveplot.curves_plotted + wait_for_graph(self.widget) self.assertEqual(numcurves(curves_plotted), 150) self.widget.curveplot.show_average() wait_for_graph(self.widget) curves_plotted = self.widget.curveplot.curves_plotted self.assertEqual(numcurves(curves_plotted), 3) self.widget.curveplot.show_individual() + wait_for_graph(self.widget) curves_plotted = self.widget.curveplot.curves_plotted self.assertEqual(numcurves(curves_plotted), 150) @@ -273,18 +286,21 @@ def test_subset(self): assert len(data) > MAX_INSTANCES_DRAWN self.send_signal("Data", data) + wait_for_graph(self.widget) sinds = self.widget.curveplot.sampled_indices self.assertEqual(len(sinds), MAX_INSTANCES_DRAWN) # the whole subset is drawn add_subset = data[:MAX_INSTANCES_DRAWN] self.send_signal("Data subset", add_subset) + wait_for_graph(self.widget) sinds = self.widget.curveplot.sampled_indices self.assertTrue(set(add_subset.ids) <= set(data[sinds].ids)) # the whole subset can not be drawn anymore add_subset = data[:MAX_INSTANCES_DRAWN+1] self.send_signal("Data subset", add_subset) + wait_for_graph(self.widget) sinds = self.widget.curveplot.sampled_indices self.assertFalse(set(add_subset.ids) <= set(data[sinds].ids)) @@ -354,6 +370,7 @@ def test_migrate_selection(self): def test_selection_changedata(self): # select something in the widget and see if it is cleared self.send_signal("Data", self.iris) + wait_for_graph(self.widget) self.widget.curveplot.MOUSE_RADIUS = 1000 self.widget.curveplot.mouse_moved_closest( (self.widget.curveplot.plot.sceneBoundingRect().center(),)) @@ -362,10 +379,12 @@ def test_selection_changedata(self): self.assertEqual(len(out), 1) # resending the exact same data should not change the selection self.send_signal("Data", self.iris) + wait_for_graph(self.widget) out2 = self.get_output("Selection") self.assertEqual(len(out), 1) # while resending the same data as a different object should self.send_signal("Data", self.iris.copy()) + wait_for_graph(self.widget) out = self.get_output("Selection") self.assertIsNone(out, None) @@ -406,6 +425,7 @@ def test_select_thick_lines(self): data = self.collagen[:100] assert MAX_INSTANCES_DRAWN >= len(data) > MAX_THICK_SELECTED self.send_signal("Data", data) + wait_for_graph(self.widget) self.widget.curveplot.make_selection(list(range(MAX_THICK_SELECTED))) self.assertEqual(2, self.widget.curveplot.pen_selected[None].width()) self.widget.curveplot.make_selection(list(range(MAX_THICK_SELECTED + 1))) @@ -418,6 +438,7 @@ def test_select_thick_lines_threshold(self): assert MAX_INSTANCES_DRAWN >= len(data) > MAX_THICK_SELECTED threshold = MAX_THICK_SELECTED self.send_signal("Data", data) + wait_for_graph(self.widget) set_curve_pens = 'orangecontrib.spectroscopy.widgets.owspectra.CurvePlot.set_curve_pens' with patch(set_curve_pens, Mock()) as m: @@ -456,6 +477,7 @@ def test_curveplot_highlight(self): data = self.iris[:10] curveplot = self.widget.curveplot self.send_signal("Data", data) + wait_for_graph(self.widget) self.assertIsNone(curveplot.highlighted) m = Mock() @@ -486,14 +508,17 @@ def test_new_data_clear_graph(self): curveplot = self.widget.curveplot curveplot.set_data(self.iris[:3], auto_update=False) curveplot.update_view() + wait_for_graph(self.widget) self.assertEqual(3, len(curveplot.curves[0][1])) curveplot.set_data(self.iris[:3], auto_update=False) self.assertEqual([], curveplot.curves) curveplot.update_view() + wait_for_graph(self.widget) self.assertEqual(3, len(curveplot.curves[0][1])) def test_save_graph(self): self.send_signal("Data", self.iris) + wait_for_graph(self.widget) with set_png_graph_save() as fname: self.widget.save_graph() self.assertGreater(os.path.getsize(fname), 10000) @@ -501,6 +526,7 @@ def test_save_graph(self): def test_peakline_keep_precision(self): self.widget.curveplot.add_peak_label() # precision 1 (no data) self.send_signal("Data", self.collagen) + wait_for_graph(self.widget) self.widget.curveplot.add_peak_label() # precision 2 (data range is collagen) label_text = self.widget.curveplot.peak_labels[0].label.textItem.toPlainText() label_text2 = self.widget.curveplot.peak_labels[1].label.textItem.toPlainText() @@ -514,6 +540,7 @@ def test_peakline_keep_precision(self): def test_peakline_remove(self): self.widget.curveplot.add_peak_label() self.send_signal("Data", self.collagen) + wait_for_graph(self.widget) self.widget.curveplot.add_peak_label() self.widget.curveplot.peak_labels[0].request_deletion() self.assertEqual(len(self.widget.curveplot.peak_labels), 1) @@ -534,8 +561,10 @@ def test_waterfall(self): vb = self.widget.curveplot.plot.vb self.widget.curveplot.waterfall = False self.widget.curveplot.update_view() + wait_for_graph(self.widget) self.assertTrue(vb.targetRect().bottom() < 10) self.widget.curveplot.waterfall_changed() + wait_for_graph(self.widget) self.assertTrue(vb.targetRect().bottom() < 1000) self.assertTrue(vb.targetRect().bottom() > 800) diff --git a/orangecontrib/spectroscopy/widgets/owspectra.py b/orangecontrib/spectroscopy/widgets/owspectra.py index b6e510c31..305c61b3c 100644 --- a/orangecontrib/spectroscopy/widgets/owspectra.py +++ b/orangecontrib/spectroscopy/widgets/owspectra.py @@ -877,6 +877,7 @@ class CurvePlot(QWidget, OWComponent, SelectionGroupMixin): new_sampling = pyqtSignal(object) highlight_changed = pyqtSignal() locked_axes_changed = pyqtSignal(bool) + graph_shown = pyqtSignal() def __init__(self, parent: OWWidget, select=SELECTNONE): QWidget.__init__(self) @@ -885,9 +886,11 @@ def __init__(self, parent: OWWidget, select=SELECTNONE): self.show_average_thread = ShowAverage(self) self.show_average_thread.shown.connect(self.rescale) + self.show_average_thread.shown.connect(self.graph_shown.emit) self.show_individual_thread = ShowIndividual(self) self.show_individual_thread.shown.connect(self.rescale) + self.show_individual_thread.shown.connect(self.graph_shown.emit) self.parent = parent From 9b0dbba3c480c56a15d0fa7d7512eae1ebeeea73 Mon Sep 17 00:00:00 2001 From: Marko Toplak Date: Tue, 5 Sep 2023 13:35:42 +0200 Subject: [PATCH 3/3] Fix shutdown thread --- orangecontrib/spectroscopy/widgets/owspectra.py | 1 + 1 file changed, 1 insertion(+) diff --git a/orangecontrib/spectroscopy/widgets/owspectra.py b/orangecontrib/spectroscopy/widgets/owspectra.py index 305c61b3c..358ca02ed 100644 --- a/orangecontrib/spectroscopy/widgets/owspectra.py +++ b/orangecontrib/spectroscopy/widgets/owspectra.py @@ -1703,6 +1703,7 @@ def intersect_curves(self, q1, q2): def shutdown(self): self.show_average_thread.shutdown() + self.show_individual_thread.shutdown() @classmethod def migrate_settings_sub(cls, settings, version):