From 7577443f95bdc6a73d9c24fab622bb7c6e4b38a8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 6 Mar 2022 17:15:43 -0500 Subject: [PATCH 01/84] Add guard for real-time-not-active last line is `None` case --- piker/ui/_curve.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index f10f874c..30a42b05 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -140,8 +140,8 @@ class FastAppendCurve(pg.PlotCurveItem): # self.last_step_pen = pg.mkPen(hcolor(color), width=2) self.last_step_pen = pg.mkPen(pen, width=2) - self._last_line: QLineF = None - self._last_step_rect: QRectF = None + self._last_line: Optional[QLineF] = None + self._last_step_rect: Optional[QRectF] = None # flat-top style histogram-like discrete curve self._step_mode: bool = step_mode @@ -388,12 +388,13 @@ class FastAppendCurve(pg.PlotCurveItem): # p.drawPath(self.path) # profiler('.drawPath()') - p.setPen(self.last_step_pen) - p.drawLine(self._last_line) - profiler('.drawLine()') + if self._last_line: + p.setPen(self.last_step_pen) + p.drawLine(self._last_line) + profiler('.drawLine()') + p.setPen(self.opts['pen']) # else: - p.setPen(self.opts['pen']) p.drawPath(self.path) profiler('.drawPath()') From 11f8c4f350664528dbe3113eccc9f71c0bbd6114 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 6 Mar 2022 17:16:07 -0500 Subject: [PATCH 02/84] Add detailed `.addItem()`` comment --- piker/ui/_chart.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 3fcaae07..9ae98418 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1046,7 +1046,17 @@ class ChartPlotWidget(pg.PlotWidget): # (we need something that avoids clutter on x-axis). self._add_sticky(name, bg_color=color) + # NOTE: this is more or less the RENDER call that tells Qt to + # start showing the generated graphics-curves. This is kind of + # of edge-triggered call where once added any + # ``QGraphicsItem.update()`` calls are automatically displayed. + # Our internal graphics objects have their own "update from + # data" style method API that allows for real-time updates on + # the next render cycle; just note a lot of the real-time + # updates are implicit and require a bit of digging to + # understand. pi.addItem(curve) + return curve, data_key # TODO: make this a ctx mngr From 7e853fe34535e65c1e2f62f26499927381c58dcd Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 9 Mar 2022 11:01:01 -0500 Subject: [PATCH 03/84] Add a downsampled line-curve support to `BarItems` In effort to start getting some graphics speedups as detailed in #109, this adds a `FastAppendCurve`to every `BarItems` as a `._ds_line` which is only displayed (instead of the normal mult-line bars curve) when the "width" of a bar is indistinguishable on screen from a line -> so once the view coordinates map to > 2 pixels on the display device. `BarItems.maybe_paint_line()` takes care of this scaling detection logic and is called by the associated view's `.sigXRangeChanged` signal handler. --- piker/ui/_interaction.py | 16 ++++++ piker/ui/_ohlc.py | 112 +++++++++++++++++++++++++++++++++++---- 2 files changed, 119 insertions(+), 9 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index dca41855..5b0145ff 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -36,6 +36,7 @@ from ..log import get_logger from ._style import _min_points_to_show from ._editors import SelectRect from . import _event +from ._ohlc import BarItems log = get_logger(__name__) @@ -429,6 +430,12 @@ class ChartView(ViewBox): def maxmin(self, callback: Callable) -> None: self._maxmin = callback + + def maybe_downsample_graphics(self): + for graphic in self._chart._graphics.values(): + if isinstance(graphic, BarItems): + graphic.maybe_paint_line() + def wheelEvent( self, ev, @@ -775,6 +782,15 @@ class ChartView(ViewBox): ''' vb.sigXRangeChanged.connect(vb._set_yrange) + + # TODO: a smarter way to avoid calling this needlessly? + # 2 things i can think of: + # - register downsample-able graphics specially and only + # iterate those. + # - only register this when certain downsampleable graphics are + # "added to scene". + vb.sigXRangeChanged.connect(vb.maybe_downsample_graphics) + # mouse wheel doesn't emit XRangeChanged vb.sigRangeChangedManually.connect(vb._set_yrange) vb.sigResized.connect(vb._set_yrange) # splitter(s) resizing diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 4331f036..8ed14080 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -29,6 +29,10 @@ from PyQt5.QtCore import QLineF, QPointF from .._profile import pg_profile_enabled from ._style import hcolor +from ..log import get_logger + + +log = get_logger(__name__) def _mk_lines_array( @@ -170,8 +174,10 @@ def gen_qpath( class BarItems(pg.GraphicsObject): - """Price range bars graphics rendered from a OHLC sequence. - """ + ''' + "Price range" bars graphics rendered from a OHLC sampled sequence. + + ''' sigPlotChanged = QtCore.pyqtSignal(object) # 0.5 is no overlap between arms, 1.0 is full overlap @@ -183,11 +189,15 @@ class BarItems(pg.GraphicsObject): plotitem: 'pg.PlotItem', # noqa pen_color: str = 'bracket', last_bar_color: str = 'bracket', - ) -> None: - super().__init__() + name: Optional[str] = None, + + ) -> None: + + super().__init__() # XXX: for the mega-lulz increasing width here increases draw # latency... so probably don't do it until we figure that out. + self._color = pen_color self.bars_pen = pg.mkPen(hcolor(pen_color), width=1) self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2) @@ -219,15 +229,20 @@ class BarItems(pg.GraphicsObject): self.start_index: int = 0 self.stop_index: int = 0 + self._in_ds: bool = False + def draw_from_data( self, data: np.ndarray, start: int = 0, + ) -> QtGui.QPainterPath: - """Draw OHLC datum graphics from a ``np.ndarray``. + ''' + Draw OHLC datum graphics from a ``np.ndarray``. This routine is usually only called to draw the initial history. - """ + + ''' hist, last = data[:-1], data[-1] self.path = gen_qpath(hist, start, self.w) @@ -249,14 +264,28 @@ class BarItems(pg.GraphicsObject): # https://doc.qt.io/qt-5/qgraphicsitem.html#update self.update() + from ._curve import FastAppendCurve + self._ds_line = FastAppendCurve( + y=data['close'], + x=data['index'], + name='ohlc_ds_line', + color=self._color, + # use_polyline=True, # pretty sure this is slower? + ) + self.update_from_array(data) + self._pi.addItem(self._ds_line) + self._ds_line.hide() + return self.path def update_from_array( self, array: np.ndarray, just_history=False, + ) -> None: - """Update the last datum's bar graphic from input data array. + ''' + Update the last datum's bar graphic from input data array. This routine should be interface compatible with ``pg.PlotCurveItem.setData()``. Normally this method in @@ -266,7 +295,16 @@ class BarItems(pg.GraphicsObject): does) so this "should" be simpler and faster. This routine should be made (transitively) as fast as possible. - """ + + ''' + # XXX: always do this? + if self._in_ds: + self._ds_line.update_from_array( + x=array['index'], + y=array['close'], + ) + return + # index = self.start_index istart, istop = self._xrange @@ -400,14 +438,59 @@ class BarItems(pg.GraphicsObject): ) + def maybe_paint_line( + self, + x_gt: float = 2., + + ) -> bool: + ''' + Call this when you want to stop drawing individual + bars and instead use a ``FastAppendCurve`` intepolation + line (normally when the width of a bar (aka 1.0 in the x) + is less then a pixel width on the device). + + ''' + # this is the ``float`` value of the "number of x units" (in + # view coords) that a pixel spans. + xs_in_px = self.pixelVectors()[0].x() + if ( + not self._in_ds + and xs_in_px >= x_gt + ): + # TODO: a `.ui()` log level? + log.info(f'downsampling to line graphic') + self._in_ds = True + self.hide() + self._pi.addItem(self._ds_line) + self._ds_line.show() + return True + + elif ( + self._in_ds + and xs_in_px < x_gt + ): + log.info(f'showing bars graphic') + self._in_ds = False + self.show() + self._ds_line.hide() + self._pi.removeItem(self._ds_line) + return False + def paint( self, p: QtGui.QPainter, opt: QtWidgets.QStyleOptionGraphicsItem, w: QtWidgets.QWidget + ) -> None: - profiler = pg.debug.Profiler(disabled=not pg_profile_enabled()) + if self._in_ds: + return + + profiler = pg.debug.Profiler( + disabled=not pg_profile_enabled(), + delayed=False, + ) # p.setCompositionMode(0) @@ -424,3 +507,14 @@ class BarItems(pg.GraphicsObject): p.setPen(self.bars_pen) p.drawPath(self.path) profiler('draw history path') + profiler.finish() + + # NOTE: for testing paint frequency as throttled by display loop. + # now = time.time() + # global _last_draw + # print(f'DRAW RATE {1/(now - _last_draw)}') + # _last_draw = now + + +# import time +# _last_draw: float = time.time() From 7811508307d7403a2ea54aa237c7d11b474fc082 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 9 Mar 2022 11:07:53 -0500 Subject: [PATCH 04/84] Add basic optional polyline support, draft out downsampling routine --- piker/ui/_curve.py | 162 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 125 insertions(+), 37 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 30a42b05..18a777a8 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -23,6 +23,7 @@ from typing import Optional import numpy as np import pyqtgraph as pg from PyQt5 import QtGui, QtWidgets +from PyQt5.QtWidgets import QGraphicsItem from PyQt5.QtCore import ( Qt, QLineF, @@ -94,6 +95,38 @@ _line_styles: dict[str, int] = { } +def downsample( + x: np.ndarray, + y: np.ndarray, + bins: int, + method: str = 'peak', + +) -> tuple[np.ndarray, np.ndarray]: + ''' + Downsample x/y data for lesser curve graphics gen. + + The "peak" method is originally copied verbatim from + ``pyqtgraph.PlotDataItem.getDisplayDataset()``. + + ''' + match method: + case 'peak': + ds = bins + n = len(x) // ds + x1 = np.empty((n, 2)) + # start of x-values; try to select a somewhat centered point + stx = ds//2 + x1[:] = x[stx:stx+n*ds:ds, np.newaxis] + x = x1.reshape(n*2) + y1 = np.empty((n, 2)) + y2 = y[:n*ds].reshape((n, ds)) + y1[:, 0] = y2.max(axis=1) + y1[:, 1] = y2.min(axis=1) + y = y1.reshape(n*2) + + return x, y + + # TODO: got a feeling that dropping this inheritance gets us even more speedups class FastAppendCurve(pg.PlotCurveItem): ''' @@ -116,17 +149,23 @@ class FastAppendCurve(pg.PlotCurveItem): fill_color: Optional[str] = None, style: str = 'solid', name: Optional[str] = None, + use_polyline: bool = False, **kwargs ) -> None: + self._name = name + # TODO: we can probably just dispense with the parent since # we're basically only using the pen setting now... super().__init__(*args, **kwargs) - self._name = name self._xrange: tuple[int, int] = self.dataBounds(ax=0) + # self._last_draw = time.time() + self._use_poly = use_polyline + self.poly = None + # all history of curve is drawn in single px thickness pen = pg.mkPen(hcolor(color)) pen.setStyle(_line_styles[style]) @@ -153,12 +192,14 @@ class FastAppendCurve(pg.PlotCurveItem): # interactions slower (such as zooming) and if so maybe if/when # we implement a "history" mode for the view we disable this in # that mode? - if step_mode: + if step_mode or self._use_poly: # don't enable caching by default for the case where the # only thing drawn is the "last" line segment which can # have a weird artifact where it won't be fully drawn to its # endpoint (something we saw on trade rate curves) - self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + self.setCacheMode( + QGraphicsItem.DeviceCoordinateCache + ) def update_from_array( self, @@ -195,12 +236,20 @@ class FastAppendCurve(pg.PlotCurveItem): x_out, y_out = x[:-1], y[:-1] if self.path is None or prepend_length > 0: - self.path = pg.functions.arrayToQPath( - x_out, - y_out, - connect='all', - finiteCheck=False, - ) + + if self._use_poly: + self.poly = pg.functions.arrayToQPolygonF( + x_out, + y_out, + ) + + else: + self.path = pg.functions.arrayToQPath( + x_out, + y_out, + connect='all', + finiteCheck=False, + ) profiler('generate fresh path') # if self._step_mode: @@ -242,18 +291,27 @@ class FastAppendCurve(pg.PlotCurveItem): new_y = y[-append_length - 2:-1] # print((new_x, new_y)) - append_path = pg.functions.arrayToQPath( - new_x, - new_y, - connect='all', - # finiteCheck=False, - ) + if self._use_poly: + union_poly = pg.functions.arrayToQPolygonF( + new_x, + new_y, + ) + + else: + append_path = pg.functions.arrayToQPath( + new_x, + new_y, + connect='all', + finiteCheck=False, + ) path = self.path # other merging ideas: # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths if self._step_mode: + assert not self._use_poly, 'Dunno howw this worx yet' + # path.addPath(append_path) self.path.connectPath(append_path) @@ -269,19 +327,26 @@ class FastAppendCurve(pg.PlotCurveItem): # # path.closeSubpath() else: - # print(f"append_path br: {append_path.boundingRect()}") - # self.path.moveTo(new_x[0], new_y[0]) - # self.path.connectPath(append_path) - path.connectPath(append_path) + if self._use_poly: + self.poly = self.poly.united(union_poly) + else: + # print(f"append_path br: {append_path.boundingRect()}") + # self.path.moveTo(new_x[0], new_y[0]) + self.path.connectPath(append_path) + # path.connectPath(append_path) + + # XXX: lol this causes a hang.. + # self.path = self.path.simplified() self.disable_cache() flip_cache = True - if ( - self._step_mode - ): - self.disable_cache() - flip_cache = True + # XXX: do we need this any more? + # if ( + # self._step_mode + # ): + # self.disable_cache() + # flip_cache = True # print(f"update br: {self.path.boundingRect()}") @@ -318,7 +383,7 @@ class FastAppendCurve(pg.PlotCurveItem): if flip_cache: # XXX: seems to be needed to avoid artifacts (see above). - self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) def disable_cache(self) -> None: ''' @@ -334,15 +399,22 @@ class FastAppendCurve(pg.PlotCurveItem): ''' Compute and then cache our rect. ''' - if self.path is None: - return QtGui.QPainterPath().boundingRect() + if self._use_poly: + if self.poly is None: + return QtGui.QPolygonF().boundingRect() + else: + br = self.boundingRect = self.poly.boundingRect + return br() else: - # dynamically override this method after initial - # path is created to avoid requiring the above None check - self.boundingRect = self._br - return self._br() + if self.path is None: + return QtGui.QPainterPath().boundingRect() + else: + # dynamically override this method after initial + # path is created to avoid requiring the above None check + self.boundingRect = self._path_br + return self._path_br() - def _br(self): + def _path_br(self): ''' Post init ``.boundingRect()```. @@ -373,7 +445,9 @@ class FastAppendCurve(pg.PlotCurveItem): ) -> None: - profiler = pg.debug.Profiler(disabled=not pg_profile_enabled()) + profiler = pg.debug.Profiler( + # disabled=False, #not pg_profile_enabled(), + ) # p.setRenderHint(p.Antialiasing, True) if ( @@ -395,12 +469,26 @@ class FastAppendCurve(pg.PlotCurveItem): p.setPen(self.opts['pen']) # else: - p.drawPath(self.path) - profiler('.drawPath()') + if self._use_poly: + assert self.poly + p.drawPolyline(self.poly) + profiler('.drawPolyline()') + else: + p.drawPath(self.path) + profiler('.drawPath()') - # TODO: try out new work from `pyqtgraph` main which - # should repair horrid perf: + # TODO: try out new work from `pyqtgraph` main which should + # repair horrid perf (pretty sure i did and it was still + # horrible?): # https://github.com/pyqtgraph/pyqtgraph/pull/2032 # if self._fill: # brush = self.opts['brush'] # p.fillPath(self.path, brush) + + # now = time.time() + # print(f'DRAW RATE {1/(now - self._last_draw)}') + # self._last_draw = now + + +# import time +# _last_draw: float = time.time() From 7c4e55ed2c6c6027eff3173e0014dc8ea4692554 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 9 Mar 2022 11:08:49 -0500 Subject: [PATCH 05/84] Add comment on how to enable `pyqtgraph` profiling --- piker/_profile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/piker/_profile.py b/piker/_profile.py index 06abc09a..ca1fc188 100644 --- a/piker/_profile.py +++ b/piker/_profile.py @@ -21,7 +21,9 @@ Profiling wrappers for internal libs. import time from functools import wraps -_pg_profile: bool = True +# NOTE: you can pass a flag to enable this: +# ``piker chart --profile``. +_pg_profile: bool = False def pg_profile_enabled() -> bool: From 02300efb59461612ab5e05bd6bc9183013b81477 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 9 Mar 2022 11:29:52 -0500 Subject: [PATCH 06/84] Use 12Hz as default fps throttle --- piker/ui/_display.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 398a180c..b761b881 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -56,7 +56,7 @@ from ..log import get_logger log = get_logger(__name__) # TODO: load this from a config.toml! -_quote_throttle_rate: int = 6 + 16 # Hz +_quote_throttle_rate: int = 12 # Hz # a working tick-type-classes template From d7a99282932b37688320bee5ff148eed4aa59ef7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 9 Mar 2022 14:48:00 -0500 Subject: [PATCH 07/84] Move graphics compression routines to new module --- piker/ui/_compression.py | 129 +++++++++++++++++++++++++++++++++++++++ piker/ui/_curve.py | 36 +---------- 2 files changed, 130 insertions(+), 35 deletions(-) create mode 100644 piker/ui/_compression.py diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py new file mode 100644 index 00000000..46969584 --- /dev/null +++ b/piker/ui/_compression.py @@ -0,0 +1,129 @@ +# piker: trading gear for hackers +# Copyright (C) Tyler Goodlet (in stewardship for pikers) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import numpy as np +from numba import ( + jit, float64, optional, int64, +) + + +def downsample( + x: np.ndarray, + y: np.ndarray, + bins: int, + method: str = 'peak', + +) -> tuple[np.ndarray, np.ndarray]: + ''' + Downsample x/y data for lesser curve graphics gen. + + The "peak" method is originally copied verbatim from + ``pyqtgraph.PlotDataItem.getDisplayDataset()``. + + ''' + # py3.10 syntax + match method: + case 'peak': + ds = bins + n = len(x) // ds + x1 = np.empty((n, 2)) + # start of x-values; try to select a somewhat centered point + stx = ds//2 + x1[:] = x[stx:stx+n*ds:ds, np.newaxis] + x = x1.reshape(n*2) + y1 = np.empty((n, 2)) + y2 = y[:n*ds].reshape((n, ds)) + y1[:, 0] = y2.max(axis=1) + y1[:, 1] = y2.min(axis=1) + y = y1.reshape(n*2) + + case '4px': + + # Ex. from infinite on downsampling viewable graphics. + # "one thing i remembered about the binning - if you are + # picking a range within your timeseries the start and end bin + # should be one more bin size outside the visual range, then + # you get better visual fidelity at the edges of the graph" + # "i didn't show it in the sample code, but it's accounted for + # in the start and end indices and number of bins" + + def build_subchart( + self, + subchart, + width, # width of screen? + chart_type, + lower, # x start? + upper, # x end? + xvals, + yvals + ): + pts_per_pixel = len(xvals) / width + if pts_per_pixel > 1: + + # this is mutated in-place + data = np.zeros((width, 4), yvals.dtype) + bins = np.zeros(width, xvals.dtype) + + nb = subset_by_x( + xvals, + yvals, + bins, + data, + lower, + # this is scaling the x-range by + # the width of the screen? + (upper-lower)/float(width), + ) + + return x, y + + +@jit(nopython=True) +def subset_by_x( + + xs: np.ndarray, + ys: np.ndarray, + bins: np.ndarray, + data: np.ndarray, + x_start: int, + step: float, + +) -> int: + count = len(xs) + # nbins = len(bins) + bincount = 0 + x_left = start + # Find the first bin + while xs[0] >= x_left + step: + x_left += step + bins[bincount] = x_left + data[bincount] = ys[0] + + for i in range(count): + x = xs[i] + y = ys[i] + if x < x_left + step: # Interval is [bin, bin+1) + data[bincount, 1] = min(y, data[bincount, 1]) + data[bincount, 2] = max(y, data[bincount, 2]) + data[bincount, 3] = y + else: + # Find the next bin + while x >= x_left + step: + x_left += step + bincount += 1 + bins[bincount] = x_left + data[bincount] = y + + return bincount diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 18a777a8..56ac9385 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -1,5 +1,5 @@ # piker: trading gear for hackers -# Copyright (C) Tyler Goodlet (in stewardship for piker0) +# Copyright (C) Tyler Goodlet (in stewardship for pikers) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -95,38 +95,6 @@ _line_styles: dict[str, int] = { } -def downsample( - x: np.ndarray, - y: np.ndarray, - bins: int, - method: str = 'peak', - -) -> tuple[np.ndarray, np.ndarray]: - ''' - Downsample x/y data for lesser curve graphics gen. - - The "peak" method is originally copied verbatim from - ``pyqtgraph.PlotDataItem.getDisplayDataset()``. - - ''' - match method: - case 'peak': - ds = bins - n = len(x) // ds - x1 = np.empty((n, 2)) - # start of x-values; try to select a somewhat centered point - stx = ds//2 - x1[:] = x[stx:stx+n*ds:ds, np.newaxis] - x = x1.reshape(n*2) - y1 = np.empty((n, 2)) - y2 = y[:n*ds].reshape((n, ds)) - y1[:, 0] = y2.max(axis=1) - y1[:, 1] = y2.min(axis=1) - y = y1.reshape(n*2) - - return x, y - - # TODO: got a feeling that dropping this inheritance gets us even more speedups class FastAppendCurve(pg.PlotCurveItem): ''' @@ -305,8 +273,6 @@ class FastAppendCurve(pg.PlotCurveItem): finiteCheck=False, ) - path = self.path - # other merging ideas: # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths if self._step_mode: From dbe55ad4d2caac410f7371ac7699804d05f076ac Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 9 Mar 2022 15:00:17 -0500 Subject: [PATCH 08/84] Pass linked charts into `BarItems` so that graphics can be cycled on downsample --- piker/ui/_chart.py | 1 + piker/ui/_ohlc.py | 39 +++++++++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 9ae98418..90420236 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -892,6 +892,7 @@ class ChartPlotWidget(pg.PlotWidget): ''' graphics = BarItems( + self.linked, self.plotItem, pen_color=self.pen_color ) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 8ed14080..374acab2 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -17,7 +17,11 @@ Super fast OHLC sampling graphics types. """ -from typing import List, Optional, Tuple +from __future__ import annotations +from typing import ( + Optional, + TYPE_CHECKING, +) import numpy as np import pyqtgraph as pg @@ -31,12 +35,15 @@ from .._profile import pg_profile_enabled from ._style import hcolor from ..log import get_logger +if TYPE_CHECKING: + from ._chart import LinkedSplits + log = get_logger(__name__) def _mk_lines_array( - data: List, + data: list, size: int, elements_step: int = 6, ) -> np.ndarray: @@ -53,7 +60,8 @@ def _mk_lines_array( def lines_from_ohlc( row: np.ndarray, w: float -) -> Tuple[QLineF]: + +) -> tuple[QLineF]: open, high, low, close, index = row[ ['open', 'high', 'low', 'close', 'index']] @@ -88,7 +96,7 @@ def lines_from_ohlc( @njit( # TODO: for now need to construct this manually for readonly arrays, see # https://github.com/numba/numba/issues/4511 - # ntypes.Tuple((float64[:], float64[:], float64[:]))( + # ntypes.tuple((float64[:], float64[:], float64[:]))( # numba_ohlc_dtype[::1], # contiguous # int64, # optional(float64), @@ -186,6 +194,7 @@ class BarItems(pg.GraphicsObject): def __init__( self, # scene: 'QGraphicsScene', # noqa + linked: LinkedSplits, plotitem: 'pg.PlotItem', # noqa pen_color: str = 'bracket', last_bar_color: str = 'bracket', @@ -194,6 +203,7 @@ class BarItems(pg.GraphicsObject): ) -> None: + self.linked = linked super().__init__() # XXX: for the mega-lulz increasing width here increases draw # latency... so probably don't do it until we figure that out. @@ -218,12 +228,12 @@ class BarItems(pg.GraphicsObject): self._pi = plotitem - self._xrange: Tuple[int, int] - self._yrange: Tuple[float, float] + self._xrange: tuple[int, int] + self._yrange: tuple[float, float] # TODO: don't render the full backing array each time # self._path_data = None - self._last_bar_lines: Optional[Tuple[QLineF, ...]] = None + self._last_bar_lines: Optional[tuple[QLineF, ...]] = None # track the current length of drawable lines within the larger array self.start_index: int = 0 @@ -457,24 +467,33 @@ class BarItems(pg.GraphicsObject): not self._in_ds and xs_in_px >= x_gt ): + linked = self.linked # TODO: a `.ui()` log level? - log.info(f'downsampling to line graphic') + log.info(f'downsampling to line graphic {linked.symbol.key}') self._in_ds = True self.hide() self._pi.addItem(self._ds_line) self._ds_line.show() + self._ds_line.update() + linked.graphics_cycle() return True elif ( self._in_ds and xs_in_px < x_gt ): - log.info(f'showing bars graphic') + linked = self.linked + log.info(f'showing bars graphic {linked.symbol.key}') self._in_ds = False self.show() + self.update() self._ds_line.hide() self._pi.removeItem(self._ds_line) - return False + linked.graphics_cycle() + return True + + # no curve change + return False def paint( self, From ab8ea41b93e1a4dcfd0a4205955131783528cf9f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 10 Mar 2022 17:39:40 -0500 Subject: [PATCH 09/84] Add an ohlcv high/low tracer with optional downsampling --- piker/ui/_compression.py | 138 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 129 insertions(+), 9 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index 46969584..7747f249 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -13,16 +13,121 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . + +''' +Graphics related downsampling routines for compressing to pixel +limits on the display device. + +''' +# from typing import Optional + import numpy as np +# from numpy.lib.recfunctions import structured_to_unstructured from numba import ( - jit, float64, optional, int64, + jit, + float64, optional, int64, ) +from ..log import get_logger + + +log = get_logger(__name__) + + +def hl2mxmn( + ohlc: np.ndarray, + downsample_by: int = 0, + +) -> np.ndarray: + ''' + Convert a OHLC struct-array containing 'high'/'low' columns + to a "joined" max/min 1-d array. + + ''' + index = ohlc['index'] + hls = ohlc[[ + 'low', + 'high', + ]] + + # XXX: don't really need this any more since we implemented + # the "tracer" routine, `numba`-style.. + # create a "max and min" sequence from ohlc datums + # hl2d = structured_to_unstructured(hls) + # hl1d = hl2d.flatten() + + mxmn = np.empty(2*hls.size, dtype=np.float64) + x = np.empty(2*hls.size, dtype=np.float64) + trace_hl(hls, mxmn, x, index[0]) + x = x + index[0] - 1 + + if not downsample_by > 2: + return mxmn, x + + dsx, dsy = downsample( + y=mxmn, + x=x, + bins=downsample_by, + ) + log.info(f'downsampling by {downsample_by}') + return dsy, dsx + + +@jit( + # TODO: the type annots.. + # float64[:](float64[:],), + nopython=True, +) +def trace_hl( + hl: 'np.ndarray', + out: np.ndarray, + x: np.ndarray, + start: int, + + # the "offset" values in the x-domain which + # place the 2 output points around each ``int`` + # master index. + margin: float = 0.43, + +) -> None: + ''' + "Trace" the outline of the high-low values of an ohlc sequence + as a line such that the maximum deviation (aka disperaion) between + bars if preserved. + + This routine is expected to modify input arrays in-place. + + ''' + last_l = hl['low'][0] + last_h = hl['high'][0] + + for i in range(hl.size): + row = hl[i] + l, h = row['low'], row['high'] + + up_diff = h - last_l + down_diff = last_h - l + + if up_diff > down_diff: + out[2*i + 1] = h + out[2*i] = last_l + else: + out[2*i + 1] = l + out[2*i] = last_h + + last_l = l + last_h = h + + x[2*i] = int(i) - margin + x[2*i + 1] = int(i) + margin + + return out + def downsample( x: np.ndarray, y: np.ndarray, - bins: int, + bins: int = 2, method: str = 'peak', ) -> tuple[np.ndarray, np.ndarray]: @@ -36,20 +141,31 @@ def downsample( # py3.10 syntax match method: case 'peak': + # breakpoint() + if bins < 2: + log.warning('No downsampling taking place?') + ds = bins n = len(x) // ds x1 = np.empty((n, 2)) + # start of x-values; try to select a somewhat centered point - stx = ds//2 + stx = ds // 2 x1[:] = x[stx:stx+n*ds:ds, np.newaxis] x = x1.reshape(n*2) + y1 = np.empty((n, 2)) y2 = y[:n*ds].reshape((n, ds)) + y1[:, 0] = y2.max(axis=1) y1[:, 1] = y2.min(axis=1) y = y1.reshape(n*2) - case '4px': + return x, y + + # TODO: this algo from infinite, see + # https://github.com/pikers/piker/issues/109 + case 'infinite_4px': # Ex. from infinite on downsampling viewable graphics. # "one thing i remembered about the binning - if you are @@ -62,7 +178,7 @@ def downsample( def build_subchart( self, subchart, - width, # width of screen? + width, # width of screen in pxs? chart_type, lower, # x start? upper, # x end? @@ -86,8 +202,9 @@ def downsample( # the width of the screen? (upper-lower)/float(width), ) + print(f'downsampled to {nb} bins') - return x, y + return x, y @jit(nopython=True) @@ -101,13 +218,16 @@ def subset_by_x( step: float, ) -> int: - count = len(xs) # nbins = len(bins) + count = len(xs) bincount = 0 - x_left = start + x_left = x_start + # Find the first bin - while xs[0] >= x_left + step: + first = xs[0] + while first >= x_left + step: x_left += step + bins[bincount] = x_left data[bincount] = ys[0] From e7dc1a036bf34d39b67c924d9cb35570cb836603 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 11 Mar 2022 13:24:07 -0500 Subject: [PATCH 10/84] Original index offset was right --- piker/ui/_compression.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index 7747f249..d86c882e 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -59,9 +59,9 @@ def hl2mxmn( mxmn = np.empty(2*hls.size, dtype=np.float64) x = np.empty(2*hls.size, dtype=np.float64) trace_hl(hls, mxmn, x, index[0]) - x = x + index[0] - 1 + x = x + index[0] - if not downsample_by > 2: + if downsample_by < 2: return mxmn, x dsx, dsy = downsample( @@ -70,6 +70,7 @@ def hl2mxmn( bins=downsample_by, ) log.info(f'downsampling by {downsample_by}') + print(f'downsampling by {downsample_by}') return dsy, dsx From 7e49b7c033ae5f7e5e48d2271c8458ccf5d4b1e7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 11 Mar 2022 14:40:17 -0500 Subject: [PATCH 11/84] Add for a `BarItems` to display a line on high uppx When a bars graphic is zoomed out enough you get a high uppx, datum units-per-pixel, and there is no point in drawing the 6-lines in each bar element-graphic if you can't see them on the screen/display device. Instead here we offer converting to a `FastAppendCurve` which traces the high-low outline and instead display that when it's impossible to see the details of bars - approximately when the uppx >= 2. There is also some draft-commented code in here for downsampling the outlines as zoom level increases but it's not fully working and should likely be factored out into a higher level api anyway. --- piker/ui/_ohlc.py | 209 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 159 insertions(+), 50 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 374acab2..f060eb6f 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -34,6 +34,8 @@ from PyQt5.QtCore import QLineF, QPointF from .._profile import pg_profile_enabled from ._style import hcolor from ..log import get_logger +from ._curve import FastAppendCurve +from ._compression import hl2mxmn if TYPE_CHECKING: from ._chart import LinkedSplits @@ -46,10 +48,12 @@ def _mk_lines_array( data: list, size: int, elements_step: int = 6, -) -> np.ndarray: - """Create an ndarray to hold lines graphics info. - """ +) -> np.ndarray: + ''' + Create an ndarray to hold lines graphics info. + + ''' return np.zeros_like( data, shape=(int(size), elements_step), @@ -107,10 +111,12 @@ def path_arrays_from_ohlc( data: np.ndarray, start: int64, bar_gap: float64 = 0.43, -) -> np.ndarray: - """Generate an array of lines objects from input ohlc data. - """ +) -> np.ndarray: + ''' + Generate an array of lines objects from input ohlc data. + + ''' size = int(data.shape[0] * 6) x = np.zeros( @@ -220,13 +226,12 @@ class BarItems(pg.GraphicsObject): # that mode? self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - # not sure if this is actually impoving anything but figured it - # was worth a shot: - # self.path.reserve(int(100e3 * 6)) - - self.path = QtGui.QPainterPath() self._pi = plotitem + self.path = QtGui.QPainterPath() + # not sure if this is actually impoving anything but figured it + # was worth a shot: + self.path.reserve(int(100e3 * 6)) self._xrange: tuple[int, int] self._yrange: tuple[float, float] @@ -239,11 +244,15 @@ class BarItems(pg.GraphicsObject): self.start_index: int = 0 self.stop_index: int = 0 + # downsampler-line state self._in_ds: bool = False + self._ds_lines: dict[int, FastAppendCurve] = {} + self._ds_line: Optional[FastAppendCurve] = None + self._ds: int = 0 def draw_from_data( self, - data: np.ndarray, + ohlc: np.ndarray, start: int = 0, ) -> QtGui.QPainterPath: @@ -253,18 +262,18 @@ class BarItems(pg.GraphicsObject): This routine is usually only called to draw the initial history. ''' - hist, last = data[:-1], data[-1] + hist, last = ohlc[:-1], ohlc[-1] self.path = gen_qpath(hist, start, self.w) # save graphics for later reference and keep track # of current internal "last index" - # self.start_index = len(data) - index = data['index'] + # self.start_index = len(ohlc) + index = ohlc['index'] self._xrange = (index[0], index[-1]) self._yrange = ( - np.nanmax(data['high']), - np.nanmin(data['low']), + np.nanmax(ohlc['high']), + np.nanmin(ohlc['low']), ) # up to last to avoid double draw of last bar @@ -274,23 +283,86 @@ class BarItems(pg.GraphicsObject): # https://doc.qt.io/qt-5/qgraphicsitem.html#update self.update() - from ._curve import FastAppendCurve - self._ds_line = FastAppendCurve( - y=data['close'], - x=data['index'], - name='ohlc_ds_line', - color=self._color, - # use_polyline=True, # pretty sure this is slower? - ) - self.update_from_array(data) - self._pi.addItem(self._ds_line) + self.update_ds_line(ohlc) + assert self._ds_line self._ds_line.hide() return self.path + def get_ds_line( + self, + ds: Optional[int] = None, + + ) -> tuple[FastAppendCurve, int]: + + if ds is None: + px_vecs = self.pixelVectors()[0] + if px_vecs: + xs_in_px = px_vecs.x() + ds = round(xs_in_px) + else: + ds = 0 + # print(f'ds is {ds}') + + return self._ds_lines.get(ds), ds + + def update_ds_line( + self, + ohlc: np.ndarray, + use_ds: bool = False, + + ) -> int: + + if not use_ds: + ds = 0 + else: + ds = None + + # determine current potential downsampling value (based on pixel + # scaling) and return any existing curve for it. + curve, ds = self.get_ds_line(ds=ds) + + # curve = self._ds_lines.get(ds) + # if current and current != curve: + # current.hide() + + # if no curve for this downsample rate yet, allowcate a new one + if not curve: + mxmn, x = hl2mxmn(ohlc, downsample_by=ds) + curve = FastAppendCurve( + y=mxmn, + x=x, + name='ohlc_ds_line', + color=self._color, + # color='dad_blue', + # use_polyline=True, # pretty sure this is slower? + ) + # self._pi.addItem(curve) + self._ds_lines[ds] = curve + self._ds_line = curve + + # elif ds != self._ds: + # print(f'ds changed {self._ds} -> {ds}') + + # TODO: we should be diffing the amount of new data which + # needs to be downsampled. Ideally we actually are just + # doing all the ds-ing in sibling actors so that the data + # can just be read and rendered to graphics on events of our + # choice. + # diff = do_diff(ohlc, new_bit) + mxmn, x = hl2mxmn(ohlc, downsample_by=ds) + + curve.update_from_array( + y=mxmn, + x=x, + ) + self._ds = ds + + return curve, ds + def update_from_array( self, - array: np.ndarray, + ohlc: np.ndarray, just_history=False, ) -> None: @@ -309,19 +381,16 @@ class BarItems(pg.GraphicsObject): ''' # XXX: always do this? if self._in_ds: - self._ds_line.update_from_array( - x=array['index'], - y=array['close'], - ) + curve, ds = self.update_ds_line(ohlc) return # index = self.start_index istart, istop = self._xrange - index = array['index'] + index = ohlc['index'] first_index, last_index = index[0], index[-1] - # length = len(array) + # length = len(ohlc) prepend_length = istart - first_index append_length = last_index - istop @@ -333,12 +402,12 @@ class BarItems(pg.GraphicsObject): if prepend_length: # new history was added and we need to render a new path - new_bars = array[:prepend_length] + new_bars = ohlc[:prepend_length] prepend_path = gen_qpath(new_bars, 0, self.w) # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path # y value not matching the first value from - # array[prepend_length + 1] ??? + # ohlc[prepend_length + 1] ??? # update path old_path = self.path @@ -350,14 +419,14 @@ class BarItems(pg.GraphicsObject): if append_length: # generate new lines objects for updatable "current bar" - self._last_bar_lines = lines_from_ohlc(array[-1], self.w) + self._last_bar_lines = lines_from_ohlc(ohlc[-1], self.w) # generate new graphics to match provided array # path appending logic: # we need to get the previous "current bar(s)" for the time step # and convert it to a sub-path to append to the historical set - # new_bars = array[istop - 1:istop + append_length - 1] - new_bars = array[-append_length - 1:-1] + # new_bars = ohlc[istop - 1:istop + append_length - 1] + new_bars = ohlc[-append_length - 1:-1] append_path = gen_qpath(new_bars, 0, self.w) self.path.moveTo(float(istop - self.w), float(new_bars[0]['open'])) self.path.addPath(append_path) @@ -370,7 +439,7 @@ class BarItems(pg.GraphicsObject): self._xrange = first_index, last_index # last bar update - i, o, h, l, last, v = array[-1][ + i, o, h, l, last, v = ohlc[-1][ ['index', 'open', 'high', 'low', 'close', 'volume'] ] # assert i == self.start_index - 1 @@ -462,36 +531,76 @@ class BarItems(pg.GraphicsObject): ''' # this is the ``float`` value of the "number of x units" (in # view coords) that a pixel spans. - xs_in_px = self.pixelVectors()[0].x() + xvec = self.pixelVectors()[0] + if xvec: + xs_in_px = xvec.x() + else: + xs_in_px = self._ds_line.pixelVectors()[0].x() + + linked = self.linked + if ( not self._in_ds and xs_in_px >= x_gt ): - linked = self.linked # TODO: a `.ui()` log level? log.info(f'downsampling to line graphic {linked.symbol.key}') - self._in_ds = True self.hide() - self._pi.addItem(self._ds_line) - self._ds_line.show() - self._ds_line.update() + # XXX: is this actually any faster? + self._pi.removeItem(self) + + curve, ds = self.get_ds_line(ds=0) + last_curve = self._ds_line + assert last_curve is curve + self._pi.addItem(curve) + curve.show() + curve.update() + linked.graphics_cycle() + self._in_ds = True return True elif ( self._in_ds and xs_in_px < x_gt ): - linked = self.linked log.info(f'showing bars graphic {linked.symbol.key}') - self._in_ds = False + + curve, ds = self.get_ds_line(ds=0) + last_curve = self._ds_line + assert last_curve is curve + curve.hide() + self._pi.removeItem(curve) + + # XXX: is this actually any faster? + self._pi.addItem(self) self.show() self.update() - self._ds_line.hide() - self._pi.removeItem(self._ds_line) + + self._in_ds = False linked.graphics_cycle() + return True + # elif ( + # self._in_ds + # and self._ds != ds + # ): + # # curve = self._ds_lines.get(ds) + # # assert self._ds_line is not curve + # if self._ds_line and self._ds_line is not curve: + # self._ds_line.hide() + + # if curve: + # # self._pi.removeItem(curve) + # curve.show() + # curve.update() + + # self._ds_line = curve + # self._ds = ds + # linked.graphics_cycle() + # return True + # no curve change return False From ea5b8f1dd0447b9784599b4a606f25285c3e2324 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 11 Mar 2022 14:49:34 -0500 Subject: [PATCH 12/84] Only trigger downsampling on manual changes, add a uppx method --- piker/ui/_interaction.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 5b0145ff..faf45c4b 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -363,7 +363,6 @@ class ChartView(ViewBox): # defaultPadding=0., **kwargs ) - # for "known y-range style" self._static_yrange = static_yrange self._maxmin = None @@ -430,7 +429,6 @@ class ChartView(ViewBox): def maxmin(self, callback: Callable) -> None: self._maxmin = callback - def maybe_downsample_graphics(self): for graphic in self._chart._graphics.values(): if isinstance(graphic, BarItems): @@ -442,7 +440,8 @@ class ChartView(ViewBox): axis=None, relayed_from: ChartView = None, ): - '''Override "center-point" location for scrolling. + ''' + Override "center-point" location for scrolling. This is an override of the ``ViewBox`` method simply changing the center of the zoom to be the y-axis. @@ -789,7 +788,7 @@ class ChartView(ViewBox): # iterate those. # - only register this when certain downsampleable graphics are # "added to scene". - vb.sigXRangeChanged.connect(vb.maybe_downsample_graphics) + vb.sigRangeChangedManually.connect(vb.maybe_downsample_graphics) # mouse wheel doesn't emit XRangeChanged vb.sigRangeChangedManually.connect(vb._set_yrange) @@ -800,3 +799,19 @@ class ChartView(ViewBox): ) -> None: self._chart._static_yrange = 'axis' + + def xs_in_px(self) -> float: + ''' + Return the "number of x units" within a single + pixel currently being displayed for relevant + graphics items which are our children. + + ''' + for graphic in self._chart._graphics.values(): + # if isinstance(graphic, BarItems): + xpx = graphic.pixelVectors()[0].x() + if xpx: + return xpx + else: + continue + return 1.0 From 09d95157dce68386f2dbf8795a6303eb6a279d8b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 11 Mar 2022 14:49:45 -0500 Subject: [PATCH 13/84] Limit real-time chart updates in "big data" cases - the chart's uppx (units-per-pixel) is > 4 (i.e. zoomed out a lot) - don't shift the chart (to keep the most recent step in view) if the last datum isn't in view (aka the user is probably looking at history) --- piker/ui/_display.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index b761b881..72273a80 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -319,13 +319,13 @@ def graphics_update_cycle( # are diffed on each draw cycle anyway; so updates to the # "curve" length is already automatic. + # compute the first available graphic's x-units-per-pixel + xpx = vlm_chart.view.xs_in_px() + # print(r) + # increment the view position by the sample offset. i_step = ohlcv.index i_diff = i_step - vars['i_last'] - if i_diff > 0: - chart.increment_view( - steps=i_diff, - ) vars['i_last'] = i_step ( @@ -341,11 +341,12 @@ def graphics_update_cycle( liv = r > i_step # the last datum is in view # don't real-time "shift" the curve to the - # left under the following conditions: + # left unless we get one of the following: if ( ( i_diff > 0 # no new sample step - and liv + and xpx < 4 # chart is zoomed out very far + and r >= i_step # the last datum isn't in view ) or trigger_all ): @@ -354,7 +355,11 @@ def graphics_update_cycle( # and then iff update curves and shift? chart.increment_view(steps=i_diff) - if vlm_chart: + if ( + vlm_chart + # if zoomed out alot don't update the last "bar" + and xpx < 4 + ): vlm_chart.update_curve_from_array('volume', array) ds.vlm_sticky.update_from_data(*array[-1][['index', 'volume']]) @@ -426,10 +431,12 @@ def graphics_update_cycle( # for typ, tick in reversed(lasts.items()): # update ohlc sampled price bars - chart.update_ohlc_from_array( - chart.name, - array, - ) + + if xpx < 4 or i_diff > 0: + chart.update_ohlc_from_array( + chart.name, + array, + ) # iterate in FIFO order per frame for typ, tick in lasts.items(): From e7481b1634d73483e9e97116f1efe459526e7bb6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 11 Mar 2022 16:49:58 -0500 Subject: [PATCH 14/84] Array diff lengths must be int --- piker/ui/_curve.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 56ac9385..429f4f0a 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -191,8 +191,8 @@ class FastAppendCurve(pg.PlotCurveItem): # compute the length diffs between the first/last index entry in # the input data and the last indexes we have on record from the # last time we updated the curve index. - prepend_length = istart - x[0] - append_length = x[-1] - istop + prepend_length = int(istart - x[0]) + append_length = int(x[-1] - istop) # step mode: draw flat top discrete "step" # over the index space for each datum. From 39b7c9340d5bcc599a4bc6b14a60de9e554fd36c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 15 Mar 2022 09:06:35 -0400 Subject: [PATCH 15/84] Add (ostensibly) working first attempt at M4 algo All the refs are in the comments and original sample code from infinite has been reworked to expect the input x/y arrays to already be sliced (though we can later support passing in the start-end indexes if desired). The new routines are `ds_m4()` the python top level API and `_m4()` the fast `numba` implementation. --- piker/ui/_compression.py | 203 +++++++++++++++++++++++++-------------- 1 file changed, 131 insertions(+), 72 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index d86c882e..125fa222 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -19,13 +19,13 @@ Graphics related downsampling routines for compressing to pixel limits on the display device. ''' -# from typing import Optional +from typing import Optional import numpy as np # from numpy.lib.recfunctions import structured_to_unstructured from numba import ( jit, - float64, optional, int64, + # float64, optional, int64, ) from ..log import get_logger @@ -36,7 +36,7 @@ log = get_logger(__name__) def hl2mxmn( ohlc: np.ndarray, - downsample_by: int = 0, + # downsample_by: int = 0, ) -> np.ndarray: ''' @@ -61,17 +61,19 @@ def hl2mxmn( trace_hl(hls, mxmn, x, index[0]) x = x + index[0] - if downsample_by < 2: - return mxmn, x + return mxmn, x - dsx, dsy = downsample( - y=mxmn, - x=x, - bins=downsample_by, - ) - log.info(f'downsampling by {downsample_by}') - print(f'downsampling by {downsample_by}') - return dsy, dsx + # if downsample_by < 2: + # return mxmn, x + + # dsx, dsy = downsample( + # y=mxmn, + # x=x, + # bins=downsample_by, + # ) + # log.info(f'downsampling by {downsample_by}') + # print(f'downsampling by {downsample_by}') + # return dsy, dsx @jit( @@ -129,8 +131,11 @@ def downsample( x: np.ndarray, y: np.ndarray, bins: int = 2, + method: str = 'peak', + **kwargs, + ) -> tuple[np.ndarray, np.ndarray]: ''' Downsample x/y data for lesser curve graphics gen. @@ -142,7 +147,6 @@ def downsample( # py3.10 syntax match method: case 'peak': - # breakpoint() if bins < 2: log.warning('No downsampling taking place?') @@ -164,87 +168,142 @@ def downsample( return x, y - # TODO: this algo from infinite, see - # https://github.com/pikers/piker/issues/109 - case 'infinite_4px': - - # Ex. from infinite on downsampling viewable graphics. - # "one thing i remembered about the binning - if you are - # picking a range within your timeseries the start and end bin - # should be one more bin size outside the visual range, then - # you get better visual fidelity at the edges of the graph" - # "i didn't show it in the sample code, but it's accounted for - # in the start and end indices and number of bins" - - def build_subchart( - self, - subchart, - width, # width of screen in pxs? - chart_type, - lower, # x start? - upper, # x end? - xvals, - yvals - ): - pts_per_pixel = len(xvals) / width - if pts_per_pixel > 1: - - # this is mutated in-place - data = np.zeros((width, 4), yvals.dtype) - bins = np.zeros(width, xvals.dtype) - - nb = subset_by_x( - xvals, - yvals, - bins, - data, - lower, - # this is scaling the x-range by - # the width of the screen? - (upper-lower)/float(width), - ) - print(f'downsampled to {nb} bins') - - return x, y + case 'm4': + return ds_m4(x, y, kwargs['px_width']) -@jit(nopython=True) -def subset_by_x( +def ds_m4( + x: np.ndarray, + y: np.ndarray, + + # this is the width of the data in view + # in display-device-local pixel units. + px_width: int, + + factor: Optional[int] = None, + +) -> tuple[np.ndarray, np.ndarray]: + ''' + Downsample using the M4 algorithm. + + ''' + + # NOTE: this method is a so called "visualization driven data + # aggregation" approach. It gives error-free line chart + # downsampling, see + # further scientific paper resources: + # - http://www.vldb.org/pvldb/vol7/p797-jugel.pdf + # - http://www.vldb.org/2014/program/papers/demo/p997-jugel.pdf + + # Details on implementation of this algo are based in, + # https://github.com/pikers/piker/issues/109 + + # XXX: from infinite on downsampling viewable graphics: + # "one thing i remembered about the binning - if you are + # picking a range within your timeseries the start and end bin + # should be one more bin size outside the visual range, then + # you get better visual fidelity at the edges of the graph" + # "i didn't show it in the sample code, but it's accounted for + # in the start and end indices and number of bins" + + assert px_width > 1 # width of screen in pxs? + + # NOTE: if we didn't pre-slice the data to downsample + # you could in theory pass these as the slicing params, + # do we care though since we can always just pre-slice the + # input? + x_start = 0 # x index start + x_end = len(x) # x index end + + # uppx: units-per-pixel + pts_per_pixel = len(x) / px_width + print(f'UPPX: {pts_per_pixel}') + + # ratio of indexed x-value to width of raster in pixels. + if factor is None: + w = (x_end-x_start) / float(px_width) + print(f' pts/pxs = {w}') + else: + w = factor + + # these are pre-allocated and mutated by ``numba`` + # code in-place. + ds = np.zeros((px_width, 4), y.dtype) + i_win = np.zeros(px_width, x.dtype) + + # call into ``numba`` + nb = _m4( + x, + y, + + i_win, + ds, + + # first index in x data to start at + x_start, + # window size for each "frame" of data to downsample (normally + # scaled by the ratio of pixels on screen to data in x-range). + w, + ) + print(f'downsampled to {nb} bins') + + return i_win, ds.flatten() + + +@jit( + nopython=True, +) +def _m4( xs: np.ndarray, ys: np.ndarray, - bins: np.ndarray, - data: np.ndarray, + + # pre-alloc array of x indices mapping to the start + # of each window used for downsampling in y. + i_win: np.ndarray, + + # pre-alloc array of output downsampled y values + ds: np.ndarray, + x_start: int, step: float, ) -> int: - # nbins = len(bins) - count = len(xs) + # nbins = len(i_win) + # count = len(xs) + bincount = 0 x_left = x_start - # Find the first bin + # Find the first window's starting index which *includes* the + # first value in the x-domain array. + # (this allows passing in an array which is indexed (and thus smaller then) + # the ``x_start`` value normally passed in - say if you normally + # want to start 0-indexed. first = xs[0] while first >= x_left + step: x_left += step - bins[bincount] = x_left - data[bincount] = ys[0] + # set all bins in the left-most entry to the starting left-most x value + # (aka a row broadcast). + i_win[bincount] = x_left + # set all y-values to the first value passed in. + ds[bincount] = ys[0] - for i in range(count): + for i in range(len(xs)): x = xs[i] y = ys[i] - if x < x_left + step: # Interval is [bin, bin+1) - data[bincount, 1] = min(y, data[bincount, 1]) - data[bincount, 2] = max(y, data[bincount, 2]) - data[bincount, 3] = y + if x < x_left + step: # the current window "step" is [bin, bin+1) + ds[bincount, 1] = min(y, ds[bincount, 1]) + ds[bincount, 2] = max(y, ds[bincount, 2]) + ds[bincount, 3] = y else: # Find the next bin while x >= x_left + step: x_left += step + bincount += 1 - bins[bincount] = x_left - data[bincount] = y + i_win[bincount] = x_left + ds[bincount] = y return bincount From 4d4f745918e18a93e8047eebc781de95ed58c9e5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 15 Mar 2022 09:11:12 -0400 Subject: [PATCH 16/84] Add `ChartPlotWidget.in_view()` shm-compatible array slicer --- piker/ui/_chart.py | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 90420236..b092de2c 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -376,12 +376,15 @@ class LinkedSplits(QWidget): ''' ln = len(self.subplots) + # proportion allocated to consumer subcharts if not prop: - # proportion allocated to consumer subcharts - if ln < 2: - prop = 1/3 - elif ln >= 2: - prop = 3/8 + prop = 3/8*5/8 + + # if ln < 2: + # prop = 3/8*5/8 + + # elif ln >= 2: + # prop = 3/8 major = 1 - prop min_h_ind = int((self.height() * prop) / ln) @@ -844,7 +847,7 @@ class ChartPlotWidget(pg.PlotWidget): log.warning(f'array for {self.name} not loaded yet?') return - begin = xlast - _bars_to_left_in_follow_mode + begin = xlast - 1000 end = xlast + _bars_from_right_in_follow_mode # remove any custom user yrange setttings @@ -858,6 +861,11 @@ class ChartPlotWidget(pg.PlotWidget): padding=0, ) view._set_yrange() + self.view.maybe_downsample_graphics() + try: + self.linked.graphics_cycle() + except: + pass def increment_view( self, @@ -1003,12 +1011,6 @@ class ChartPlotWidget(pg.PlotWidget): # on data reads and makes graphics rendering no faster # clipToView=True, - # TODO: see how this handles with custom ohlcv bars graphics - # and/or if we can implement something similar for OHLC graphics - # autoDownsample=True, - # downsample=60, - # downsampleMethod='subsample', - **pdi_kwargs, ) @@ -1198,6 +1200,23 @@ class ChartPlotWidget(pg.PlotWidget): else: return ohlc['index'][-1] + def in_view( + self, + array: np.ndarray, + + ) -> np.ndarray: + ''' + Slice an input struct array providing only datums + "in view" of this chart. + + ''' + l, lbar, rbar, r = self.bars_range() + ifirst = array[0]['index'] + # slice data by offset from the first index + # available in the passed datum set. + start = lbar - ifirst + return array[lbar - ifirst:(rbar - ifirst) + 1] + def maxmin( self, name: Optional[str] = None, From d02b1a17ad249f6c18e8b23d11e046888a84e730 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 15 Mar 2022 14:03:44 -0400 Subject: [PATCH 17/84] Fix x-range -> # of frames calculation Obviously determining the x-range from indices was wrong and was the reason for the incorrect (downsampled) output size XD. Instead correctly determine the x range and start value from the *values of* the input x-array. Pretty sure this makes the implementation nearly production ready. Relates to #109 --- piker/ui/_compression.py | 64 ++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index 125fa222..be35244d 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -19,7 +19,7 @@ Graphics related downsampling routines for compressing to pixel limits on the display device. ''' -from typing import Optional +import math import numpy as np # from numpy.lib.recfunctions import structured_to_unstructured @@ -141,7 +141,9 @@ def downsample( Downsample x/y data for lesser curve graphics gen. The "peak" method is originally copied verbatim from - ``pyqtgraph.PlotDataItem.getDisplayDataset()``. + ``pyqtgraph.PlotDataItem.getDisplayDataset()`` which gets + all credit, though we will likely drop this in favor of the M4 + algo below. ''' # py3.10 syntax @@ -180,14 +182,13 @@ def ds_m4( # in display-device-local pixel units. px_width: int, - factor: Optional[int] = None, - -) -> tuple[np.ndarray, np.ndarray]: +) -> tuple[int, np.ndarray, np.ndarray]: ''' Downsample using the M4 algorithm. - ''' + This is more or less an OHLC style sampling of a line-style series. + ''' # NOTE: this method is a so called "visualization driven data # aggregation" approach. It gives error-free line chart # downsampling, see @@ -212,24 +213,34 @@ def ds_m4( # you could in theory pass these as the slicing params, # do we care though since we can always just pre-slice the # input? - x_start = 0 # x index start - x_end = len(x) # x index end + x_start = x[0] # x value start/lowest in domain + x_end = x[-1] # x end value/highest in domain - # uppx: units-per-pixel - pts_per_pixel = len(x) / px_width - print(f'UPPX: {pts_per_pixel}') + # XXX: always round up on the input pixels + px_width = math.ceil(px_width) + + x_range = x_end - x_start # ratio of indexed x-value to width of raster in pixels. - if factor is None: - w = (x_end-x_start) / float(px_width) - print(f' pts/pxs = {w}') - else: - w = factor + # this is more or less, uppx: units-per-pixel. + w = x_range / float(px_width) + + # ensure we make more then enough + # frames (windows) for the output pixel + frames = px_width + + # if we have more and then exact integer's + # (uniform quotient output) worth of datum-domain-points + # per windows-frame, add one more window to ensure + # we have room for all output down-samples. + pts_per_pixel, r = divmod(len(x), px_width) + if r: + frames += 1 # these are pre-allocated and mutated by ``numba`` # code in-place. - ds = np.zeros((px_width, 4), y.dtype) - i_win = np.zeros(px_width, x.dtype) + y_out = np.zeros((frames, 4), y.dtype) + i_win = np.zeros(frames, x.dtype) # call into ``numba`` nb = _m4( @@ -237,7 +248,7 @@ def ds_m4( y, i_win, - ds, + y_out, # first index in x data to start at x_start, @@ -245,9 +256,8 @@ def ds_m4( # scaled by the ratio of pixels on screen to data in x-range). w, ) - print(f'downsampled to {nb} bins') - return i_win, ds.flatten() + return nb, i_win, y_out @jit( @@ -275,13 +285,11 @@ def _m4( bincount = 0 x_left = x_start - # Find the first window's starting index which *includes* the - # first value in the x-domain array. - # (this allows passing in an array which is indexed (and thus smaller then) - # the ``x_start`` value normally passed in - say if you normally - # want to start 0-indexed. - first = xs[0] - while first >= x_left + step: + # Find the first window's starting value which *includes* the + # first value in the x-domain array, i.e. the first + # "left-side-of-window" **plus** the downsampling step, + # creates a window which includes the first x **value**. + while xs[0] >= x_left + step: x_left += step # set all bins in the left-most entry to the starting left-most x value From f1f257d4a23c79297ae8895175fbaf15e8dcede2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 15 Mar 2022 14:13:06 -0400 Subject: [PATCH 18/84] Profiler format, code stretch --- piker/fsp/_engine.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/piker/fsp/_engine.py b/piker/fsp/_engine.py index c9c53d60..0776c7a2 100644 --- a/piker/fsp/_engine.py +++ b/piker/fsp/_engine.py @@ -261,7 +261,10 @@ async def cascade( destination shm array buffer. ''' - profiler = pg.debug.Profiler(delayed=False, disabled=False) + profiler = pg.debug.Profiler( + delayed=False, + disabled=False + ) if loglevel: get_console_log(loglevel) From 8f26335aeaf95da18783b4e524fe5bfe50228470 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 16 Mar 2022 07:24:14 -0400 Subject: [PATCH 19/84] Add display loop profiling Probably the best place to root the profiler since we can get a better top down view of bottlenecks in the graphics stack. More, - add in draft M4 downsampling code (commented) after getting it mostly working; next step is to move this processing into an FSP subactor. - always update the vlm chart last y-axis sticky - set call `.default_view()` just before inf sleep on startup --- piker/ui/_display.py | 147 +++++++++++++++++++++++++++++++------------ 1 file changed, 106 insertions(+), 41 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 72273a80..e64a16d9 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -29,6 +29,8 @@ from typing import Optional, Any, Callable import numpy as np import tractor import trio +import pyqtgraph as pg +from PyQt5.QtCore import QLineF from .. import brokers from ..data.feed import open_feed @@ -178,7 +180,6 @@ async def graphics_update_loop( vlm_sticky = vlm_chart._ysticks['volume'] maxmin = partial(chart_maxmin, chart, vlm_chart) - chart.default_view() last_bars_range: tuple[float, float] ( last_bars_range, @@ -258,6 +259,8 @@ async def graphics_update_loop( } }) + chart.default_view() + # main loop async for quotes in stream: ds.quotes = quotes @@ -295,6 +298,10 @@ def graphics_update_cycle( # TODO: eventually optimize this whole graphics stack with ``numba`` # hopefully XD + profiler = pg.debug.Profiler( + disabled=False, # not pg_profile_enabled(), + delayed=False, + ) # unpack multi-referenced components chart = ds.chart vlm_chart = ds.vlm_chart @@ -305,6 +312,8 @@ def graphics_update_cycle( vars = ds.vars tick_margin = vars['tick_margin'] + update_uppx = 5 + for sym, quote in ds.quotes.items(): # NOTE: vlm may be written by the ``brokerd`` backend @@ -319,10 +328,6 @@ def graphics_update_cycle( # are diffed on each draw cycle anyway; so updates to the # "curve" length is already automatic. - # compute the first available graphic's x-units-per-pixel - xpx = vlm_chart.view.xs_in_px() - # print(r) - # increment the view position by the sample offset. i_step = ohlcv.index i_diff = i_step - vars['i_last'] @@ -338,8 +343,63 @@ def graphics_update_cycle( l, lbar, rbar, r = brange mx = mx_in_view + tick_margin mn = mn_in_view - tick_margin + profiler('maxmin call') liv = r > i_step # the last datum is in view + # compute the first available graphic's x-units-per-pixel + xpx = vlm_chart.view.xs_in_px() + + in_view = chart.in_view(ohlcv.array) + + if lbar != rbar: + # view box width in pxs + w = chart.view.boundingRect().width() + + # TODO: a better way to get this? + # i would guess the esiest way is to just + # get the ``.boundingRect()`` of the curve + # in view but maybe there's something smarter? + # Currently we're just mapping the rbar, lbar to + # pixels via: + cw = chart.view.mapViewToDevice(QLineF(lbar, 0, rbar, 0)).length() + # is this faster? + # cw = chart.mapFromView(QLineF(lbar, 0 , rbar, 0)).length() + + profiler( + f'view width pxs: {w}\n' + f'curve width pxs: {cw}\n' + f'sliced in view: {in_view.size}' + ) + + # compress bars to m4 line(s) if uppx is high enough + # if in_view.size > cw: + # from ._compression import ds_m4, hl2mxmn + + # mxmn, x = hl2mxmn(in_view) + # profiler('hl tracer') + + # nb, x, y = ds_m4( + # x=x, + # y=mxmn, + # # TODO: this needs to actually be the width + # # in pixels of the visible curve since we don't + # # want to downsample any 'zeros' around the curve, + # # just the values that make up the curve graphic, + # # i think? + # px_width=cw, + # ) + # profiler( + # 'm4 downsampled\n' + # f' ds bins: {nb}\n' + # f' x.shape: {x.shape}\n' + # f' y.shape: {y.shape}\n' + # f' x: {x}\n' + # f' y: {y}\n' + # ) + # breakpoint() + + # assert y.size == mxmn.size + # don't real-time "shift" the curve to the # left unless we get one of the following: if ( @@ -355,36 +415,37 @@ def graphics_update_cycle( # and then iff update curves and shift? chart.increment_view(steps=i_diff) - if ( - vlm_chart - # if zoomed out alot don't update the last "bar" - and xpx < 4 - ): - vlm_chart.update_curve_from_array('volume', array) + if vlm_chart: + # always update y-label ds.vlm_sticky.update_from_data(*array[-1][['index', 'volume']]) if ( - mx_vlm_in_view > vars['last_mx_vlm'] + (xpx < update_uppx or i_diff > 0) or trigger_all ): - # print(f'mx vlm: {last_mx_vlm} -> {mx_vlm_in_view}') - vlm_chart.view._set_yrange( - yrange=(0, mx_vlm_in_view * 1.375) - ) - vars['last_mx_vlm'] = mx_vlm_in_view + vlm_chart.update_curve_from_array('volume', array) - for curve_name, flow in vlm_chart._flows.items(): - update_fsp_chart( - vlm_chart, - flow.shm, - curve_name, - array_key=curve_name, - ) - # is this even doing anything? - flow.plot.vb._set_yrange( - autoscale_linked_plots=False, - name=curve_name, - ) + if ( + mx_vlm_in_view != vars['last_mx_vlm'] + ): + # print(f'mx vlm: {last_mx_vlm} -> {mx_vlm_in_view}') + vlm_chart.view._set_yrange( + yrange=(0, mx_vlm_in_view * 1.375) + ) + vars['last_mx_vlm'] = mx_vlm_in_view + + for curve_name, flow in vlm_chart._flows.items(): + update_fsp_chart( + vlm_chart, + flow.shm, + curve_name, + array_key=curve_name, + ) + # is this even doing anything? + flow.plot.vb._set_yrange( + autoscale_linked_plots=False, + name=curve_name, + ) ticks_frame = quote.get('ticks', ()) @@ -431,8 +492,10 @@ def graphics_update_cycle( # for typ, tick in reversed(lasts.items()): # update ohlc sampled price bars - - if xpx < 4 or i_diff > 0: + if ( + xpx < update_uppx + or i_diff > 0 + ): chart.update_ohlc_from_array( chart.name, array, @@ -600,8 +663,8 @@ async def display_symbol_data( f'step:1s ' ) - linkedsplits = godwidget.linkedsplits - linkedsplits._symbol = symbol + linked = godwidget.linkedsplits + linked._symbol = symbol # generate order mode side-pane UI # A ``FieldsForm`` form to configure order entry @@ -611,7 +674,7 @@ async def display_symbol_data( godwidget.pp_pane = pp_pane # create main OHLC chart - chart = linkedsplits.plot_ohlc_main( + chart = linked.plot_ohlc_main( symbol, bars, sidepane=pp_pane, @@ -641,8 +704,8 @@ async def display_symbol_data( # NOTE: we must immediately tell Qt to show the OHLC chart # to avoid a race where the subplots get added/shown to # the linked set *before* the main price chart! - linkedsplits.show() - linkedsplits.focus() + linked.show() + linked.focus() await trio.sleep(0) vlm_chart: Optional[ChartPlotWidget] = None @@ -652,7 +715,7 @@ async def display_symbol_data( if has_vlm(ohlcv): vlm_chart = await ln.start( open_vlm_displays, - linkedsplits, + linked, ohlcv, ) @@ -660,7 +723,7 @@ async def display_symbol_data( # from an input config. ln.start_soon( start_fsp_displays, - linkedsplits, + linked, ohlcv, loading_sym_key, loglevel, @@ -669,7 +732,7 @@ async def display_symbol_data( # start graphics update loop after receiving first live quote ln.start_soon( graphics_update_loop, - linkedsplits, + linked, feed.stream, ohlcv, wap_in_history, @@ -687,17 +750,19 @@ async def display_symbol_data( # let Qt run to render all widgets and make sure the # sidepanes line up vertically. await trio.sleep(0) - linkedsplits.resize_sidepanes() + linked.resize_sidepanes() # NOTE: we pop the volume chart from the subplots set so # that it isn't double rendered in the display loop # above since we do a maxmin calc on the volume data to # determine if auto-range adjustements should be made. - linkedsplits.subplots.pop('volume', None) + linked.subplots.pop('volume', None) # TODO: make this not so shit XD # close group status sbar._status_groups[loading_sym_key][1]() # let the app run.. bby + chart.default_view() + # linked.graphics_cycle() await trio.sleep_forever() From 03a08b5f63fe52d2a089eccba9e6e0b59a2dbf36 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 16 Mar 2022 07:28:03 -0400 Subject: [PATCH 20/84] Add curve px width getter `ChartPlotWidget.curve_width_pxs()` now can be used to get the total horizontal (x) pixels on screen that are occupied by the current curve graphics for a given chart. This will be used for downsampling large data sets to the pixel domain using M4. --- piker/ui/_chart.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index b092de2c..553583d4 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1,5 +1,5 @@ # piker: trading gear for hackers -# Copyright (C) Tyler Goodlet (in stewardship for piker0) +# Copyright (C) Tyler Goodlet (in stewardship for pikers) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -22,7 +22,11 @@ from __future__ import annotations from typing import Optional, TYPE_CHECKING from PyQt5 import QtCore, QtWidgets -from PyQt5.QtCore import Qt +from PyQt5.QtCore import ( + Qt, + QLineF, + QPointF, +) from PyQt5.QtWidgets import ( QFrame, QWidget, @@ -824,14 +828,24 @@ class ChartPlotWidget(pg.PlotWidget): return int(vr.left()), int(vr.right()) def bars_range(self) -> tuple[int, int, int, int]: - """Return a range tuple for the bars present in view. - """ + ''' + Return a range tuple for the bars present in view. + + ''' l, r = self.view_range() array = self._arrays[self.name] lbar = max(l, array[0]['index']) rbar = min(r, array[-1]['index']) return l, lbar, rbar, r + def curve_width_pxs( + self, + ) -> float: + _, lbar, rbar, _ = self.bars_range() + return self.view.mapViewToDevice( + QLineF(lbar, 0, rbar, 0) + ).length() + def default_view( self, index: int = -1, @@ -864,7 +878,7 @@ class ChartPlotWidget(pg.PlotWidget): self.view.maybe_downsample_graphics() try: self.linked.graphics_cycle() - except: + except IndexError: pass def increment_view( From bedb55b79deaaba5b8b4391320a64c107136ab1c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 16 Mar 2022 09:54:52 -0400 Subject: [PATCH 21/84] Use service cancel method for graceful teardown --- piker/_daemon.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/piker/_daemon.py b/piker/_daemon.py index b4eed03d..053d4864 100644 --- a/piker/_daemon.py +++ b/piker/_daemon.py @@ -35,10 +35,10 @@ log = get_logger(__name__) _root_dname = 'pikerd' -_registry_addr = ('127.0.0.1', 6116) +_registry_addr = ('127.0.0.1', 1616) _tractor_kwargs: dict[str, Any] = { # use a different registry addr then tractor's default - 'arbiter_addr': _registry_addr + 'arbiter_addr': _registry_addr } _root_modules = [ __name__, @@ -91,14 +91,18 @@ class Services(BaseModel): log.info( f'`pikerd` service {name} started with value {first}' ) - # wait on any context's return value - ctx_res = await ctx.result() - - # wait on any error from the sub-actor - # NOTE: this will block indefinitely until cancelled - # either by error from the target context function or by - # being cancelled here by the surrounding cancel scope - return (await portal.result(), ctx_res) + try: + # wait on any context's return value + ctx_res = await ctx.result() + except tractor.ContextCancelled: + return await self.cancel_service(name) + else: + # wait on any error from the sub-actor + # NOTE: this will block indefinitely until + # cancelled either by error from the target + # context function or by being cancelled here by + # the surrounding cancel scope + return (await portal.result(), ctx_res) cs, first = await self.service_n.start(open_context_in_task) @@ -110,14 +114,17 @@ class Services(BaseModel): # TODO: per service cancellation by scope, we aren't using this # anywhere right? - # async def cancel_service( - # self, - # name: str, - # ) -> Any: - # log.info(f'Cancelling `pikerd` service {name}') - # cs, portal = self.service_tasks[name] - # cs.cancel() - # return await portal.cancel_actor() + async def cancel_service( + self, + name: str, + ) -> Any: + log.info(f'Cancelling `pikerd` service {name}') + cs, portal = self.service_tasks[name] + # XXX: not entirely sure why this is required, + # and should probably be better fine tuned in + # ``tractor``? + cs.cancel() + return await portal.cancel_actor() _services: Optional[Services] = None @@ -372,6 +379,7 @@ async def maybe_spawn_daemon( async with tractor.wait_for_actor(service_name) as portal: lock.release() yield portal + await portal.cancel_actor() async def spawn_brokerd( From 1ad83e4556387032cc5e8fe0c21d5d43a71ab13d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 17 Mar 2022 09:00:59 -0400 Subject: [PATCH 22/84] WIP add non-working m4 ds code to ohlc graphic --- piker/ui/_ohlc.py | 52 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index f060eb6f..de914862 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -25,6 +25,7 @@ from typing import ( import numpy as np import pyqtgraph as pg +from numpy.lib import recfunctions as rfn from numba import njit, float64, int64 # , optional from PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtCore import QLineF, QPointF @@ -35,7 +36,7 @@ from .._profile import pg_profile_enabled from ._style import hcolor from ..log import get_logger from ._curve import FastAppendCurve -from ._compression import hl2mxmn +from ._compression import hl2mxmn, ds_m4 if TYPE_CHECKING: from ._chart import LinkedSplits @@ -226,7 +227,6 @@ class BarItems(pg.GraphicsObject): # that mode? self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - self._pi = plotitem self.path = QtGui.QPainterPath() # not sure if this is actually impoving anything but figured it @@ -320,24 +320,47 @@ class BarItems(pg.GraphicsObject): # determine current potential downsampling value (based on pixel # scaling) and return any existing curve for it. - curve, ds = self.get_ds_line(ds=ds) + curve, ds = self.get_ds_line(ds=0) # curve = self._ds_lines.get(ds) # if current and current != curve: # current.hide() # if no curve for this downsample rate yet, allowcate a new one + + # if not self.linked.chart: + # return None, None + + index = ohlc['index'] + flat = rfn.structured_to_unstructured( + ohlc[['open', 'high', 'low', 'close']] + ).flatten() + xpts = np.linspace(start=index[0] - 0.5, stop=index[-1] + 0.5, num=4*len(ohlc)) + + # bins, x, ds = ds_m4( + # xpts, + # flat, + # # px_width=self.linked.chart.curve_width_pxs() + # px_width=self.getViewBox().width(), + # ) + # breakpoint() + + mxmn, x = hl2mxmn(ohlc) + + # if self._ds_line: + # self._pi.removeItem(self._ds_line) + if not curve: - mxmn, x = hl2mxmn(ohlc, downsample_by=ds) curve = FastAppendCurve( + # y=ds.flatten(), y=mxmn, x=x, - name='ohlc_ds_line', + name='ds', color=self._color, # color='dad_blue', # use_polyline=True, # pretty sure this is slower? ) - # self._pi.addItem(curve) + self._pi.addItem(curve) self._ds_lines[ds] = curve self._ds_line = curve @@ -350,12 +373,24 @@ class BarItems(pg.GraphicsObject): # can just be read and rendered to graphics on events of our # choice. # diff = do_diff(ohlc, new_bit) - mxmn, x = hl2mxmn(ohlc, downsample_by=ds) + # mxmn, x = hl2mxmn(ohlc, downsample_by=ds) + # always refresh data bounds until we get diffing + # working properly, see below.. + # curve._xrange = curve.dataBounds(ax=0) + + # TODO: we need to do a diff here to determine + # which ohlc samples have not yet been converted + # to tracer lines. + # index = ohlc['index'] + # istart, istop = curve._xrange + # curve.path = None + # print(x[-10:]) curve.update_from_array( y=mxmn, x=x, ) + # curve.update() self._ds = ds return curve, ds @@ -380,9 +415,8 @@ class BarItems(pg.GraphicsObject): ''' # XXX: always do this? - if self._in_ds: + if self._ds_line: curve, ds = self.update_ds_line(ohlc) - return # index = self.start_index istart, istop = self._xrange From 772f8712721488384e9f325a03d26ff3295b4fcd Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 18 Mar 2022 10:59:05 -0400 Subject: [PATCH 23/84] Use units by default for continuous futes --- piker/clearing/_allocate.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/piker/clearing/_allocate.py b/piker/clearing/_allocate.py index 6ef38692..7868666d 100644 --- a/piker/clearing/_allocate.py +++ b/piker/clearing/_allocate.py @@ -178,7 +178,9 @@ class Allocator(BaseModel): l_sub_pp = (self.currency_limit - live_cost_basis) / price else: - raise ValueError(f"Not valid size unit '{size}'") + raise ValueError( + f"Not valid size unit '{size_unit}'" + ) # an entry (adding-to or starting a pp) if ( @@ -290,7 +292,7 @@ def mk_allocator( # default allocation settings defaults: dict[str, float] = { 'account': None, # select paper by default - 'size_unit': 'currency', #_size_units['currency'], + 'size_unit': 'currency', 'units_limit': 400, 'currency_limit': 5e3, 'slots': 4, @@ -318,11 +320,14 @@ def mk_allocator( asset_type = symbol.type_key - # specific configs by asset class / type - if asset_type in ('future', 'option', 'futures_option'): - + if asset_type in ( + 'future', + 'continuous_future', + 'option', + 'futures_option', + ): # since it's harder to know how currency "applies" in this case # given leverage properties alloc.size_unit = '# units' From c4242acc21f0eb71c7b2bd2ecb58252f88255c2a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 18 Mar 2022 15:07:48 -0400 Subject: [PATCH 24/84] Pass in fqsn from chart UI components --- piker/ui/_display.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index e64a16d9..70c4b49d 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -299,7 +299,7 @@ def graphics_update_cycle( # hopefully XD profiler = pg.debug.Profiler( - disabled=False, # not pg_profile_enabled(), + disabled=True, # not pg_profile_enabled(), delayed=False, ) # unpack multi-referenced components @@ -396,7 +396,6 @@ def graphics_update_cycle( # f' x: {x}\n' # f' y: {y}\n' # ) - # breakpoint() # assert y.size == mxmn.size From 56c163cdd786ace50082b99c1af829618cea1350 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 20 Mar 2022 12:53:44 -0400 Subject: [PATCH 25/84] Make `ChartPlotWidget.default_view()` pin to L1 Instead of using a guess about how many x-indexes to reset the last datum in-view to, calculate and shift the latest index such that it's just before any L1 spread labels on the y-axis. This makes the view placement "widget aware" and gives a much more cross-display UX. Summary: - add `ChartPlotWidget.pre_l1_x()` which returns a `tuple` of x view-coord points for the absolute x-pos and length of any L1 line/labels - make `.default_view()` only shift to see the xlast just outside the l1 but keep whatever view range xfirst as the first datum in view - drop `LevelLine.right_point()` since this is now just a `.pre_l1_x()` call and can be retrieved from the line's internal chart ref - drop `._style.bars_from/to_..` vars since we aren't using hard coded offsets any more --- piker/ui/_anchors.py | 1 - piker/ui/_chart.py | 52 ++++++++++++++++++++++++++++++++++---------- piker/ui/_lines.py | 40 ++++++++++++---------------------- piker/ui/_style.py | 7 +++--- 4 files changed, 58 insertions(+), 42 deletions(-) diff --git a/piker/ui/_anchors.py b/piker/ui/_anchors.py index e7b6d72b..2d4345e2 100644 --- a/piker/ui/_anchors.py +++ b/piker/ui/_anchors.py @@ -25,7 +25,6 @@ from PyQt5.QtCore import QPointF from PyQt5.QtWidgets import QGraphicsPathItem if TYPE_CHECKING: - from ._axes import PriceAxis from ._chart import ChartPlotWidget from ._label import Label diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 553583d4..705e3c84 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -25,7 +25,7 @@ from PyQt5 import QtCore, QtWidgets from PyQt5.QtCore import ( Qt, QLineF, - QPointF, + # QPointF, ) from PyQt5.QtWidgets import ( QFrame, @@ -56,8 +56,6 @@ from ._style import ( CHART_MARGINS, _xaxis_at, _min_points_to_show, - _bars_from_right_in_follow_mode, - _bars_to_left_in_follow_mode, ) from ..data.feed import Feed from ..data._source import Symbol @@ -846,29 +844,62 @@ class ChartPlotWidget(pg.PlotWidget): QLineF(lbar, 0, rbar, 0) ).length() - def default_view( + def pre_l1_x( self, - index: int = -1, + view_coords: bool = False, - ) -> None: + ) -> tuple[float, float]: + ''' + Return the scene x-coord for the value just before + the L1 labels on the y-axis as well as the length + of that L1 label from the y-axis. + + ''' + l1_len = self._max_l1_line_len + ryaxis = self.getAxis('right') + ryaxis_x = ryaxis.pos().x() + up_to_l1_sc = ryaxis_x - l1_len + if not view_coords: + return up_to_l1_sc, l1_len + else: + view = self.view + line = view.mapToView( + QLineF(up_to_l1_sc, 0, ryaxis_x, 0) + ) + return line.x1(), line.length() + + def default_view(self) -> None: ''' Set the view box to the "default" startup view of the scene. ''' try: - xlast = self._arrays[self.name][index]['index'] + index = self._arrays[self.name]['index'] except IndexError: log.warning(f'array for {self.name} not loaded yet?') return - begin = xlast - 1000 - end = xlast + _bars_from_right_in_follow_mode + xfirst, xlast = index[0], index[-1] + view = self.view + vr = view.viewRange() + marker_pos, l1_len = self.pre_l1_x(view_coords=True) + end = xlast + l1_len + xl = vr[0][0] + begin = max(xl, xfirst) + + # print( + # f'view range: {vr}\n' + # f'xlast: {xlast}\n' + # f'marker pos: {marker_pos}\n' + # f'l1 len: {l1_len}\n' + # f'begin: {begin}\n' + # f'end: {end}\n' + # ) # remove any custom user yrange setttings if self._static_yrange == 'axis': self._static_yrange = None - view = self.view view.setXRange( min=begin, max=end, @@ -1228,7 +1259,6 @@ class ChartPlotWidget(pg.PlotWidget): ifirst = array[0]['index'] # slice data by offset from the first index # available in the passed datum set. - start = lbar - ifirst return array[lbar - ifirst:(rbar - ifirst) + 1] def maxmin( diff --git a/piker/ui/_lines.py b/piker/ui/_lines.py index d148049e..df29e350 100644 --- a/piker/ui/_lines.py +++ b/piker/ui/_lines.py @@ -20,7 +20,7 @@ Lines for orders, alerts, L2. """ from functools import partial from math import floor -from typing import Tuple, Optional, List, Callable +from typing import Optional, Callable import pyqtgraph as pg from pyqtgraph import Point, functions as fn @@ -32,7 +32,6 @@ from ._anchors import ( marker_right_points, vbr_left, right_axis, - # pp_tight_and_right, # wanna keep it straight in the long run gpath_pin, ) from ..calc import humanize @@ -104,8 +103,8 @@ class LevelLine(pg.InfiniteLine): # list of labels anchored at one of the 2 line endpoints # inside the viewbox - self._labels: List[Label] = [] - self._markers: List[(int, Label)] = [] + self._labels: list[Label] = [] + self._markers: list[(int, Label)] = [] # whenever this line is moved trigger label updates self.sigPositionChanged.connect(self.on_pos_change) @@ -124,7 +123,7 @@ class LevelLine(pg.InfiniteLine): self._y_incr_mult = 1 / chart.linked.symbol.tick_size self._right_end_sc: float = 0 - def txt_offsets(self) -> Tuple[int, int]: + def txt_offsets(self) -> tuple[int, int]: return 0, 0 @property @@ -315,17 +314,6 @@ class LevelLine(pg.InfiniteLine): # TODO: enter labels edit mode print(f'double click {ev}') - def right_point( - self, - ) -> float: - - chart = self._chart - l1_len = chart._max_l1_line_len - ryaxis = chart.getAxis('right') - up_to_l1_sc = ryaxis.pos().x() - l1_len - - return up_to_l1_sc - def paint( self, @@ -422,23 +410,23 @@ class LevelLine(pg.InfiniteLine): ) -> QtWidgets.QGraphicsPathItem: + self._marker = path + self._marker.setPen(self.currentPen) + self._marker.setBrush(fn.mkBrush(self.currentPen.color())) # add path to scene self.getViewBox().scene().addItem(path) - self._marker = path - - rsc = self.right_point() - - self._marker.setPen(self.currentPen) - self._marker.setBrush(fn.mkBrush(self.currentPen.color())) + # place to just-left of L1 labels + rsc = self._chart.pre_l1_x()[0] path.setPos(QPointF(rsc, self.scene_y())) return path def hoverEvent(self, ev): - """Mouse hover callback. + ''' + Mouse hover callback. - """ + ''' cur = self._chart.linked.cursor # hovered @@ -614,7 +602,8 @@ def order_line( **line_kwargs, ) -> LevelLine: - '''Convenience routine to add a line graphic representing an order + ''' + Convenience routine to add a line graphic representing an order execution submitted to the EMS via the chart's "order mode". ''' @@ -689,7 +678,6 @@ def order_line( return f'{account}: ' - label.fields = { 'size': size, 'size_digits': 0, diff --git a/piker/ui/_style.py b/piker/ui/_style.py index c60cb077..22731093 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -14,9 +14,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" +''' Qt UI styling. -""" + +''' from typing import Optional, Dict import math @@ -202,8 +203,6 @@ _xaxis_at = 'bottom' # charting config CHART_MARGINS = (0, 0, 2, 2) _min_points_to_show = 6 -_bars_to_left_in_follow_mode = int(61*6) -_bars_from_right_in_follow_mode = round(0.16 * _bars_to_left_in_follow_mode) _tina_mode = False From 1a0e89d07e4b314f1ce5fec832566a3dfd9c4975 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 21 Mar 2022 09:20:54 -0400 Subject: [PATCH 26/84] Even more correct "default view" snap-to-pp-marker This makes the `'r'` hotkey snap the last bar to the middle of the pp line arrow marker no matter the zoom level. Now we also boot with approximately the most number of x units on screen that keep the bars graphics drawn in full (just before downsampling to a line). Moved some internals around to get this all in place, - drop `_anchors.marker_right_points()` and move it to a chart method. - change `.pre_l1_x()` -> `.pre_l1_xs()` and just have it return the two view-mapped x values from the former method. --- piker/ui/_anchors.py | 28 --------------- piker/ui/_annotate.py | 6 ++-- piker/ui/_chart.py | 80 ++++++++++++++++++++++++++++++------------- piker/ui/_lines.py | 7 ++-- 4 files changed, 61 insertions(+), 60 deletions(-) diff --git a/piker/ui/_anchors.py b/piker/ui/_anchors.py index 2d4345e2..5d8217c8 100644 --- a/piker/ui/_anchors.py +++ b/piker/ui/_anchors.py @@ -29,34 +29,6 @@ if TYPE_CHECKING: from ._label import Label -def marker_right_points( - chart: ChartPlotWidget, # noqa - marker_size: int = 20, - -) -> (float, float, float): - ''' - Return x-dimension, y-axis-aware, level-line marker oriented scene - values. - - X values correspond to set the end of a level line, end of - a paried level line marker, and the right most side of the "right" - axis respectively. - - ''' - # TODO: compute some sensible maximum value here - # and use a humanized scheme to limit to that length. - l1_len = chart._max_l1_line_len - ryaxis = chart.getAxis('right') - - r_axis_x = ryaxis.pos().x() - up_to_l1_sc = r_axis_x - l1_len - 10 - - marker_right = up_to_l1_sc - (1.375 * 2 * marker_size) - line_end = marker_right - (6/16 * marker_size) - - return line_end, marker_right, r_axis_x - - def vbr_left( label: Label, diff --git a/piker/ui/_annotate.py b/piker/ui/_annotate.py index 6af8ffe7..6e0e84d1 100644 --- a/piker/ui/_annotate.py +++ b/piker/ui/_annotate.py @@ -26,8 +26,6 @@ from PyQt5.QtWidgets import QGraphicsPathItem from pyqtgraph import Point, functions as fn, Color import numpy as np -from ._anchors import marker_right_points - def mk_marker_path( @@ -116,7 +114,7 @@ class LevelMarker(QGraphicsPathItem): self.get_level = get_level self._on_paint = on_paint - self.scene_x = lambda: marker_right_points(chart)[1] + self.scene_x = lambda: chart.marker_right_points()[1] self.level: float = 0 self.keep_in_view = keep_in_view @@ -169,7 +167,7 @@ class LevelMarker(QGraphicsPathItem): vr = view.state['viewRange'] ymn, ymx = vr[1] - # _, marker_right, _ = marker_right_points(line._chart) + # _, marker_right, _ = line._chart.marker_right_points() x = self.scene_x() if self.style == '>|': # short style, points "down-to" line diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 705e3c84..b208cece 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -844,31 +844,52 @@ class ChartPlotWidget(pg.PlotWidget): QLineF(lbar, 0, rbar, 0) ).length() - def pre_l1_x( - self, - view_coords: bool = False, - - ) -> tuple[float, float]: + def pre_l1_xs(self) -> tuple[float, float]: ''' - Return the scene x-coord for the value just before + Return the view x-coord for the value just before the L1 labels on the y-axis as well as the length of that L1 label from the y-axis. ''' + line_end, marker_right, yaxis_x = self.marker_right_points() + view = self.view + line = view.mapToView( + QLineF(line_end, 0, yaxis_x, 0) + ) + return line.x1(), line.length() + + def marker_right_points( + self, + marker_size: int = 20, + + ) -> (float, float, float): + ''' + Return x-dimension, y-axis-aware, level-line marker oriented scene + values. + + X values correspond to set the end of a level line, end of + a paried level line marker, and the right most side of the "right" + axis respectively. + + ''' + # TODO: compute some sensible maximum value here + # and use a humanized scheme to limit to that length. l1_len = self._max_l1_line_len ryaxis = self.getAxis('right') - ryaxis_x = ryaxis.pos().x() - up_to_l1_sc = ryaxis_x - l1_len - if not view_coords: - return up_to_l1_sc, l1_len - else: - view = self.view - line = view.mapToView( - QLineF(up_to_l1_sc, 0, ryaxis_x, 0) - ) - return line.x1(), line.length() - def default_view(self) -> None: + r_axis_x = ryaxis.pos().x() + up_to_l1_sc = r_axis_x - l1_len - 10 + + marker_right = up_to_l1_sc - (1.375 * 2 * marker_size) + line_end = marker_right - (6/16 * marker_size) + + return line_end, marker_right, r_axis_x + + def default_view( + self, + steps_on_screen: Optional[int] = None + + ) -> None: ''' Set the view box to the "default" startup view of the scene. @@ -880,15 +901,25 @@ class ChartPlotWidget(pg.PlotWidget): return xfirst, xlast = index[0], index[-1] - view = self.view - vr = view.viewRange() - marker_pos, l1_len = self.pre_l1_x(view_coords=True) - end = xlast + l1_len - xl = vr[0][0] - begin = max(xl, xfirst) + brange = l, lbar, rbar, r = self.bars_range() + marker_pos, l1_len = self.pre_l1_xs() + end = xlast + l1_len + + if ( + rbar < 0 + or l < xfirst + ): + # set fixed bars count on screen that approx includes as + # many bars as possible before a downsample line is shown. + begin = xlast - round(6116 / 6) + + else: + begin = end - (r - l) + + # for debugging # print( - # f'view range: {vr}\n' + # f'bars range: {brange}\n' # f'xlast: {xlast}\n' # f'marker pos: {marker_pos}\n' # f'l1 len: {l1_len}\n' @@ -900,6 +931,7 @@ class ChartPlotWidget(pg.PlotWidget): if self._static_yrange == 'axis': self._static_yrange = None + view = self.view view.setXRange( min=begin, max=end, diff --git a/piker/ui/_lines.py b/piker/ui/_lines.py index df29e350..421d4ec8 100644 --- a/piker/ui/_lines.py +++ b/piker/ui/_lines.py @@ -29,7 +29,6 @@ from PyQt5.QtCore import QPointF from ._annotate import qgo_draw_markers, LevelMarker from ._anchors import ( - marker_right_points, vbr_left, right_axis, gpath_pin, @@ -333,7 +332,7 @@ class LevelLine(pg.InfiniteLine): vb_left, vb_right = self._endPoints vb = self.getViewBox() - line_end, marker_right, r_axis_x = marker_right_points(self._chart) + line_end, marker_right, r_axis_x = self._chart.marker_right_points() if self.show_markers and self.markers: @@ -399,7 +398,7 @@ class LevelLine(pg.InfiniteLine): def scene_endpoint(self) -> QPointF: if not self._right_end_sc: - line_end, _, _ = marker_right_points(self._chart) + line_end, _, _ = self._chart.marker_right_points() self._right_end_sc = line_end - 10 return QPointF(self._right_end_sc, self.scene_y()) @@ -417,7 +416,7 @@ class LevelLine(pg.InfiniteLine): self.getViewBox().scene().addItem(path) # place to just-left of L1 labels - rsc = self._chart.pre_l1_x()[0] + rsc = self._chart.pre_l1_xs()[0] path.setPos(QPointF(rsc, self.scene_y())) return path From ed03d77e6e02594716dc38a84bd032e9182a2e42 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 21 Mar 2022 15:25:45 -0400 Subject: [PATCH 27/84] Make a derivs intrustment type table for alloc config checks --- piker/clearing/_allocate.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/piker/clearing/_allocate.py b/piker/clearing/_allocate.py index 7868666d..71d7d9a0 100644 --- a/piker/clearing/_allocate.py +++ b/piker/clearing/_allocate.py @@ -284,6 +284,14 @@ class Allocator(BaseModel): return round(prop * self.slots) +_derivs = ( + 'future', + 'continuous_future', + 'option', + 'futures_option', +) + + def mk_allocator( symbol: Symbol, @@ -322,12 +330,7 @@ def mk_allocator( # specific configs by asset class / type - if asset_type in ( - 'future', - 'continuous_future', - 'option', - 'futures_option', - ): + if asset_type in _derivs: # since it's harder to know how currency "applies" in this case # given leverage properties alloc.size_unit = '# units' @@ -350,7 +353,7 @@ def mk_allocator( if startup_size > alloc.units_limit: alloc.units_limit = startup_size - if asset_type in ('future', 'option', 'futures_option'): + if asset_type in _derivs: alloc.slots = alloc.units_limit return alloc From ff009934120b44015bd6fcc426e2b3e31a69cf6a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 21 Mar 2022 15:27:46 -0400 Subject: [PATCH 28/84] Call default view on symbol switch --- piker/ui/_chart.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index b208cece..60d502be 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -240,6 +240,12 @@ class GodWidget(QWidget): # resume feeds *after* rendering chart view asap chart.resume_all_feeds() + # TODO: we need a check to see if the chart + # last had the xlast in view, if so then shift so it's + # still in view, if the user was viewing history then + # do nothing yah? + chart.default_view() + self.linkedsplits = linkedsplits symbol = linkedsplits.symbol if symbol is not None: @@ -904,7 +910,7 @@ class ChartPlotWidget(pg.PlotWidget): brange = l, lbar, rbar, r = self.bars_range() marker_pos, l1_len = self.pre_l1_xs() - end = xlast + l1_len + end = xlast + l1_len + 1 if ( rbar < 0 From 7edfe68d4dcbc08cf85f281fc3a4b82e08487b8b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 21 Mar 2022 18:51:59 -0400 Subject: [PATCH 29/84] M4 workin bishhhhh --- piker/ui/_ohlc.py | 77 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 19 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index de914862..24ae5d13 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -217,6 +217,7 @@ class BarItems(pg.GraphicsObject): self._color = pen_color self.bars_pen = pg.mkPen(hcolor(pen_color), width=1) self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2) + self._array = None # NOTE: this prevents redraws on mouse interaction which is # a huge boon for avg interaction latency. @@ -249,6 +250,7 @@ class BarItems(pg.GraphicsObject): self._ds_lines: dict[int, FastAppendCurve] = {} self._ds_line: Optional[FastAppendCurve] = None self._ds: int = 0 + self._xs_in_px: float = 0 def draw_from_data( self, @@ -283,10 +285,11 @@ class BarItems(pg.GraphicsObject): # https://doc.qt.io/qt-5/qgraphicsitem.html#update self.update() - self.update_ds_line(ohlc) - assert self._ds_line - self._ds_line.hide() + # self.update_ds_line(ohlc) + # assert self._ds_line + # self._ds_line.hide() + self._array = ohlc return self.path def get_ds_line( @@ -332,28 +335,47 @@ class BarItems(pg.GraphicsObject): # return None, None index = ohlc['index'] + + chart = self.linked.chart + if not chart: + return + else: + px_width = round(chart.curve_width_pxs()) + flat = rfn.structured_to_unstructured( ohlc[['open', 'high', 'low', 'close']] ).flatten() - xpts = np.linspace(start=index[0] - 0.5, stop=index[-1] + 0.5, num=4*len(ohlc)) + xpts = np.linspace( + start=index[0] - 0.5, + stop=index[-1] + 0.5, + num=4*len(ohlc), + ) + bins, x, y = ds_m4( + xpts, + flat, + px_width=px_width * 8, + ) + # x4 = np.zeros(y.shape) + x = np.broadcast_to(x[:, None], y.shape) #.flatten() + x = (x + np.array([-0.5, 0, 0, 0.5])).flatten() - # bins, x, ds = ds_m4( - # xpts, - # flat, - # # px_width=self.linked.chart.curve_width_pxs() - # px_width=self.getViewBox().width(), + # x = np.linspace( + # start=x[0] - 0.5, + # stop=x[-1] + 0.5, + # num=4*len(x), # ) + y = y.flatten() # breakpoint() - mxmn, x = hl2mxmn(ohlc) + # y, x = hl2mxmn(ohlc) # if self._ds_line: # self._pi.removeItem(self._ds_line) if not curve: curve = FastAppendCurve( - # y=ds.flatten(), - y=mxmn, + y=y, + # y=mxmn, x=x, name='ds', color=self._color, @@ -363,6 +385,7 @@ class BarItems(pg.GraphicsObject): self._pi.addItem(curve) self._ds_lines[ds] = curve self._ds_line = curve + curve.ds = px_width # elif ds != self._ds: # print(f'ds changed {self._ds} -> {ds}') @@ -386,8 +409,13 @@ class BarItems(pg.GraphicsObject): # istart, istop = curve._xrange # curve.path = None # print(x[-10:]) + if px_width != curve.ds: + print(f'redrawing {curve.ds} -> {px_width}') + curve.path = None + curve.ds = px_width + curve.update_from_array( - y=mxmn, + y=y, x=x, ) # curve.update() @@ -415,8 +443,8 @@ class BarItems(pg.GraphicsObject): ''' # XXX: always do this? - if self._ds_line: - curve, ds = self.update_ds_line(ohlc) + # if self._ds_line: + curve, ds = self.update_ds_line(ohlc) # index = self.start_index istart, istop = self._xrange @@ -507,6 +535,8 @@ class BarItems(pg.GraphicsObject): if flip_cache: self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + self._array = ohlc + def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect @@ -530,10 +560,12 @@ class BarItems(pg.GraphicsObject): mx_y = hb_br.y() mn_y = hb_tl.y() - body_line = self._last_bar_lines[0] - if body_line: - mx_y = max(mx_y, max(body_line.y1(), body_line.y2())) - mn_y = min(mn_y, min(body_line.y1(), body_line.y2())) + last_lines = self._last_bar_lines + if last_lines: + body_line = self._last_bar_lines[0] + if body_line: + mx_y = max(mx_y, max(body_line.y1(), body_line.y2())) + mn_y = min(mn_y, min(body_line.y1(), body_line.y2())) return QtCore.QRectF( @@ -563,6 +595,9 @@ class BarItems(pg.GraphicsObject): is less then a pixel width on the device). ''' + if not self._ds_line: + return False + # this is the ``float`` value of the "number of x units" (in # view coords) that a pixel spans. xvec = self.pixelVectors()[0] @@ -571,6 +606,10 @@ class BarItems(pg.GraphicsObject): else: xs_in_px = self._ds_line.pixelVectors()[0].x() + if xs_in_px != self._xs_in_px and self._array is not None: + self.update_ds_line(self._array) + self._xs_in_px = xs_in_px + linked = self.linked if ( From 08f90c275c3a1c866e236df6406d62e0a4a12010 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Mar 2022 09:54:59 -0400 Subject: [PATCH 30/84] Add OHLC to m4 line converters Helpers to quickly convert ohlc struct-array sequences into lines for consumption by the m4 downsampler. Strip trailing zero entries from the `ds_m4()` output if found (avoids lines back to origin). --- piker/ui/_compression.py | 55 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index be35244d..4c7da0d8 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -22,7 +22,7 @@ limits on the display device. import math import numpy as np -# from numpy.lib.recfunctions import structured_to_unstructured +from numpy.lib import recfunctions as rfn from numba import ( jit, # float64, optional, int64, @@ -174,6 +174,51 @@ def downsample( return ds_m4(x, y, kwargs['px_width']) +def ohlc_flatten( + ohlc: np.ndarray, + +) -> tuple[np.ndarray, np.ndarray]: + ''' + Convert an OHLCV struct-array into a flat ready-for-line-plotting + 1-d array that is 4 times the size with x-domain values distributed + evenly (by 0.5 steps) over each index. + + ''' + index = ohlc['index'] + + flat = rfn.structured_to_unstructured( + ohlc[['open', 'high', 'low', 'close']] + ).flatten() + + x = np.linspace( + start=index[0] - 0.5, + stop=index[-1] + 0.5, + num=4*len(ohlc), + ) + return x, flat + + +def ohlc_to_m4_line( + ohlc: np.ndarray, + px_width: int, + +) -> tuple[np.ndarray, np.ndarray]: + ''' + Convert an OHLC struct-array to a m4 downsampled 1-d array. + + ''' + xpts, flat = ohlc_flatten(ohlc) + bins, x, y = ds_m4( + xpts, + flat, + px_width=px_width * 16, + ) + x = np.broadcast_to(x[:, None], y.shape) + x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() + y = y.flatten() + return x, y + + def ds_m4( x: np.ndarray, y: np.ndarray, @@ -233,7 +278,7 @@ def ds_m4( # (uniform quotient output) worth of datum-domain-points # per windows-frame, add one more window to ensure # we have room for all output down-samples. - pts_per_pixel, r = divmod(len(x), px_width) + pts_per_pixel, r = divmod(len(x), frames) if r: frames += 1 @@ -257,6 +302,12 @@ def ds_m4( w, ) + # filter out any overshoot in the input allocation arrays by + # removing zero-ed tail entries which should start at a certain + # index. + i_win = i_win[i_win != 0] + y_out = y_out[:i_win.size] + return nb, i_win, y_out From 03e0e3e76b0bcb9bf2a89051d83ce5292c31f60e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Mar 2022 09:59:11 -0400 Subject: [PATCH 31/84] Delegate to m4 ohlc helper for curve, only ds on uppx steps > 2 --- piker/ui/_ohlc.py | 96 ++++++++++++++--------------------------------- 1 file changed, 29 insertions(+), 67 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 24ae5d13..4c694291 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -25,7 +25,6 @@ from typing import ( import numpy as np import pyqtgraph as pg -from numpy.lib import recfunctions as rfn from numba import njit, float64, int64 # , optional from PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtCore import QLineF, QPointF @@ -36,7 +35,12 @@ from .._profile import pg_profile_enabled from ._style import hcolor from ..log import get_logger from ._curve import FastAppendCurve -from ._compression import hl2mxmn, ds_m4 +from ._compression import ( + # hl2mxmn, + # ohlc_flatten, + ohlc_to_m4_line, + # ds_m4, +) if TYPE_CHECKING: from ._chart import LinkedSplits @@ -298,16 +302,19 @@ class BarItems(pg.GraphicsObject): ) -> tuple[FastAppendCurve, int]: - if ds is None: - px_vecs = self.pixelVectors()[0] - if px_vecs: - xs_in_px = px_vecs.x() - ds = round(xs_in_px) - else: - ds = 0 - # print(f'ds is {ds}') + px_vecs = self.pixelVectors()[0] - return self._ds_lines.get(ds), ds + if not px_vecs and self._ds_line: + px_vecs = self._ds_line.pixelVectors()[0] + + if px_vecs: + xs_in_px = px_vecs.x() + ds = round(xs_in_px) + else: + ds = 0 + + return self._ds_line, ds + # return self._ds_line.get(ds), ds def update_ds_line( self, @@ -323,18 +330,7 @@ class BarItems(pg.GraphicsObject): # determine current potential downsampling value (based on pixel # scaling) and return any existing curve for it. - curve, ds = self.get_ds_line(ds=0) - - # curve = self._ds_lines.get(ds) - # if current and current != curve: - # current.hide() - - # if no curve for this downsample rate yet, allowcate a new one - - # if not self.linked.chart: - # return None, None - - index = ohlc['index'] + curve, ds = self.get_ds_line() chart = self.linked.chart if not chart: @@ -342,50 +338,28 @@ class BarItems(pg.GraphicsObject): else: px_width = round(chart.curve_width_pxs()) - flat = rfn.structured_to_unstructured( - ohlc[['open', 'high', 'low', 'close']] - ).flatten() - xpts = np.linspace( - start=index[0] - 0.5, - stop=index[-1] + 0.5, - num=4*len(ohlc), - ) - bins, x, y = ds_m4( - xpts, - flat, - px_width=px_width * 8, - ) - # x4 = np.zeros(y.shape) - x = np.broadcast_to(x[:, None], y.shape) #.flatten() - x = (x + np.array([-0.5, 0, 0, 0.5])).flatten() - - # x = np.linspace( - # start=x[0] - 0.5, - # stop=x[-1] + 0.5, - # num=4*len(x), - # ) - y = y.flatten() - # breakpoint() - - # y, x = hl2mxmn(ohlc) - # if self._ds_line: # self._pi.removeItem(self._ds_line) + # old tracer with no downsampling + # y, x = hl2mxmn(ohlc) + x, y = ohlc_to_m4_line(ohlc, px_width) + if not curve: + curve = FastAppendCurve( y=y, - # y=mxmn, x=x, name='ds', color=self._color, # color='dad_blue', # use_polyline=True, # pretty sure this is slower? ) + curve.hide() self._pi.addItem(curve) self._ds_lines[ds] = curve self._ds_line = curve - curve.ds = px_width + return curve, ds # elif ds != self._ds: # print(f'ds changed {self._ds} -> {ds}') @@ -396,29 +370,17 @@ class BarItems(pg.GraphicsObject): # can just be read and rendered to graphics on events of our # choice. # diff = do_diff(ohlc, new_bit) - # mxmn, x = hl2mxmn(ohlc, downsample_by=ds) # always refresh data bounds until we get diffing - # working properly, see below.. - # curve._xrange = curve.dataBounds(ax=0) - - # TODO: we need to do a diff here to determine - # which ohlc samples have not yet been converted - # to tracer lines. - # index = ohlc['index'] - # istart, istop = curve._xrange - # curve.path = None - # print(x[-10:]) - if px_width != curve.ds: - print(f'redrawing {curve.ds} -> {px_width}') + # working properly, see above.. + if abs(ds - self._ds) > 2: + log.info(f'sampler change: {self._ds} -> {ds}') curve.path = None - curve.ds = px_width curve.update_from_array( y=y, x=x, ) - # curve.update() self._ds = ds return curve, ds From 44f3a08ef19a03cc6e085b4fcb4bdab6c67c6af6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 23 Mar 2022 12:29:57 -0400 Subject: [PATCH 32/84] Add optional uppx log scaling to m4 sampler We were previously ad-hoc scaling up the px count/width to get more detail at lower uppx values. Add a log scaling sigmoid that range scales between 1 < px_width < 16. Add in a flag to use the mxmn OH tracer in `ohlc_flatten()` if desired. --- piker/ui/_compression.py | 65 ++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index 4c7da0d8..d0f15b24 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -20,6 +20,7 @@ limits on the display device. ''' import math +from typing import Optional import numpy as np from numpy.lib import recfunctions as rfn @@ -34,11 +35,7 @@ from ..log import get_logger log = get_logger(__name__) -def hl2mxmn( - ohlc: np.ndarray, - # downsample_by: int = 0, - -) -> np.ndarray: +def hl2mxmn(ohlc: np.ndarray) -> np.ndarray: ''' Convert a OHLC struct-array containing 'high'/'low' columns to a "joined" max/min 1-d array. @@ -50,12 +47,6 @@ def hl2mxmn( 'high', ]] - # XXX: don't really need this any more since we implemented - # the "tracer" routine, `numba`-style.. - # create a "max and min" sequence from ohlc datums - # hl2d = structured_to_unstructured(hls) - # hl1d = hl2d.flatten() - mxmn = np.empty(2*hls.size, dtype=np.float64) x = np.empty(2*hls.size, dtype=np.float64) trace_hl(hls, mxmn, x, index[0]) @@ -63,18 +54,6 @@ def hl2mxmn( return mxmn, x - # if downsample_by < 2: - # return mxmn, x - - # dsx, dsy = downsample( - # y=mxmn, - # x=x, - # bins=downsample_by, - # ) - # log.info(f'downsampling by {downsample_by}') - # print(f'downsampling by {downsample_by}') - # return dsy, dsx - @jit( # TODO: the type annots.. @@ -176,6 +155,7 @@ def downsample( def ohlc_flatten( ohlc: np.ndarray, + use_mxmn: bool = False, ) -> tuple[np.ndarray, np.ndarray]: ''' @@ -186,15 +166,18 @@ def ohlc_flatten( ''' index = ohlc['index'] - flat = rfn.structured_to_unstructured( - ohlc[['open', 'high', 'low', 'close']] - ).flatten() + if use_mxmn: + flat, x = hl2mxmn(ohlc) + else: + flat = rfn.structured_to_unstructured( + ohlc[['open', 'high', 'low', 'close']] + ).flatten() - x = np.linspace( - start=index[0] - 0.5, - stop=index[-1] + 0.5, - num=4*len(ohlc), - ) + x = np.linspace( + start=index[0] - 0.5, + stop=index[-1] + 0.5, + num=len(flat), + ) return x, flat @@ -202,16 +185,33 @@ def ohlc_to_m4_line( ohlc: np.ndarray, px_width: int, + uppx: Optional[float] = None, + ) -> tuple[np.ndarray, np.ndarray]: ''' Convert an OHLC struct-array to a m4 downsampled 1-d array. ''' xpts, flat = ohlc_flatten(ohlc) + + if uppx: + # optionally log-scale down the "supposed pxs on screen" + # as the units-per-px (uppx) get's large. + scaler = round( + max( + # NOTE: found that a 16x px width brought greater + # detail, likely due to dpi scaling? + # px_width=px_width * 16, + 32 / (1 + math.log(uppx, 2)), + 1 + ) + ) + px_width *= scaler + bins, x, y = ds_m4( xpts, flat, - px_width=px_width * 16, + px_width=px_width, ) x = np.broadcast_to(x[:, None], y.shape) x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() @@ -313,6 +313,7 @@ def ds_m4( @jit( nopython=True, + nogil=True, ) def _m4( From 1abe513ecb1a0fe3951e3ec444e9c8f74f0f69b1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 23 Mar 2022 12:32:55 -0400 Subject: [PATCH 33/84] Add our own `FastAppendCurve.clear()`, try mem reso In an effort to try and make `QPainterPath.reserve()` work, add internal logic to use the same object without de-allocating memory from a previous path write/creation. Note this required the addition of a `._redraw` flag (to be used in `.clear()` and a small patch to `pyqtgraph.functions.arrayToQPath` to allow passing in an existing path (thus reusing the same underlying mem alloc) which will likely be first pushed to our fork. --- piker/ui/_curve.py | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 429f4f0a..da2e6d23 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -124,6 +124,7 @@ class FastAppendCurve(pg.PlotCurveItem): ) -> None: self._name = name + self.path: Optional[QtGui.QPainterPath] = None # TODO: we can probably just dispense with the parent since # we're basically only using the pen setting now... @@ -133,6 +134,7 @@ class FastAppendCurve(pg.PlotCurveItem): # self._last_draw = time.time() self._use_poly = use_polyline self.poly = None + self._redraw: bool = False # all history of curve is drawn in single px thickness pen = pg.mkPen(hcolor(color)) @@ -203,8 +205,11 @@ class FastAppendCurve(pg.PlotCurveItem): # by default we only pull data up to the last (current) index x_out, y_out = x[:-1], y[:-1] - if self.path is None or prepend_length > 0: - + if ( + self.path is None + or prepend_length > 0 + or self._redraw + ): if self._use_poly: self.poly = pg.functions.arrayToQPolygonF( x_out, @@ -217,8 +222,19 @@ class FastAppendCurve(pg.PlotCurveItem): y_out, connect='all', finiteCheck=False, + path=self.path, ) + # reserve mem allocs see: + # - https://doc.qt.io/qt-5/qpainterpath.html#reserve + # - https://doc.qt.io/qt-5/qpainterpath.html#capacity + # - https://doc.qt.io/qt-5/qpainterpath.html#clear + # XXX: right now this is based on had hoc checks on a + # hidpi 3840x2160 4k monitor but we should optimize for + # the target display(s) on the sys. + self.path.reserve(int(500e3)) + profiler('generate fresh path') + self._redraw = False # if self._step_mode: # self.path.closeSubpath() @@ -351,6 +367,28 @@ class FastAppendCurve(pg.PlotCurveItem): # XXX: seems to be needed to avoid artifacts (see above). self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + def clear(self): + ''' + Clear internal graphics making object ready for full re-draw. + + ''' + # NOTE: original code from ``pg.PlotCurveItem`` + self.xData = None ## raw values + self.yData = None + self._renderSegmentList = None + # self.path = None + self.fillPath = None + self._fillPathList = None + self._mouseShape = None + self._mouseBounds = None + self._boundsCache = [None, None] + #del self.xData, self.yData, self.xDisp, self.yDisp, self.path + + # path reservation aware non-mem de-alloc cleaning + if self.path: + self.path.clear() + self._redraw = True + def disable_cache(self) -> None: ''' Disable the use of the pixel coordinate cache and trigger a geo event. From 88a7314bd0443ba9a3522edd9785e4895c223d0a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 23 Mar 2022 17:29:28 -0400 Subject: [PATCH 34/84] Add optional mxmn HL tracer support to m4 sampler --- piker/ui/_compression.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index d0f15b24..3c5470f4 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -185,14 +185,19 @@ def ohlc_to_m4_line( ohlc: np.ndarray, px_width: int, + downsample: bool = False, uppx: Optional[float] = None, + pretrace: bool = False, ) -> tuple[np.ndarray, np.ndarray]: ''' Convert an OHLC struct-array to a m4 downsampled 1-d array. ''' - xpts, flat = ohlc_flatten(ohlc) + xpts, flat = ohlc_flatten( + ohlc, + use_mxmn=pretrace, + ) if uppx: # optionally log-scale down the "supposed pxs on screen" @@ -208,15 +213,19 @@ def ohlc_to_m4_line( ) px_width *= scaler - bins, x, y = ds_m4( - xpts, - flat, - px_width=px_width, - ) - x = np.broadcast_to(x[:, None], y.shape) - x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() - y = y.flatten() - return x, y + if downsample: + bins, x, y = ds_m4( + xpts, + flat, + px_width=px_width, + ) + x = np.broadcast_to(x[:, None], y.shape) + x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() + y = y.flatten() + + return x, y + else: + return xpts, flat def ds_m4( From 3a6c5a2fbdebc1121de7bd65ddc776e08091d339 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 23 Mar 2022 17:29:56 -0400 Subject: [PATCH 35/84] Try supporting reuse of path allocation --- piker/ui/_curve.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index da2e6d23..2aeb60cf 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -376,7 +376,6 @@ class FastAppendCurve(pg.PlotCurveItem): self.xData = None ## raw values self.yData = None self._renderSegmentList = None - # self.path = None self.fillPath = None self._fillPathList = None self._mouseShape = None @@ -389,6 +388,10 @@ class FastAppendCurve(pg.PlotCurveItem): self.path.clear() self._redraw = True + # XXX: if not trying to leverage `.reserve()` allocs + # then you might as well create a new one.. + # self.path = None + def disable_cache(self) -> None: ''' Disable the use of the pixel coordinate cache and trigger a geo event. From 561d7e03499a5e264937f1ff94431214eaf70246 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 24 Mar 2022 09:21:44 -0400 Subject: [PATCH 36/84] Only clear/redraw curve on uppx diffs > 2 Only if the uppx increases by more then 2 we redraw the entire line otherwise just ds with previous params and update the current curve. This *should* avoid strange lower sample rate artefacts from showing on updates. Summary: - stash both uppx and px width in `._dsi` (downsample info) - use the new `ohlc_to_m4_line()` flags - add notes about using `.reserve()` and friends - always delete last `._array` ref prior to line updates --- piker/ui/_ohlc.py | 86 +++++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 4c694291..c6dbf67c 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -36,10 +36,7 @@ from ._style import hcolor from ..log import get_logger from ._curve import FastAppendCurve from ._compression import ( - # hl2mxmn, - # ohlc_flatten, ohlc_to_m4_line, - # ds_m4, ) if TYPE_CHECKING: @@ -234,9 +231,6 @@ class BarItems(pg.GraphicsObject): self._pi = plotitem self.path = QtGui.QPainterPath() - # not sure if this is actually impoving anything but figured it - # was worth a shot: - self.path.reserve(int(100e3 * 6)) self._xrange: tuple[int, int] self._yrange: tuple[float, float] @@ -253,7 +247,7 @@ class BarItems(pg.GraphicsObject): self._in_ds: bool = False self._ds_lines: dict[int, FastAppendCurve] = {} self._ds_line: Optional[FastAppendCurve] = None - self._ds: int = 0 + self._dsi: tuple[int, int] = 0, 0 self._xs_in_px: float = 0 def draw_from_data( @@ -319,18 +313,12 @@ class BarItems(pg.GraphicsObject): def update_ds_line( self, ohlc: np.ndarray, - use_ds: bool = False, ) -> int: - if not use_ds: - ds = 0 - else: - ds = None - # determine current potential downsampling value (based on pixel # scaling) and return any existing curve for it. - curve, ds = self.get_ds_line() + curve, uppx = self.get_ds_line() chart = self.linked.chart if not chart: @@ -341,12 +329,46 @@ class BarItems(pg.GraphicsObject): # if self._ds_line: # self._pi.removeItem(self._ds_line) - # old tracer with no downsampling - # y, x = hl2mxmn(ohlc) - x, y = ohlc_to_m4_line(ohlc, px_width) + # log.info(f'current dsi: {self._dsi}') + ds_uppx, ds_px_width = self._dsi + + if ( + abs(uppx - ds_uppx) >= 2 + or ds_px_width == 0 + ): + ds_uppx, ds_px_width = dsi = (uppx, px_width) + log.info(f'sampler change: {self._dsi} -> {dsi}') + self._dsi = dsi + + if curve: + # trigger a full redraw of the curve path since + # we have downsampled another "level" using m4. + curve.clear() + + # always refresh data bounds until we get diffing + # working properly, see above.. + x, y = ohlc_to_m4_line( + ohlc, + ds_px_width, + uppx=ds_uppx, + # pretrace=True, + + # activate m4 ds? + downsample=True, + ) if not curve: + # TODO: figuring out the most optimial size for the ideal + # curve-path by, + # - calcing the display's max px width `.screen()` + # - drawing a curve and figuring out it's capacity: + # https://doc.qt.io/qt-5/qpainterpath.html#capacity + # - reserving that cap for each curve-mapped-to-shm with + + # - leveraging clearing when needed to redraw the entire + # curve that does not release mem allocs: + # https://doc.qt.io/qt-5/qpainterpath.html#clear curve = FastAppendCurve( y=y, x=x, @@ -357,12 +379,9 @@ class BarItems(pg.GraphicsObject): ) curve.hide() self._pi.addItem(curve) - self._ds_lines[ds] = curve + self._ds_lines[ds_uppx] = curve self._ds_line = curve - return curve, ds - - # elif ds != self._ds: - # print(f'ds changed {self._ds} -> {ds}') + return curve, ds_uppx # TODO: we should be diffing the amount of new data which # needs to be downsampled. Ideally we actually are just @@ -371,19 +390,11 @@ class BarItems(pg.GraphicsObject): # choice. # diff = do_diff(ohlc, new_bit) - # always refresh data bounds until we get diffing - # working properly, see above.. - if abs(ds - self._ds) > 2: - log.info(f'sampler change: {self._ds} -> {ds}') - curve.path = None - curve.update_from_array( y=y, x=x, ) - self._ds = ds - - return curve, ds + return curve, ds_uppx def update_from_array( self, @@ -406,6 +417,8 @@ class BarItems(pg.GraphicsObject): ''' # XXX: always do this? # if self._ds_line: + del self._array + self._array = ohlc curve, ds = self.update_ds_line(ohlc) # index = self.start_index @@ -497,8 +510,6 @@ class BarItems(pg.GraphicsObject): if flip_cache: self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - self._array = ohlc - def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect @@ -568,7 +579,10 @@ class BarItems(pg.GraphicsObject): else: xs_in_px = self._ds_line.pixelVectors()[0].x() - if xs_in_px != self._xs_in_px and self._array is not None: + if ( + # xs_in_px != self._xs_in_px + self._array is not None + ): self.update_ds_line(self._array) self._xs_in_px = xs_in_px @@ -619,7 +633,7 @@ class BarItems(pg.GraphicsObject): # elif ( # self._in_ds - # and self._ds != ds + # and self._dsi != ds # ): # # curve = self._ds_lines.get(ds) # # assert self._ds_line is not curve @@ -632,7 +646,7 @@ class BarItems(pg.GraphicsObject): # curve.update() # self._ds_line = curve - # self._ds = ds + # self._dsi = ds # linked.graphics_cycle() # return True From b262532fd4fb3fea4fc833e18f4b8d0b6f5cb55e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 24 Mar 2022 13:22:30 -0400 Subject: [PATCH 37/84] Allocate m4 output arrays in `numba` code, avoid segfaults? --- piker/ui/_compression.py | 47 ++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index 3c5470f4..9887138e 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -207,7 +207,7 @@ def ohlc_to_m4_line( # NOTE: found that a 16x px width brought greater # detail, likely due to dpi scaling? # px_width=px_width * 16, - 32 / (1 + math.log(uppx, 2)), + 64 / (1 + math.log(uppx, 2)), 1 ) ) @@ -291,18 +291,16 @@ def ds_m4( if r: frames += 1 - # these are pre-allocated and mutated by ``numba`` - # code in-place. - y_out = np.zeros((frames, 4), y.dtype) - i_win = np.zeros(frames, x.dtype) - # call into ``numba`` - nb = _m4( + nb, i_win, y_out = _m4( x, y, - i_win, - y_out, + frames, + + # TODO: see func below.. + # i_win, + # y_out, # first index in x data to start at x_start, @@ -322,19 +320,25 @@ def ds_m4( @jit( nopython=True, - nogil=True, + # nogil=True, ) def _m4( xs: np.ndarray, ys: np.ndarray, + frames: int, + + # TODO: using this approach by having the ``.zeros()`` alloc lines + # below, in put python was causing segs faults and alloc crashes.. + # we might need to see how it behaves with shm arrays and consider + # allocating them once at startup? + # pre-alloc array of x indices mapping to the start # of each window used for downsampling in y. - i_win: np.ndarray, - + # i_win: np.ndarray, # pre-alloc array of output downsampled y values - ds: np.ndarray, + # y_out: np.ndarray, x_start: int, step: float, @@ -343,6 +347,11 @@ def _m4( # nbins = len(i_win) # count = len(xs) + # these are pre-allocated and mutated by ``numba`` + # code in-place. + y_out = np.zeros((frames, 4), ys.dtype) + i_win = np.zeros(frames, xs.dtype) + bincount = 0 x_left = x_start @@ -357,15 +366,15 @@ def _m4( # (aka a row broadcast). i_win[bincount] = x_left # set all y-values to the first value passed in. - ds[bincount] = ys[0] + y_out[bincount] = ys[0] for i in range(len(xs)): x = xs[i] y = ys[i] if x < x_left + step: # the current window "step" is [bin, bin+1) - ds[bincount, 1] = min(y, ds[bincount, 1]) - ds[bincount, 2] = max(y, ds[bincount, 2]) - ds[bincount, 3] = y + y_out[bincount, 1] = min(y, y_out[bincount, 1]) + y_out[bincount, 2] = max(y, y_out[bincount, 2]) + y_out[bincount, 3] = y else: # Find the next bin while x >= x_left + step: @@ -373,6 +382,6 @@ def _m4( bincount += 1 i_win[bincount] = x_left - ds[bincount] = y + y_out[bincount] = y - return bincount + return bincount, i_win, y_out From 2b127429925998ff071e7024595c36ee5c3138ef Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 24 Mar 2022 13:23:34 -0400 Subject: [PATCH 38/84] More ems resiliency: discard broken client dialogs --- piker/clearing/_ems.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index c49ff4bf..a5d04f0c 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -261,7 +261,15 @@ async def clear_dark_triggers( f'pred for {oid} was already removed!?' ) - await ems_client_order_stream.send(msg) + try: + await ems_client_order_stream.send(msg) + except ( + trio.ClosedResourceError, + ): + log.warning( + f'client {ems_client_order_stream} stream is broke' + ) + break else: # condition scan loop complete log.debug(f'execs are {execs}') @@ -573,8 +581,16 @@ async def translate_and_relay_brokerd_events( # fan-out-relay position msgs immediately by # broadcasting updates on all client streams - for client_stream in router.clients: - await client_stream.send(pos_msg) + for client_stream in router.clients.copy(): + try: + await client_stream.send(pos_msg) + except( + trio.ClosedResourceError, + trio.BrokenResourceError, + ): + router.clients.remove(client_stream) + log.warning( + f'client for {client_stream} was already closed?') continue From 9b2ec871a043be747971b946326641fbde0b0ea1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 24 Mar 2022 13:29:45 -0400 Subject: [PATCH 39/84] Clear ds line graphics on switch back to bars --- piker/ui/_ohlc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index c6dbf67c..3887b9ba 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -599,8 +599,10 @@ class BarItems(pg.GraphicsObject): self._pi.removeItem(self) curve, ds = self.get_ds_line(ds=0) + last_curve = self._ds_line assert last_curve is curve + self._pi.addItem(curve) curve.show() curve.update() @@ -619,6 +621,7 @@ class BarItems(pg.GraphicsObject): last_curve = self._ds_line assert last_curve is curve curve.hide() + curve.clear() self._pi.removeItem(curve) # XXX: is this actually any faster? From 5e161aa2512c6480c4d6cf07b96d21fdb646fcb1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Mar 2022 13:07:58 -0400 Subject: [PATCH 40/84] Always clear previous downsample curve on switch Pretty sure this was most of the cause of the stale (more downsampled) curves showing when zooming in and out from bars mode quickly. All this stuff needs to get factored out into a new abstraction anyway, but i think this get's mostly correct functionality. Only draw new ds curve on uppx steps >= 4 and stop adding/removing graphics objects from the scene; doesn't seem to speed anything up afaict. Add better reporting of ds scale changes. --- piker/ui/_ohlc.py | 71 ++++++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 3887b9ba..a7218fd9 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -175,6 +175,7 @@ def gen_qpath( data, start, # XXX: do we need this? w, + ) -> QtGui.QPainterPath: profiler = pg.debug.Profiler(disabled=not pg_profile_enabled()) @@ -319,6 +320,7 @@ class BarItems(pg.GraphicsObject): # determine current potential downsampling value (based on pixel # scaling) and return any existing curve for it. curve, uppx = self.get_ds_line() + # print(f'uppx: {uppx}') chart = self.linked.chart if not chart: @@ -326,30 +328,37 @@ class BarItems(pg.GraphicsObject): else: px_width = round(chart.curve_width_pxs()) + if px_width == 0: + return + # if self._ds_line: # self._pi.removeItem(self._ds_line) # log.info(f'current dsi: {self._dsi}') - ds_uppx, ds_px_width = self._dsi + old_dsi = ds_uppx, ds_px_width = self._dsi + changed = False if ( - abs(uppx - ds_uppx) >= 2 - or ds_px_width == 0 + abs(uppx - ds_uppx) >= 4 + # or not self._in_ds ): - ds_uppx, ds_px_width = dsi = (uppx, px_width) - log.info(f'sampler change: {self._dsi} -> {dsi}') - self._dsi = dsi - + changed = True if curve: # trigger a full redraw of the curve path since # we have downsampled another "level" using m4. curve.clear() + ds_uppx, ds_px_width = dsi = (uppx, px_width) + self._dsi = dsi + + if changed: + log.info(f'sampler change: {old_dsi} -> {dsi}') + # always refresh data bounds until we get diffing # working properly, see above.. x, y = ohlc_to_m4_line( ohlc, - ds_px_width, + px_width=ds_px_width, uppx=ds_uppx, # pretrace=True, @@ -417,9 +426,9 @@ class BarItems(pg.GraphicsObject): ''' # XXX: always do this? # if self._ds_line: - del self._array + # del self._array self._array = ohlc - curve, ds = self.update_ds_line(ohlc) + self.update_ds_line(ohlc) # index = self.start_index istart, istop = self._xrange @@ -568,7 +577,8 @@ class BarItems(pg.GraphicsObject): is less then a pixel width on the device). ''' - if not self._ds_line: + curve = self._ds_line + if not curve: return False # this is the ``float`` value of the "number of x units" (in @@ -579,14 +589,19 @@ class BarItems(pg.GraphicsObject): else: xs_in_px = self._ds_line.pixelVectors()[0].x() + linked = self.linked + if ( # xs_in_px != self._xs_in_px self._array is not None ): - self.update_ds_line(self._array) - self._xs_in_px = xs_in_px + # print('refreshing curve') + out = self.update_ds_line(self._array) + if not out: + print("NOTHING!?") + return - linked = self.linked + curve, ds = out if ( not self._in_ds @@ -596,14 +611,20 @@ class BarItems(pg.GraphicsObject): log.info(f'downsampling to line graphic {linked.symbol.key}') self.hide() # XXX: is this actually any faster? - self._pi.removeItem(self) + # self._pi.removeItem(self) - curve, ds = self.get_ds_line(ds=0) + curve, ds = out + self._xs_in_px = xs_in_px + # curve, ds = self.get_ds_line(ds=0) + curve.clear() + curve.update() - last_curve = self._ds_line - assert last_curve is curve + curve, out = self.update_ds_line(self._array) - self._pi.addItem(curve) + # curve = self._ds_line + # assert last_curve is curve + + # self._pi.addItem(curve) curve.show() curve.update() @@ -617,15 +638,15 @@ class BarItems(pg.GraphicsObject): ): log.info(f'showing bars graphic {linked.symbol.key}') - curve, ds = self.get_ds_line(ds=0) - last_curve = self._ds_line - assert last_curve is curve - curve.hide() + # curve, ds = self.get_ds_line() + curve = self._ds_line + # assert last_curve is curve curve.clear() - self._pi.removeItem(curve) + curve.hide() + # self._pi.removeItem(curve) # XXX: is this actually any faster? - self._pi.addItem(self) + # self._pi.addItem(self) self.show() self.update() From d59442e3b1070fa529771645399a9f1712fc33d2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Mar 2022 13:15:59 -0400 Subject: [PATCH 41/84] Bump up resolution log scaling a mag --- piker/ui/_compression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index 9887138e..1fa9dcae 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -207,7 +207,7 @@ def ohlc_to_m4_line( # NOTE: found that a 16x px width brought greater # detail, likely due to dpi scaling? # px_width=px_width * 16, - 64 / (1 + math.log(uppx, 2)), + 128 / (1 + math.log(uppx, 2)), 1 ) ) From 86da64c2c2464bf257260fb209df06984863f738 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Mar 2022 13:17:06 -0400 Subject: [PATCH 42/84] Show baseline bars length on in view read < 6 --- piker/ui/_chart.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 60d502be..eda53bd5 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -915,6 +915,7 @@ class ChartPlotWidget(pg.PlotWidget): if ( rbar < 0 or l < xfirst + or (rbar - lbar) < 6 ): # set fixed bars count on screen that approx includes as # many bars as possible before a downsample line is shown. From 28bf8853aa786ac9a1c2baefc15a1b2322550572 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Mar 2022 14:11:31 -0400 Subject: [PATCH 43/84] Drop commented line from pq method copy/paste --- piker/ui/_curve.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 2aeb60cf..92396987 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -381,7 +381,6 @@ class FastAppendCurve(pg.PlotCurveItem): self._mouseShape = None self._mouseBounds = None self._boundsCache = [None, None] - #del self.xData, self.yData, self.xDisp, self.yDisp, self.path # path reservation aware non-mem de-alloc cleaning if self.path: From 5800c1090106d25fc75c14c6fdc53f838f8719bb Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 30 Mar 2022 15:43:14 -0400 Subject: [PATCH 44/84] First try, drop `FastAppendCurve` inheritance from `pg.PlotCurveItem` --- piker/ui/_curve.py | 56 +++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 92396987..7c83129b 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -95,8 +95,7 @@ _line_styles: dict[str, int] = { } -# TODO: got a feeling that dropping this inheritance gets us even more speedups -class FastAppendCurve(pg.PlotCurveItem): +class FastAppendCurve(pg.GraphicsObject): ''' A faster, append friendly version of ``pyqtgraph.PlotCurveItem`` built for real-time data updates. @@ -111,7 +110,11 @@ class FastAppendCurve(pg.PlotCurveItem): ''' def __init__( self, + + x: np.ndarray, + y: np.ndarray, *args, + step_mode: bool = False, color: str = 'default_lightest', fill_color: Optional[str] = None, @@ -123,13 +126,19 @@ class FastAppendCurve(pg.PlotCurveItem): ) -> None: + # brutaaalll, see comments within.. + self.yData = y + self.xData = x + self._name = name self.path: Optional[QtGui.QPainterPath] = None # TODO: we can probably just dispense with the parent since # we're basically only using the pen setting now... super().__init__(*args, **kwargs) - self._xrange: tuple[int, int] = self.dataBounds(ax=0) + + # self._xrange: tuple[int, int] = self.dataBounds(ax=0) + self._xrange: Optional[tuple[int, int]] = None # self._last_draw = time.time() self._use_poly = use_polyline @@ -143,7 +152,7 @@ class FastAppendCurve(pg.PlotCurveItem): if 'dash' in style: pen.setDashPattern([8, 3]) - self.setPen(pen) + self._pen = pen # last segment is drawn in 2px thickness for emphasis # self.last_step_pen = pg.mkPen(hcolor(color), width=2) @@ -156,7 +165,7 @@ class FastAppendCurve(pg.PlotCurveItem): self._step_mode: bool = step_mode # self._fill = True - self.setBrush(hcolor(fill_color or color)) + self._brush = pg.functions.mkBrush(hcolor(fill_color or color)) # TODO: one question still remaining is if this makes trasform # interactions slower (such as zooming) and if so maybe if/when @@ -171,6 +180,8 @@ class FastAppendCurve(pg.PlotCurveItem): QGraphicsItem.DeviceCoordinateCache ) + self.update() + def update_from_array( self, x: np.ndarray, @@ -187,7 +198,11 @@ class FastAppendCurve(pg.PlotCurveItem): profiler = pg.debug.Profiler(disabled=not pg_profile_enabled()) flip_cache = False - istart, istop = self._xrange + if self._xrange: + istart, istop = self._xrange + else: + istart, istop = x[0], x[-1] + # print(f"xrange: {self._xrange}") # compute the length diffs between the first/last index entry in @@ -367,20 +382,19 @@ class FastAppendCurve(pg.PlotCurveItem): # XXX: seems to be needed to avoid artifacts (see above). self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + # XXX: lol brutal, the internals of `CurvePoint` (inherited by + # our `LineDot`) required ``.getData()`` to work.. + def getData(self): + return self.xData, self.yData + def clear(self): ''' Clear internal graphics making object ready for full re-draw. ''' # NOTE: original code from ``pg.PlotCurveItem`` - self.xData = None ## raw values + self.xData = None self.yData = None - self._renderSegmentList = None - self.fillPath = None - self._fillPathList = None - self._mouseShape = None - self._mouseBounds = None - self._boundsCache = [None, None] # path reservation aware non-mem de-alloc cleaning if self.path: @@ -460,19 +474,17 @@ class FastAppendCurve(pg.PlotCurveItem): self._step_mode and self._last_step_rect ): - brush = self.opts['brush'] + brush = self._brush + # p.drawLines(*tuple(filter(bool, self._last_step_lines))) # p.drawRect(self._last_step_rect) p.fillRect(self._last_step_rect, brush) - # p.drawPath(self.path) - # profiler('.drawPath()') - if self._last_line: p.setPen(self.last_step_pen) p.drawLine(self._last_line) profiler('.drawLine()') - p.setPen(self.opts['pen']) + p.setPen(self._pen) # else: if self._use_poly: @@ -490,11 +502,3 @@ class FastAppendCurve(pg.PlotCurveItem): # if self._fill: # brush = self.opts['brush'] # p.fillPath(self.path, brush) - - # now = time.time() - # print(f'DRAW RATE {1/(now - self._last_draw)}') - # self._last_draw = now - - -# import time -# _last_draw: float = time.time() From 8627f6f6c5f4d870f5a6d539270b5fd1a34d5fee Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 14 Mar 2022 06:04:18 -0400 Subject: [PATCH 45/84] Add no-path guard now that we can use a poly --- piker/ui/_curve.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 7c83129b..63c14159 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -466,9 +466,8 @@ class FastAppendCurve(pg.GraphicsObject): ) -> None: profiler = pg.debug.Profiler( - # disabled=False, #not pg_profile_enabled(), + disabled=not pg_profile_enabled(), ) - # p.setRenderHint(p.Antialiasing, True) if ( self._step_mode @@ -486,13 +485,14 @@ class FastAppendCurve(pg.GraphicsObject): profiler('.drawLine()') p.setPen(self._pen) - # else: + path = self.path if self._use_poly: assert self.poly p.drawPolyline(self.poly) profiler('.drawPolyline()') - else: - p.drawPath(self.path) + + elif path: + p.drawPath(path) profiler('.drawPath()') # TODO: try out new work from `pyqtgraph` main which should From 947a51415371c751d001de41b7116e809a5e8f04 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 31 Mar 2022 19:04:52 -0400 Subject: [PATCH 46/84] Add "native" downsampling to our `FastAppendCurve` Build out an interface that makes it super easy to downsample curves using the m4 algorithm while keeping our incremental `QPainterPath` update feature. A lot of hard work and tinkering went into getting this working all in-thread correctly and there are quite a few details.. New interface methods: - `.x_uppx()` which returns the x-axis "view units per pixel" - `.px_width()` which returns the total (rounded) x-axis pixels spanned by the curve in view. - `.should_ds_or_redraw()` a predicate which checks internal state to see if either downsampling of the curve should take place, or the curve should have all downsampling removed and be redrawn with source array data. - `.downsample()` the actual ds processing routine which delegates into the m4 algo impl. - `.maybe_downsample()` a simple update method which can be called by the view box when the user changes the zoom level. Implementation details/changes: - make `.update_from_array()` check for downsample (or revert to source aka de-downsample) conditions exist and then downsample and re-draw path graphics accordingly. - in order to even further speed up path appends (since our main bottleneck is measured to be `QPainter.drawPath()` calls with large paths which are frequently updates), add a secondary path `.fast_path` which is the path that is real-time updates by incremental appends and which is painted separately for speed in `.pain()`. - drop all the `QPolyLine` stuff since it was tested to be much slower in general and especially so for append-updates. - stop disabling the cache settings on updates since it doesn't seem to be required any more? - more move toward deprecating and removing all lingering interface requirements from `pg.PlotCurveItem` (like `.xData`/`.yData`). - adjust `.paint()` and `.boundingRect()` to compensate for the new `.fast_path` - add a butt-load of profiling B) --- piker/ui/_curve.py | 361 ++++++++++++++++++++++++++++++++------------- 1 file changed, 261 insertions(+), 100 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 63c14159..a4e73194 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -34,6 +34,14 @@ from PyQt5.QtCore import ( from .._profile import pg_profile_enabled from ._style import hcolor +from ._compression import ( + # ohlc_to_m4_line, + ds_m4, +) +from ..log import get_logger + + +log = get_logger(__name__) def step_path_arrays_from_1d( @@ -120,19 +128,22 @@ class FastAppendCurve(pg.GraphicsObject): fill_color: Optional[str] = None, style: str = 'solid', name: Optional[str] = None, - use_polyline: bool = False, + use_fpath: bool = True, **kwargs ) -> None: # brutaaalll, see comments within.. - self.yData = y - self.xData = x + self._y = self.yData = y + self._x = self.xData = x self._name = name self.path: Optional[QtGui.QPainterPath] = None + self.use_fpath = use_fpath + self.fast_path: Optional[QtGui.QPainterPath] = None + # TODO: we can probably just dispense with the parent since # we're basically only using the pen setting now... super().__init__(*args, **kwargs) @@ -141,9 +152,8 @@ class FastAppendCurve(pg.GraphicsObject): self._xrange: Optional[tuple[int, int]] = None # self._last_draw = time.time() - self._use_poly = use_polyline - self.poly = None - self._redraw: bool = False + self._in_ds: bool = False + self._last_uppx: float = 0 # all history of curve is drawn in single px thickness pen = pg.mkPen(hcolor(color)) @@ -171,7 +181,7 @@ class FastAppendCurve(pg.GraphicsObject): # interactions slower (such as zooming) and if so maybe if/when # we implement a "history" mode for the view we disable this in # that mode? - if step_mode or self._use_poly: + if step_mode: # don't enable caching by default for the case where the # only thing drawn is the "last" line segment which can # have a weird artifact where it won't be fully drawn to its @@ -182,6 +192,107 @@ class FastAppendCurve(pg.GraphicsObject): self.update() + def x_uppx(self) -> int: + + px_vecs = self.pixelVectors()[0] + if px_vecs: + xs_in_px = px_vecs.x() + return round(xs_in_px) + else: + return 0 + + def px_width(self) -> float: + + vb = self.getViewBox() + if not vb: + return 0 + + vr = self.viewRect() + l, r = int(vr.left()), int(vr.right()) + + start, stop = self._xrange + lbar = max(l, start) + rbar = min(r, stop) + + return vb.mapViewToDevice( + QLineF(lbar, 0, rbar, 0) + ).length() + + def should_ds_or_redraw( + self, + + ) -> tuple[bool, bool]: + + uppx = self.x_uppx() + px_width = self.px_width() + uppx_diff = abs(uppx - self._last_uppx) + self._last_uppx = uppx + + should_redraw: bool = False + should_ds: bool = False + + if ( + uppx <= 4 + ): + # trigger redraw or original non-downsampled data + if self._in_ds: + print('REVERTING BACK TO SRC DATA') + # clear downsampled curve(s) and expect + # refresh of path segments. + should_redraw = True + + elif ( + uppx_diff >= 4 + or self._step_mode and uppx_diff >= 1 + ): + log.info( + f'{self._name} downsampler change: {self._last_uppx} -> {uppx}' + ) + should_ds = {'px_width': px_width, 'uppx': uppx} + should_redraw = True + + return should_ds, should_redraw + + def downsample( + self, + x, + y, + px_width, + uppx, + + ) -> tuple[np.ndarray, np.ndarray]: + + # downsample whenever more then 1 pixels per datum can be shown. + # always refresh data bounds until we get diffing + # working properly, see above.. + bins, x, y = ds_m4( + x, + y, + px_width=px_width, + uppx=uppx, + log_scale=bool(uppx) + ) + x = np.broadcast_to(x[:, None], y.shape) + # x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() + x = (x + np.array([-0.5, 0, 0, 0.5])).flatten() + y = y.flatten() + + # presumably? + self._in_ds = True + return x, y + + def maybe_downsample( + self, + ) -> None: + ''' + Simple update call but with previously cached arrays data. + + ''' + # print('DS CALLED FROM INTERACTION?') + # presume this is a so called "interaction update", see + # ``ChartView.maybe_downsample_graphics()``. + self.update_from_array(self._x, self._y) + def update_from_array( self, x: np.ndarray, @@ -195,61 +306,84 @@ class FastAppendCurve(pg.GraphicsObject): a length diff. ''' - profiler = pg.debug.Profiler(disabled=not pg_profile_enabled()) + profiler = pg.debug.Profiler( + msg=f'{self._name}.update_from_array()', + disabled=not pg_profile_enabled(), + ) flip_cache = False if self._xrange: istart, istop = self._xrange else: - istart, istop = x[0], x[-1] - + self._xrange = istart, istop = x[0], x[-1] # print(f"xrange: {self._xrange}") + should_ds, should_redraw = self.should_ds_or_redraw() + # compute the length diffs between the first/last index entry in # the input data and the last indexes we have on record from the # last time we updated the curve index. prepend_length = int(istart - x[0]) append_length = int(x[-1] - istop) - - # step mode: draw flat top discrete "step" - # over the index space for each datum. - if self._step_mode: - x_out, y_out = step_path_arrays_from_1d(x[:-1], y[:-1]) - - else: - # by default we only pull data up to the last (current) index - x_out, y_out = x[:-1], y[:-1] + no_path_yet = self.path is None if ( - self.path is None + should_redraw or should_ds + or self.path is None or prepend_length > 0 - or self._redraw ): - if self._use_poly: - self.poly = pg.functions.arrayToQPolygonF( - x_out, - y_out, + # step mode: draw flat top discrete "step" + # over the index space for each datum. + if self._step_mode: + x_out, y_out = step_path_arrays_from_1d( + x[:-1], y[:-1] ) + profiler('generated step arrays') else: - self.path = pg.functions.arrayToQPath( + # by default we only pull data up to the last (current) index + x_out, y_out = x[:-1], y[:-1] + + if should_ds: + x_out, y_out = self.downsample( x_out, y_out, - connect='all', - finiteCheck=False, - path=self.path, + **should_ds, ) - # reserve mem allocs see: - # - https://doc.qt.io/qt-5/qpainterpath.html#reserve - # - https://doc.qt.io/qt-5/qpainterpath.html#capacity - # - https://doc.qt.io/qt-5/qpainterpath.html#clear - # XXX: right now this is based on had hoc checks on a - # hidpi 3840x2160 4k monitor but we should optimize for - # the target display(s) on the sys. + profiler(f'path downsample redraw={should_ds}') + self._in_ds = True + + if should_redraw: + profiler('path reversion to non-ds') + if self.path: + self.path.clear() + + if self.fast_path: + self.fast_path.clear() + + if should_redraw and not should_ds: + log.info(f'DEDOWN -> {self._name}') + self._in_ds = False + + # else: + self.path = pg.functions.arrayToQPath( + x_out, + y_out, + connect='all', + finiteCheck=False, + path=self.path, + ) + # reserve mem allocs see: + # - https://doc.qt.io/qt-5/qpainterpath.html#reserve + # - https://doc.qt.io/qt-5/qpainterpath.html#capacity + # - https://doc.qt.io/qt-5/qpainterpath.html#clear + # XXX: right now this is based on had hoc checks on a + # hidpi 3840x2160 4k monitor but we should optimize for + # the target display(s) on the sys. + if no_path_yet: self.path.reserve(int(500e3)) - profiler('generate fresh path') - self._redraw = False + profiler('generated fresh path') # if self._step_mode: # self.path.closeSubpath() @@ -271,7 +405,9 @@ class FastAppendCurve(pg.GraphicsObject): # # self.path.moveTo(new_x[0], new_y[0]) # self.path.connectPath(old_path) - elif append_length > 0: + elif ( + append_length > 0 + ): if self._step_mode: new_x, new_y = step_path_arrays_from_1d( x[-append_length - 2:-1], @@ -290,53 +426,60 @@ class FastAppendCurve(pg.GraphicsObject): new_y = y[-append_length - 2:-1] # print((new_x, new_y)) - if self._use_poly: - union_poly = pg.functions.arrayToQPolygonF( + profiler('diffed append arrays') + + if should_ds: + new_x, new_y = self.downsample( new_x, new_y, + **should_ds, ) + profiler(f'fast path downsample redraw={should_ds}') - else: - append_path = pg.functions.arrayToQPath( - new_x, - new_y, - connect='all', - finiteCheck=False, - ) + append_path = pg.functions.arrayToQPath( + new_x, + new_y, + connect='all', + finiteCheck=False, + path=self.fast_path, + ) - # other merging ideas: - # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths - if self._step_mode: - assert not self._use_poly, 'Dunno howw this worx yet' - - # path.addPath(append_path) - self.path.connectPath(append_path) - - # TODO: try out new work from `pyqtgraph` main which - # should repair horrid perf: - # https://github.com/pyqtgraph/pyqtgraph/pull/2032 - # ok, nope still horrible XD - # if self._fill: - # # XXX: super slow set "union" op - # self.path = self.path.united(append_path).simplified() - - # # path.addPath(append_path) - # # path.closeSubpath() - - else: - if self._use_poly: - self.poly = self.poly.united(union_poly) + if self.use_fpath: + # an attempt at trying to make append-updates faster.. + if self.fast_path is None: + self.fast_path = append_path + self.fast_path.reserve(int(6e3)) else: + self.fast_path.connectPath(append_path) + size = self.fast_path.capacity() + profiler(f'connected fast path w size: {size}') + # print(f"append_path br: {append_path.boundingRect()}") # self.path.moveTo(new_x[0], new_y[0]) - self.path.connectPath(append_path) # path.connectPath(append_path) # XXX: lol this causes a hang.. # self.path = self.path.simplified() + else: + size = self.path.capacity() + profiler(f'connected history path w size: {size}') + self.path.connectPath(append_path) - self.disable_cache() - flip_cache = True + # other merging ideas: + # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths + # path.addPath(append_path) + # path.closeSubpath() + + # TODO: try out new work from `pyqtgraph` main which + # should repair horrid perf: + # https://github.com/pyqtgraph/pyqtgraph/pull/2032 + # ok, nope still horrible XD + # if self._fill: + # # XXX: super slow set "union" op + # self.path = self.path.united(append_path).simplified() + + # self.disable_cache() + # flip_cache = True # XXX: do we need this any more? # if ( @@ -345,8 +488,6 @@ class FastAppendCurve(pg.GraphicsObject): # self.disable_cache() # flip_cache = True - # print(f"update br: {self.path.boundingRect()}") - # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. self.xData = x @@ -366,6 +507,11 @@ class FastAppendCurve(pg.GraphicsObject): x_last - 0.5, 0, x_last + 0.5, y_last ) + # print( + # f"path br: {self.path.boundingRect()}", + # f"fast path br: {self.fast_path.boundingRect()}", + # f"last rect br: {self._last_step_rect}", + # ) else: # print((x[-1], y_last)) self._last_line = QLineF( @@ -373,19 +519,28 @@ class FastAppendCurve(pg.GraphicsObject): x[-1], y_last ) + profiler('draw last segment') + # trigger redraw of path # do update before reverting to cache mode - self.prepareGeometryChange() + # self.prepareGeometryChange() self.update() + profiler('.update()') if flip_cache: # XXX: seems to be needed to avoid artifacts (see above). self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + self._x, self._y = x, y + # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. def getData(self): - return self.xData, self.yData + return self._x, self._y + + # TODO: drop the above after ``Cursor`` re-work + def get_arrays(self) -> tuple[np.ndarray, np.ndarray]: + return self._x, self._y def clear(self): ''' @@ -396,14 +551,20 @@ class FastAppendCurve(pg.GraphicsObject): self.xData = None self.yData = None + # XXX: previously, if not trying to leverage `.reserve()` allocs + # then you might as well create a new one.. + # self.path = None + # path reservation aware non-mem de-alloc cleaning if self.path: self.path.clear() - self._redraw = True - # XXX: if not trying to leverage `.reserve()` allocs - # then you might as well create a new one.. - # self.path = None + if self.fast_path: + # self.fast_path.clear() + self.fast_path = None + + # self.disable_cache() + # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) def disable_cache(self) -> None: ''' @@ -419,20 +580,13 @@ class FastAppendCurve(pg.GraphicsObject): ''' Compute and then cache our rect. ''' - if self._use_poly: - if self.poly is None: - return QtGui.QPolygonF().boundingRect() - else: - br = self.boundingRect = self.poly.boundingRect - return br() + if self.path is None: + return QtGui.QPainterPath().boundingRect() else: - if self.path is None: - return QtGui.QPainterPath().boundingRect() - else: - # dynamically override this method after initial - # path is created to avoid requiring the above None check - self.boundingRect = self._path_br - return self._path_br() + # dynamically override this method after initial + # path is created to avoid requiring the above None check + self.boundingRect = self._path_br + return self._path_br() def _path_br(self): ''' @@ -441,6 +595,11 @@ class FastAppendCurve(pg.GraphicsObject): ''' hb = self.path.controlPointRect() hb_size = hb.size() + + fp = self.fast_path + if fp: + fhb = fp.controlPointRect() + hb_size = fhb.size() + hb_size # print(f'hb_size: {hb_size}') w = hb_size.width() + 1 @@ -466,6 +625,7 @@ class FastAppendCurve(pg.GraphicsObject): ) -> None: profiler = pg.debug.Profiler( + msg=f'{self._name}.paint()', disabled=not pg_profile_enabled(), ) @@ -486,14 +646,15 @@ class FastAppendCurve(pg.GraphicsObject): p.setPen(self._pen) path = self.path - if self._use_poly: - assert self.poly - p.drawPolyline(self.poly) - profiler('.drawPolyline()') - elif path: + if path: + profiler('.drawPath(path)') p.drawPath(path) - profiler('.drawPath()') + + fp = self.fast_path + if fp: + p.drawPath(fp) + profiler('.drawPath(fast_path)') # TODO: try out new work from `pyqtgraph` main which should # repair horrid perf (pretty sure i did and it was still From 6410c68e2e047170212778576976cc6500ccfacc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 1 Apr 2022 13:27:07 -0400 Subject: [PATCH 47/84] Add global profile timeout var --- piker/_profile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/_profile.py b/piker/_profile.py index ca1fc188..2e726e80 100644 --- a/piker/_profile.py +++ b/piker/_profile.py @@ -24,6 +24,7 @@ from functools import wraps # NOTE: you can pass a flag to enable this: # ``piker chart --profile``. _pg_profile: bool = False +ms_slower_then: float = 10 def pg_profile_enabled() -> bool: From 5128e4c304afb101e0b7552ae5293137f27bd0f5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 1 Apr 2022 13:28:50 -0400 Subject: [PATCH 48/84] Make `BarItems` use our line curve for downsampling Drop all the logic originally in `.update_ds_line()` which is now done internal to our `FastAppendCurve`. Add incremental update of the flattened OHLC -> line curve (unfortunately using `np.concatenate()` for the moment) and maintain a new `._ds_line_xy` arrays tuple which keeps the internal state. Add `.maybe_downsample()` as per the new interaction update method requirement. Draft out some fast path curve stuff like in our line graphic. Short-circuit bars path updates when we downsample to line. Oh, and add a ton more profiling in prep for getting all this stuff faf. --- piker/ui/_ohlc.py | 349 ++++++++++++++++++++-------------------------- 1 file changed, 152 insertions(+), 197 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index a7218fd9..2fb21c97 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -31,13 +31,11 @@ from PyQt5.QtCore import QLineF, QPointF # from numba import types as ntypes # from ..data._source import numba_ohlc_dtype -from .._profile import pg_profile_enabled +from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor from ..log import get_logger from ._curve import FastAppendCurve -from ._compression import ( - ohlc_to_m4_line, -) +from ._compression import ohlc_flatten if TYPE_CHECKING: from ._chart import LinkedSplits @@ -46,29 +44,16 @@ if TYPE_CHECKING: log = get_logger(__name__) -def _mk_lines_array( - data: list, - size: int, - elements_step: int = 6, - -) -> np.ndarray: - ''' - Create an ndarray to hold lines graphics info. - - ''' - return np.zeros_like( - data, - shape=(int(size), elements_step), - dtype=object, - ) - - -def lines_from_ohlc( +def bar_from_ohlc_row( row: np.ndarray, w: float ) -> tuple[QLineF]: + ''' + Generate the minimal ``QLineF`` lines to construct a single + OHLC "bar" for use in the "last datum" of a series. + ''' open, high, low, close, index = row[ ['open', 'high', 'low', 'close', 'index']] @@ -178,7 +163,11 @@ def gen_qpath( ) -> QtGui.QPainterPath: - profiler = pg.debug.Profiler(disabled=not pg_profile_enabled()) + profiler = pg.debug.Profiler( + msg=f'gen_qpath ohlc', + disabled=not pg_profile_enabled(), + gt=ms_slower_then, + ) x, y, c = path_arrays_from_ohlc(data, start, bar_gap=w) profiler("generate stream with numba") @@ -202,7 +191,6 @@ class BarItems(pg.GraphicsObject): def __init__( self, - # scene: 'QGraphicsScene', # noqa linked: LinkedSplits, plotitem: 'pg.PlotItem', # noqa pen_color: str = 'bracket', @@ -211,15 +199,17 @@ class BarItems(pg.GraphicsObject): name: Optional[str] = None, ) -> None: - - self.linked = linked super().__init__() + self.linked = linked # XXX: for the mega-lulz increasing width here increases draw # latency... so probably don't do it until we figure that out. self._color = pen_color self.bars_pen = pg.mkPen(hcolor(pen_color), width=1) self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2) - self._array = None + + self._ds_line_xy: Optional[ + tuple[np.ndarray, np.ndarray] + ] = None # NOTE: this prevents redraws on mouse interaction which is # a huge boon for avg interaction latency. @@ -232,6 +222,7 @@ class BarItems(pg.GraphicsObject): self._pi = plotitem self.path = QtGui.QPainterPath() + self.fast_path = QtGui.QPainterPath() self._xrange: tuple[int, int] self._yrange: tuple[float, float] @@ -246,7 +237,6 @@ class BarItems(pg.GraphicsObject): # downsampler-line state self._in_ds: bool = False - self._ds_lines: dict[int, FastAppendCurve] = {} self._ds_line: Optional[FastAppendCurve] = None self._dsi: tuple[int, int] = 0, 0 self._xs_in_px: float = 0 @@ -278,96 +268,29 @@ class BarItems(pg.GraphicsObject): ) # up to last to avoid double draw of last bar - self._last_bar_lines = lines_from_ohlc(last, self.w) + self._last_bar_lines = bar_from_ohlc_row(last, self.w) # trigger render # https://doc.qt.io/qt-5/qgraphicsitem.html#update self.update() - # self.update_ds_line(ohlc) - # assert self._ds_line - # self._ds_line.hide() - - self._array = ohlc + x, y = self._ds_line_xy = ohlc_flatten(ohlc) + self.update_ds_line(x, y) + self._ds_xrange = (index[0], index[-1]) return self.path - def get_ds_line( - self, - ds: Optional[int] = None, - - ) -> tuple[FastAppendCurve, int]: - - px_vecs = self.pixelVectors()[0] - - if not px_vecs and self._ds_line: - px_vecs = self._ds_line.pixelVectors()[0] - - if px_vecs: - xs_in_px = px_vecs.x() - ds = round(xs_in_px) - else: - ds = 0 - - return self._ds_line, ds - # return self._ds_line.get(ds), ds - def update_ds_line( self, - ohlc: np.ndarray, + x, + y, - ) -> int: + ) -> FastAppendCurve: # determine current potential downsampling value (based on pixel # scaling) and return any existing curve for it. - curve, uppx = self.get_ds_line() - # print(f'uppx: {uppx}') - - chart = self.linked.chart - if not chart: - return - else: - px_width = round(chart.curve_width_pxs()) - - if px_width == 0: - return - - # if self._ds_line: - # self._pi.removeItem(self._ds_line) - - # log.info(f'current dsi: {self._dsi}') - old_dsi = ds_uppx, ds_px_width = self._dsi - - changed = False - if ( - abs(uppx - ds_uppx) >= 4 - # or not self._in_ds - ): - changed = True - if curve: - # trigger a full redraw of the curve path since - # we have downsampled another "level" using m4. - curve.clear() - - ds_uppx, ds_px_width = dsi = (uppx, px_width) - self._dsi = dsi - - if changed: - log.info(f'sampler change: {old_dsi} -> {dsi}') - - # always refresh data bounds until we get diffing - # working properly, see above.. - x, y = ohlc_to_m4_line( - ohlc, - px_width=ds_px_width, - uppx=ds_uppx, - # pretrace=True, - - # activate m4 ds? - downsample=True, - ) + curve = self._ds_line if not curve: - # TODO: figuring out the most optimial size for the ideal # curve-path by, # - calcing the display's max px width `.screen()` @@ -381,16 +304,13 @@ class BarItems(pg.GraphicsObject): curve = FastAppendCurve( y=y, x=x, - name='ds', + name='OHLC', color=self._color, - # color='dad_blue', - # use_polyline=True, # pretty sure this is slower? ) curve.hide() self._pi.addItem(curve) - self._ds_lines[ds_uppx] = curve self._ds_line = curve - return curve, ds_uppx + return curve # TODO: we should be diffing the amount of new data which # needs to be downsampled. Ideally we actually are just @@ -403,7 +323,7 @@ class BarItems(pg.GraphicsObject): y=y, x=x, ) - return curve, ds_uppx + return curve def update_from_array( self, @@ -424,14 +344,14 @@ class BarItems(pg.GraphicsObject): This routine should be made (transitively) as fast as possible. ''' - # XXX: always do this? - # if self._ds_line: - # del self._array - self._array = ohlc - self.update_ds_line(ohlc) + profiler = pg.debug.Profiler( + disabled=not pg_profile_enabled(), + gt=ms_slower_then, + ) # index = self.start_index istart, istop = self._xrange + ds_istart, ds_istop = self._ds_xrange index = ohlc['index'] first_index, last_index = index[0], index[-1] @@ -440,52 +360,110 @@ class BarItems(pg.GraphicsObject): prepend_length = istart - first_index append_length = last_index - istop + ds_prepend_length = ds_istart - first_index + ds_append_length = last_index - ds_istop + flip_cache = False - # TODO: allow mapping only a range of lines thus - # only drawing as many bars as exactly specified. + # TODO: to make the downsampling faster + # - allow mapping only a range of lines thus only drawing as + # many bars as exactly specified. + # - move ohlc "flattening" to a shmarr + # - maybe move all this embedded logic to a higher + # level type? + + fx, fy = self._ds_line_xy if prepend_length: - # new history was added and we need to render a new path - new_bars = ohlc[:prepend_length] - prepend_path = gen_qpath(new_bars, 0, self.w) + prepend_bars = ohlc[:prepend_length] - # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path - # y value not matching the first value from - # ohlc[prepend_length + 1] ??? - - # update path - old_path = self.path - self.path = prepend_path - self.path.addPath(old_path) - - # trigger redraw despite caching - self.prepareGeometryChange() + if ds_prepend_length: + ds_prepend_bars = ohlc[:ds_prepend_length] + pre_x, pre_y = ohlc_flatten(ds_prepend_bars) + fx = np.concatenate((pre_x, fx)) + fy = np.concatenate((pre_y, fy)) + profiler('ds line prepend diff complete') if append_length: - # generate new lines objects for updatable "current bar" - self._last_bar_lines = lines_from_ohlc(ohlc[-1], self.w) - # generate new graphics to match provided array # path appending logic: # we need to get the previous "current bar(s)" for the time step # and convert it to a sub-path to append to the historical set # new_bars = ohlc[istop - 1:istop + append_length - 1] - new_bars = ohlc[-append_length - 1:-1] - append_path = gen_qpath(new_bars, 0, self.w) - self.path.moveTo(float(istop - self.w), float(new_bars[0]['open'])) + append_bars = ohlc[-append_length - 1:-1] + # print(f'ohlc bars to append size: {append_bars.size}\n') + + if ds_append_length: + ds_append_bars = ohlc[-ds_append_length - 1:-1] + post_x, post_y = ohlc_flatten(ds_append_bars) + # print(f'ds curve to append sizes: {(post_x.size, post_y.size)}') + fx = np.concatenate((fx, post_x)) + fy = np.concatenate((fy, post_y)) + + profiler('ds line append diff complete') + + profiler('array diffs complete') + + # does this work? + last = ohlc[-1] + fy[-1] = last['close'] + + # incremental update and cache line datums + self._ds_line_xy = fx, fy + + # maybe downsample to line + ds = self.maybe_downsample() + if ds: + # if we downsample to a line don't bother with + # any more path generation / updates + self._ds_xrange = first_index, last_index + profiler('downsampled to line') + return + + # path updates + if prepend_length: + # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path + # y value not matching the first value from + # ohlc[prepend_length + 1] ??? + prepend_path = gen_qpath(prepend_bars, 0, self.w) + old_path = self.path + self.path = prepend_path + self.path.addPath(old_path) + profiler('path PREPEND') + + if append_length: + append_path = gen_qpath(append_bars, 0, self.w) + + self.path.moveTo( + float(istop - self.w), + float(append_bars[0]['open']) + ) self.path.addPath(append_path) - # trigger redraw despite caching - self.prepareGeometryChange() - self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - flip_cache = True + profiler('path APPEND') + # fp = self.fast_path + # if fp is None: + # self.fast_path = append_path + + # else: + # fp.moveTo(float(istop - self.w), float(new_bars[0]['open'])) + # fp.addPath(append_path) + + # self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) + # flip_cache = True + self._xrange = first_index, last_index + # trigger redraw despite caching + self.prepareGeometryChange() + + # generate new lines objects for updatable "current bar" + self._last_bar_lines = bar_from_ohlc_row(last, self.w) + # last bar update - i, o, h, l, last, v = ohlc[-1][ + i, o, h, l, last, v = last[ ['index', 'open', 'high', 'low', 'close', 'volume'] ] # assert i == self.start_index - 1 @@ -514,7 +492,10 @@ class BarItems(pg.GraphicsObject): # now out of date / from some previous sample. It's weird # though because i've seen it do this to bars i - 3 back? + profiler('last bar set') + self.update() + profiler('.update()') if flip_cache: self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) @@ -536,7 +517,20 @@ class BarItems(pg.GraphicsObject): # apparently this a lot faster says the docs? # https://doc.qt.io/qt-5/qpainterpath.html#controlPointRect hb = self.path.controlPointRect() - hb_tl, hb_br = hb.topLeft(), hb.bottomRight() + hb_tl, hb_br = ( + hb.topLeft(), + hb.bottomRight(), + ) + + # fp = self.fast_path + # if fp: + # fhb = fp.controlPointRect() + # print((hb_tl, hb_br)) + # print(fhb) + # hb_tl, hb_br = ( + # fhb.topLeft() + hb.topLeft(), + # fhb.bottomRight() + hb.bottomRight(), + # ) # need to include last bar height or BR will be off mx_y = hb_br.y() @@ -565,7 +559,7 @@ class BarItems(pg.GraphicsObject): ) - def maybe_paint_line( + def maybe_downsample( self, x_gt: float = 2., @@ -583,54 +577,35 @@ class BarItems(pg.GraphicsObject): # this is the ``float`` value of the "number of x units" (in # view coords) that a pixel spans. - xvec = self.pixelVectors()[0] - if xvec: - xs_in_px = xvec.x() - else: - xs_in_px = self._ds_line.pixelVectors()[0].x() + xs_in_px = self._ds_line.x_uppx() linked = self.linked if ( - # xs_in_px != self._xs_in_px - self._array is not None + self._ds_line_xy is not None ): - # print('refreshing curve') - out = self.update_ds_line(self._array) - if not out: - print("NOTHING!?") - return - - curve, ds = out + curve = self.update_ds_line( + *self._ds_line_xy, + ) if ( not self._in_ds and xs_in_px >= x_gt ): # TODO: a `.ui()` log level? - log.info(f'downsampling to line graphic {linked.symbol.key}') + log.info( + f'downsampling to line graphic {linked.symbol.key}' + ) self.hide() # XXX: is this actually any faster? # self._pi.removeItem(self) - curve, ds = out self._xs_in_px = xs_in_px - # curve, ds = self.get_ds_line(ds=0) - curve.clear() - curve.update() - - curve, out = self.update_ds_line(self._array) - - # curve = self._ds_line - # assert last_curve is curve # self._pi.addItem(curve) curve.show() - curve.update() - linked.graphics_cycle() self._in_ds = True - return True elif ( self._in_ds @@ -638,10 +613,7 @@ class BarItems(pg.GraphicsObject): ): log.info(f'showing bars graphic {linked.symbol.key}') - # curve, ds = self.get_ds_line() curve = self._ds_line - # assert last_curve is curve - curve.clear() curve.hide() # self._pi.removeItem(curve) @@ -651,31 +623,9 @@ class BarItems(pg.GraphicsObject): self.update() self._in_ds = False - linked.graphics_cycle() - - return True - - # elif ( - # self._in_ds - # and self._dsi != ds - # ): - # # curve = self._ds_lines.get(ds) - # # assert self._ds_line is not curve - # if self._ds_line and self._ds_line is not curve: - # self._ds_line.hide() - - # if curve: - # # self._pi.removeItem(curve) - # curve.show() - # curve.update() - - # self._ds_line = curve - # self._dsi = ds - # linked.graphics_cycle() - # return True # no curve change - return False + return self._in_ds def paint( self, @@ -690,7 +640,7 @@ class BarItems(pg.GraphicsObject): profiler = pg.debug.Profiler( disabled=not pg_profile_enabled(), - delayed=False, + gt=ms_slower_then, ) # p.setCompositionMode(0) @@ -708,6 +658,11 @@ class BarItems(pg.GraphicsObject): p.setPen(self.bars_pen) p.drawPath(self.path) profiler('draw history path') + + # if self.fast_path: + # p.drawPath(self.fast_path) + # profiler('draw fast path') + profiler.finish() # NOTE: for testing paint frequency as throttled by display loop. From 5da9f7fdb4b67bf5e1e2db6248c3ce934edaa41d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 1 Apr 2022 13:46:37 -0400 Subject: [PATCH 49/84] Add more frequent ds steps when zooming out; use profiler gt --- piker/ui/_curve.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index a4e73194..2148eb3c 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -32,7 +32,7 @@ from PyQt5.QtCore import ( QPointF, ) -from .._profile import pg_profile_enabled +from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor from ._compression import ( # ohlc_to_m4_line, @@ -192,6 +192,10 @@ class FastAppendCurve(pg.GraphicsObject): self.update() + # TODO: probably stick this in a new parent + # type which will contain our own version of + # what ``PlotCurveItem`` had in terms of base + # functionality? A `FlowGraphic` maybe? def x_uppx(self) -> int: px_vecs = self.pixelVectors()[0] @@ -225,14 +229,17 @@ class FastAppendCurve(pg.GraphicsObject): uppx = self.x_uppx() px_width = self.px_width() - uppx_diff = abs(uppx - self._last_uppx) + # uppx_diff = abs(uppx - self._last_uppx) + uppx_diff = (uppx - self._last_uppx) self._last_uppx = uppx should_redraw: bool = False should_ds: bool = False + # print(uppx_diff) + if ( - uppx <= 4 + uppx <= 8 ): # trigger redraw or original non-downsampled data if self._in_ds: @@ -243,7 +250,8 @@ class FastAppendCurve(pg.GraphicsObject): elif ( uppx_diff >= 4 - or self._step_mode and uppx_diff >= 1 + or uppx_diff <= -2 + or self._step_mode and abs(uppx_diff) >= 1 ): log.info( f'{self._name} downsampler change: {self._last_uppx} -> {uppx}' @@ -309,6 +317,7 @@ class FastAppendCurve(pg.GraphicsObject): profiler = pg.debug.Profiler( msg=f'{self._name}.update_from_array()', disabled=not pg_profile_enabled(), + gt=ms_slower_then, ) flip_cache = False @@ -625,8 +634,9 @@ class FastAppendCurve(pg.GraphicsObject): ) -> None: profiler = pg.debug.Profiler( - msg=f'{self._name}.paint()', + msg=f'FastAppendCurve.paint(): `{self._name}`', disabled=not pg_profile_enabled(), + gt=ms_slower_then, ) if ( @@ -638,6 +648,7 @@ class FastAppendCurve(pg.GraphicsObject): # p.drawLines(*tuple(filter(bool, self._last_step_lines))) # p.drawRect(self._last_step_rect) p.fillRect(self._last_step_rect, brush) + profiler('.fillRect()') if self._last_line: p.setPen(self.last_step_pen) @@ -648,8 +659,8 @@ class FastAppendCurve(pg.GraphicsObject): path = self.path if path: - profiler('.drawPath(path)') p.drawPath(path) + profiler('.drawPath(path)') fp = self.fast_path if fp: From 5bcd6ac494ac8ace8ad31862950eff2e7cfebb67 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 1 Apr 2022 13:47:24 -0400 Subject: [PATCH 50/84] Move px width log scaling into `ds_m4()` --- piker/ui/_compression.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index 1fa9dcae..1e178036 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -199,25 +199,13 @@ def ohlc_to_m4_line( use_mxmn=pretrace, ) - if uppx: - # optionally log-scale down the "supposed pxs on screen" - # as the units-per-px (uppx) get's large. - scaler = round( - max( - # NOTE: found that a 16x px width brought greater - # detail, likely due to dpi scaling? - # px_width=px_width * 16, - 128 / (1 + math.log(uppx, 2)), - 1 - ) - ) - px_width *= scaler - if downsample: bins, x, y = ds_m4( xpts, flat, px_width=px_width, + uppx=uppx, + log_scale=bool(uppx) ) x = np.broadcast_to(x[:, None], y.shape) x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() @@ -235,6 +223,8 @@ def ds_m4( # this is the width of the data in view # in display-device-local pixel units. px_width: int, + uppx: Optional[float] = None, + log_scale: bool = True, ) -> tuple[int, np.ndarray, np.ndarray]: ''' @@ -261,6 +251,22 @@ def ds_m4( # "i didn't show it in the sample code, but it's accounted for # in the start and end indices and number of bins" + # optionally log-scale down the "supposed pxs on screen" + # as the units-per-px (uppx) get's large. + if log_scale: + assert uppx, 'You must provide a `uppx` value to use log scaling!' + + scaler = round( + max( + # NOTE: found that a 16x px width brought greater + # detail, likely due to dpi scaling? + # px_width=px_width * 16, + 2**6 / (1 + math.log(uppx, 2)), + 1 + ) + ) + px_width *= scaler + assert px_width > 1 # width of screen in pxs? # NOTE: if we didn't pre-slice the data to downsample @@ -320,7 +326,7 @@ def ds_m4( @jit( nopython=True, - # nogil=True, + nogil=True, ) def _m4( From 49c25eeef405cfa959c09b84d2c36fb5e0aaac8e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 1 Apr 2022 13:49:42 -0400 Subject: [PATCH 51/84] Index must be int bro.. --- piker/ui/_cursor.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/piker/ui/_cursor.py b/piker/ui/_cursor.py index e006858e..ad3ddc98 100644 --- a/piker/ui/_cursor.py +++ b/piker/ui/_cursor.py @@ -95,22 +95,24 @@ class LineDot(pg.CurvePoint): def event( self, - ev: QtCore.QEvent, - ) -> None: + ) -> bool: if not isinstance( ev, QtCore.QDynamicPropertyChangeEvent ) or self.curve() is None: return False + # TODO: get rid of this ``.getData()`` and + # make a more pythonic api to retreive backing + # numpy arrays... (x, y) = self.curve().getData() index = self.property('index') # first = self._plot._arrays['ohlc'][0]['index'] # first = x[0] # i = index - first if index: - i = index - x[0] + i = round(index - x[0]) if i > 0 and i < len(y): newPos = (index, y[i]) QtWidgets.QGraphicsItem.setPos(self, *newPos) @@ -405,6 +407,7 @@ class Cursor(pg.GraphicsObject): slot=self.mouseMoved, delay=_debounce_delay, ) + px_enter = pg.SignalProxy( plot.sig_mouse_enter, rateLimit=_mouse_rate_limit, From 1cf6ba789cbe7ef790e8fd0a6b0a80f0ceb6462d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 1 Apr 2022 13:51:21 -0400 Subject: [PATCH 52/84] Remove units vlm cuve once the $vlm one comes up --- piker/ui/_fsp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index a1193327..95823948 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -246,7 +246,6 @@ async def run_fsp_ui( overlay=True, color='default_light', array_key=name, - separate_axes=conf.get('separate_axes', False), **conf.get('chart_kwargs', {}) ) # specially store ref to shm for lookup in display loop @@ -769,6 +768,7 @@ async def open_vlm_displays( # displayed and the curves are effectively the same minus # liquidity events (well at least on low OHLC periods - 1s). vlm_curve.hide() + chart.removeItem(vlm_curve) # use slightly less light (then bracket) gray # for volume from "main exchange" and a more "bluey" From 3b90b1f9602e55b54fc79335c1f2d0370463a0af Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 3 Apr 2022 23:52:09 -0400 Subject: [PATCH 53/84] Unify into a single update method: `.update_graphics_from_array()` --- piker/ui/_chart.py | 103 +++++++++++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 41 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index eda53bd5..354faf50 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -838,8 +838,12 @@ class ChartPlotWidget(pg.PlotWidget): ''' l, r = self.view_range() array = self._arrays[self.name] - lbar = max(l, array[0]['index']) - rbar = min(r, array[-1]['index']) + start, stop = self._xrange = ( + array[0]['index'], + array[-1]['index'], + ) + lbar = max(l, start) + rbar = min(r, stop) return l, lbar, rbar, r def curve_width_pxs( @@ -907,7 +911,7 @@ class ChartPlotWidget(pg.PlotWidget): return xfirst, xlast = index[0], index[-1] - brange = l, lbar, rbar, r = self.bars_range() + l, lbar, rbar, r = self.bars_range() marker_pos, l1_len = self.pre_l1_xs() end = xlast + l1_len + 1 @@ -986,7 +990,8 @@ class ChartPlotWidget(pg.PlotWidget): graphics = BarItems( self.linked, self.plotItem, - pen_color=self.pen_color + pen_color=self.pen_color, + name=name, ) # adds all bar/candle graphics objects for each data point in @@ -1175,29 +1180,13 @@ class ChartPlotWidget(pg.PlotWidget): ) return last - def update_ohlc_from_array( + def update_graphics_from_array( self, - graphics_name: str, - array: np.ndarray, - **kwargs, - ) -> pg.GraphicsObject: - ''' - Update the named internal graphics from ``array``. - - ''' - self._arrays[self.name] = array - graphics = self._graphics[graphics_name] - graphics.update_from_array(array, **kwargs) - return graphics - - def update_curve_from_array( - self, - - graphics_name: str, - array: np.ndarray, + array: Optional[np.ndarray] = None, array_key: Optional[str] = None, + **kwargs, ) -> pg.GraphicsObject: @@ -1205,31 +1194,60 @@ class ChartPlotWidget(pg.PlotWidget): Update the named internal graphics from ``array``. ''' - assert len(array) + if array is not None: + assert len(array) + data_key = array_key or graphics_name - if graphics_name not in self._flows: - self._arrays[self.name] = array - else: + data_key = self.name + + if array is not None: + # write array to internal graphics table self._arrays[data_key] = array + else: + array = self._arrays[data_key] - curve = self._graphics[graphics_name] + # array key and graphics "name" might be different.. + graphics = self._graphics[graphics_name] - # NOTE: back when we weren't implementing the curve graphics - # ourselves you'd have updates using this method: - # curve.setData(y=array[graphics_name], x=array['index'], **kwargs) + # compute "in-view" indices + l, lbar, rbar, r = self.bars_range() + indexes = array['index'] + ifirst = indexes[0] + ilast = indexes[-1] - # NOTE: graphics **must** implement a diff based update - # operation where an internal ``FastUpdateCurve._xrange`` is - # used to determine if the underlying path needs to be - # pre/ap-pended. - curve.update_from_array( - x=array['index'], - y=array[data_key], - **kwargs - ) + lbar_i = max(l, ifirst) - ifirst + rbar_i = min(r, ilast) - ifirst - return curve + # TODO: we could do it this way as well no? + # to_draw = array[lbar - ifirst:(rbar - ifirst) + 1] + in_view = array[lbar_i: rbar_i] + + if not in_view.size: + return graphics + + if isinstance(graphics, BarItems): + graphics.update_from_array( + array, + in_view, + view_range=(lbar_i, rbar_i), + + **kwargs, + ) + + else: + graphics.update_from_array( + x=array['index'], + y=array[data_key], + + x_iv=in_view['index'], + y_iv=in_view[data_key], + view_range=(lbar_i, rbar_i), + + **kwargs + ) + + return graphics # def _label_h(self, yhigh: float, ylow: float) -> float: # # compute contents label "height" in view terms @@ -1260,6 +1278,9 @@ class ChartPlotWidget(pg.PlotWidget): # print(f"bounds (ylow, yhigh): {(ylow, yhigh)}") + # TODO: pretty sure we can just call the cursor + # directly not? i don't wee why we need special "signal proxies" + # for this lul.. def enterEvent(self, ev): # noqa # pg.PlotWidget.enterEvent(self, ev) self.sig_mouse_enter.emit(self) From ef03b8e987e9706a1a753b682cfa0c6c476005b4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 00:10:13 -0400 Subject: [PATCH 54/84] Attempt only rendering ohlc bars in view and ds-ing otherwise --- piker/ui/_ohlc.py | 423 ++++++++++++++++++++++++---------------------- 1 file changed, 220 insertions(+), 203 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 2fb21c97..482d6135 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -157,23 +157,40 @@ def path_arrays_from_ohlc( def gen_qpath( - data, - start, # XXX: do we need this? - w, + data: np.ndarray, + start: int, # XXX: do we need this? + w: float, + path: Optional[QtGui.QPainterPath] = None, ) -> QtGui.QPainterPath: + path_was_none = path is None + profiler = pg.debug.Profiler( - msg=f'gen_qpath ohlc', + msg='gen_qpath ohlc', disabled=not pg_profile_enabled(), gt=ms_slower_then, ) - x, y, c = path_arrays_from_ohlc(data, start, bar_gap=w) + x, y, c = path_arrays_from_ohlc( + data, + start, + bar_gap=w, + ) profiler("generate stream with numba") # TODO: numba the internals of this! - path = pg.functions.arrayToQPath(x, y, connect=c) + path = pg.functions.arrayToQPath( + x, + y, + connect=c, + path=path, + ) + + # avoid mem allocs if possible + if path_was_none: + path.reserve(path.capacity()) + profiler("generate path with arrayToQPath") return path @@ -206,6 +223,7 @@ class BarItems(pg.GraphicsObject): self._color = pen_color self.bars_pen = pg.mkPen(hcolor(pen_color), width=1) self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2) + self._name = name self._ds_line_xy: Optional[ tuple[np.ndarray, np.ndarray] @@ -226,6 +244,7 @@ class BarItems(pg.GraphicsObject): self._xrange: tuple[int, int] self._yrange: tuple[float, float] + self._vrange = None # TODO: don't render the full backing array each time # self._path_data = None @@ -254,7 +273,6 @@ class BarItems(pg.GraphicsObject): ''' hist, last = ohlc[:-1], ohlc[-1] - self.path = gen_qpath(hist, start, self.w) # save graphics for later reference and keep track @@ -270,65 +288,46 @@ class BarItems(pg.GraphicsObject): # up to last to avoid double draw of last bar self._last_bar_lines = bar_from_ohlc_row(last, self.w) + x, y = self._ds_line_xy = ohlc_flatten(ohlc) + + # TODO: figuring out the most optimial size for the ideal + # curve-path by, + # - calcing the display's max px width `.screen()` + # - drawing a curve and figuring out it's capacity: + # https://doc.qt.io/qt-5/qpainterpath.html#capacity + # - reserving that cap for each curve-mapped-to-shm with + + # - leveraging clearing when needed to redraw the entire + # curve that does not release mem allocs: + # https://doc.qt.io/qt-5/qpainterpath.html#clear + curve = FastAppendCurve( + y=y, + x=x, + name='OHLC', + color=self._color, + ) + curve.hide() + self._pi.addItem(curve) + self._ds_line = curve + + self._ds_xrange = (index[0], index[-1]) + # trigger render # https://doc.qt.io/qt-5/qgraphicsitem.html#update self.update() - x, y = self._ds_line_xy = ohlc_flatten(ohlc) - self.update_ds_line(x, y) - self._ds_xrange = (index[0], index[-1]) return self.path - def update_ds_line( - self, - x, - y, - - ) -> FastAppendCurve: - - # determine current potential downsampling value (based on pixel - # scaling) and return any existing curve for it. - curve = self._ds_line - - if not curve: - # TODO: figuring out the most optimial size for the ideal - # curve-path by, - # - calcing the display's max px width `.screen()` - # - drawing a curve and figuring out it's capacity: - # https://doc.qt.io/qt-5/qpainterpath.html#capacity - # - reserving that cap for each curve-mapped-to-shm with - - # - leveraging clearing when needed to redraw the entire - # curve that does not release mem allocs: - # https://doc.qt.io/qt-5/qpainterpath.html#clear - curve = FastAppendCurve( - y=y, - x=x, - name='OHLC', - color=self._color, - ) - curve.hide() - self._pi.addItem(curve) - self._ds_line = curve - return curve - - # TODO: we should be diffing the amount of new data which - # needs to be downsampled. Ideally we actually are just - # doing all the ds-ing in sibling actors so that the data - # can just be read and rendered to graphics on events of our - # choice. - # diff = do_diff(ohlc, new_bit) - - curve.update_from_array( - y=y, - x=x, - ) - return curve - def update_from_array( self, + + # full array input history ohlc: np.ndarray, - just_history=False, + + # pre-sliced array data that's "in view" + ohlc_iv: np.ndarray, + + view_range: Optional[tuple[int, int]] = None, ) -> None: ''' @@ -357,14 +356,102 @@ class BarItems(pg.GraphicsObject): first_index, last_index = index[0], index[-1] # length = len(ohlc) - prepend_length = istart - first_index - append_length = last_index - istop + # prepend_length = istart - first_index + # append_length = last_index - istop - ds_prepend_length = ds_istart - first_index - ds_append_length = last_index - ds_istop + # ds_prepend_length = ds_istart - first_index + # ds_append_length = last_index - ds_istop flip_cache = False + x_gt = 2 + if self._ds_line: + uppx = self._ds_line.x_uppx() + else: + uppx = 0 + + should_line = self._in_ds + if ( + self._in_ds + and uppx < x_gt + ): + should_line = False + + elif ( + not self._in_ds + and uppx >= x_gt + ): + should_line = True + + if ( + should_line + ): + # update the line graphic + # x, y = self._ds_line_xy = ohlc_flatten(ohlc_iv) + x, y = self._ds_line_xy = ohlc_flatten(ohlc) + x_iv, y_iv = self._ds_line_xy = ohlc_flatten(ohlc_iv) + profiler('flattening bars to line') + + # TODO: we should be diffing the amount of new data which + # needs to be downsampled. Ideally we actually are just + # doing all the ds-ing in sibling actors so that the data + # can just be read and rendered to graphics on events of our + # choice. + # diff = do_diff(ohlc, new_bit) + curve = self._ds_line + curve.update_from_array( + x=x, + y=y, + x_iv=x_iv, + y_iv=y_iv, + view_range=view_range, # hack + ) + profiler('udated ds line') + + if not self._in_ds: + # hide bars and show line + self.hide() + # XXX: is this actually any faster? + # self._pi.removeItem(self) + + # TODO: a `.ui()` log level? + log.info( + f'downsampling to line graphic {self._name}' + ) + + # self._pi.addItem(curve) + curve.show() + curve.update() + self._in_ds = True + + # stop here since we don't need to update bars path any more + # as we delegate to the downsample line with updates. + return + + elif ( + not should_line + and self._in_ds + ): + # flip back to bars graphics and hide the downsample line. + log.info(f'showing bars graphic {self._name}') + + curve = self._ds_line + curve.hide() + # self._pi.removeItem(curve) + + # XXX: is this actually any faster? + # self._pi.addItem(self) + self.show() + self._in_ds = False + + # generate in_view path + self.path = gen_qpath( + ohlc_iv, + 0, + self.w, + # path=self.path, + ) + # TODO: to make the downsampling faster # - allow mapping only a range of lines thus only drawing as # many bars as exactly specified. @@ -372,87 +459,97 @@ class BarItems(pg.GraphicsObject): # - maybe move all this embedded logic to a higher # level type? - fx, fy = self._ds_line_xy + # ohlc = in_view - if prepend_length: - # new history was added and we need to render a new path - prepend_bars = ohlc[:prepend_length] + # if prepend_length: + # # new history was added and we need to render a new path + # prepend_bars = ohlc[:prepend_length] - if ds_prepend_length: - ds_prepend_bars = ohlc[:ds_prepend_length] - pre_x, pre_y = ohlc_flatten(ds_prepend_bars) - fx = np.concatenate((pre_x, fx)) - fy = np.concatenate((pre_y, fy)) - profiler('ds line prepend diff complete') + # if ds_prepend_length: + # ds_prepend_bars = ohlc[:ds_prepend_length] + # pre_x, pre_y = ohlc_flatten(ds_prepend_bars) + # fx = np.concatenate((pre_x, fx)) + # fy = np.concatenate((pre_y, fy)) + # profiler('ds line prepend diff complete') - if append_length: - # generate new graphics to match provided array - # path appending logic: - # we need to get the previous "current bar(s)" for the time step - # and convert it to a sub-path to append to the historical set - # new_bars = ohlc[istop - 1:istop + append_length - 1] - append_bars = ohlc[-append_length - 1:-1] - # print(f'ohlc bars to append size: {append_bars.size}\n') + # if append_length: + # # generate new graphics to match provided array + # # path appending logic: + # # we need to get the previous "current bar(s)" for the time step + # # and convert it to a sub-path to append to the historical set + # # new_bars = ohlc[istop - 1:istop + append_length - 1] + # append_bars = ohlc[-append_length - 1:-1] + # # print(f'ohlc bars to append size: {append_bars.size}\n') - if ds_append_length: - ds_append_bars = ohlc[-ds_append_length - 1:-1] - post_x, post_y = ohlc_flatten(ds_append_bars) - # print(f'ds curve to append sizes: {(post_x.size, post_y.size)}') - fx = np.concatenate((fx, post_x)) - fy = np.concatenate((fy, post_y)) + # if ds_append_length: + # ds_append_bars = ohlc[-ds_append_length - 1:-1] + # post_x, post_y = ohlc_flatten(ds_append_bars) + # print( + # f'ds curve to append sizes: {(post_x.size, post_y.size)}' + # ) + # fx = np.concatenate((fx, post_x)) + # fy = np.concatenate((fy, post_y)) - profiler('ds line append diff complete') + # profiler('ds line append diff complete') profiler('array diffs complete') # does this work? last = ohlc[-1] - fy[-1] = last['close'] + # fy[-1] = last['close'] - # incremental update and cache line datums - self._ds_line_xy = fx, fy + # # incremental update and cache line datums + # self._ds_line_xy = fx, fy # maybe downsample to line - ds = self.maybe_downsample() - if ds: - # if we downsample to a line don't bother with - # any more path generation / updates - self._ds_xrange = first_index, last_index - profiler('downsampled to line') - return + # ds = self.maybe_downsample() + # if ds: + # # if we downsample to a line don't bother with + # # any more path generation / updates + # self._ds_xrange = first_index, last_index + # profiler('downsampled to line') + # return + + # print(in_view.size) + + # if self.path: + # self.path = path + # self.path.reserve(path.capacity()) + # self.path.swap(path) # path updates - if prepend_length: - # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path - # y value not matching the first value from - # ohlc[prepend_length + 1] ??? - prepend_path = gen_qpath(prepend_bars, 0, self.w) - old_path = self.path - self.path = prepend_path - self.path.addPath(old_path) - profiler('path PREPEND') + # if prepend_length: + # # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path + # # y value not matching the first value from + # # ohlc[prepend_length + 1] ??? + # prepend_path = gen_qpath(prepend_bars, 0, self.w) + # old_path = self.path + # self.path = prepend_path + # self.path.addPath(old_path) + # profiler('path PREPEND') - if append_length: - append_path = gen_qpath(append_bars, 0, self.w) + # if append_length: + # append_path = gen_qpath(append_bars, 0, self.w) - self.path.moveTo( - float(istop - self.w), - float(append_bars[0]['open']) - ) - self.path.addPath(append_path) + # self.path.moveTo( + # float(istop - self.w), + # float(append_bars[0]['open']) + # ) + # self.path.addPath(append_path) - profiler('path APPEND') - # fp = self.fast_path - # if fp is None: - # self.fast_path = append_path + # profiler('path APPEND') + # fp = self.fast_path + # if fp is None: + # self.fast_path = append_path - # else: - # fp.moveTo(float(istop - self.w), float(new_bars[0]['open'])) - # fp.addPath(append_path) - - # self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # flip_cache = True + # else: + # fp.moveTo( + # float(istop - self.w), float(new_bars[0]['open']) + # ) + # fp.addPath(append_path) + # self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) + # flip_cache = True self._xrange = first_index, last_index @@ -559,74 +656,6 @@ class BarItems(pg.GraphicsObject): ) - def maybe_downsample( - self, - x_gt: float = 2., - - ) -> bool: - ''' - Call this when you want to stop drawing individual - bars and instead use a ``FastAppendCurve`` intepolation - line (normally when the width of a bar (aka 1.0 in the x) - is less then a pixel width on the device). - - ''' - curve = self._ds_line - if not curve: - return False - - # this is the ``float`` value of the "number of x units" (in - # view coords) that a pixel spans. - xs_in_px = self._ds_line.x_uppx() - - linked = self.linked - - if ( - self._ds_line_xy is not None - ): - curve = self.update_ds_line( - *self._ds_line_xy, - ) - - if ( - not self._in_ds - and xs_in_px >= x_gt - ): - # TODO: a `.ui()` log level? - log.info( - f'downsampling to line graphic {linked.symbol.key}' - ) - self.hide() - # XXX: is this actually any faster? - # self._pi.removeItem(self) - - self._xs_in_px = xs_in_px - - # self._pi.addItem(curve) - curve.show() - - self._in_ds = True - - elif ( - self._in_ds - and xs_in_px < x_gt - ): - log.info(f'showing bars graphic {linked.symbol.key}') - - curve = self._ds_line - curve.hide() - # self._pi.removeItem(curve) - - # XXX: is this actually any faster? - # self._pi.addItem(self) - self.show() - self.update() - - self._in_ds = False - - # no curve change - return self._in_ds - def paint( self, p: QtGui.QPainter, @@ -657,20 +686,8 @@ class BarItems(pg.GraphicsObject): p.setPen(self.bars_pen) p.drawPath(self.path) - profiler('draw history path') + profiler(f'draw history path: {self.path.capacity()}') # if self.fast_path: # p.drawPath(self.fast_path) # profiler('draw fast path') - - profiler.finish() - - # NOTE: for testing paint frequency as throttled by display loop. - # now = time.time() - # global _last_draw - # print(f'DRAW RATE {1/(now - _last_draw)}') - # _last_draw = now - - -# import time -# _last_draw: float = time.time() From a9e1c6c50edbc5c1ba02c987a664b177620eed1d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 00:35:32 -0400 Subject: [PATCH 55/84] Make panning pause feeds, call into update method from downsampler cb loop --- piker/ui/_interaction.py | 80 ++++++++++++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index faf45c4b..adc551dc 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -20,6 +20,7 @@ Chart view box primitives """ from __future__ import annotations from contextlib import asynccontextmanager +# import itertools import time from typing import Optional, Callable @@ -36,7 +37,7 @@ from ..log import get_logger from ._style import _min_points_to_show from ._editors import SelectRect from . import _event -from ._ohlc import BarItems +# from ._ohlc import BarItems log = get_logger(__name__) @@ -319,6 +320,7 @@ async def handle_viewmode_mouse( ): # when in order mode, submit execution # msg.event.accept() + # breakpoint() view.order_mode.submit_order() @@ -384,6 +386,29 @@ class ChartView(ViewBox): self.order_mode: bool = False self.setFocusPolicy(QtCore.Qt.StrongFocus) + self._ic = None + + def start_ic( + self, + ) -> None: + if self._ic is None: + self.chart.pause_all_feeds() + self._ic = trio.Event() + + def signal_ic( + self, + *args, + # ev = None, + ) -> None: + if args: + print(f'range change dun: {args}') + else: + print('proxy called') + + if self._ic: + self._ic.set() + self._ic = None + self.chart.resume_all_feeds() @asynccontextmanager async def open_async_input_handler( @@ -429,11 +454,6 @@ class ChartView(ViewBox): def maxmin(self, callback: Callable) -> None: self._maxmin = callback - def maybe_downsample_graphics(self): - for graphic in self._chart._graphics.values(): - if isinstance(graphic, BarItems): - graphic.maybe_paint_line() - def wheelEvent( self, ev, @@ -542,6 +562,11 @@ class ChartView(ViewBox): self._resetTarget() self.scaleBy(s, focal) self.sigRangeChangedManually.emit(mask) + + # self._ic.set() + # self._ic = None + # self.chart.resume_all_feeds() + ev.accept() def mouseDragEvent( @@ -624,6 +649,11 @@ class ChartView(ViewBox): # XXX: WHY ev.accept() + self.start_ic() + # if self._ic is None: + # self.chart.pause_all_feeds() + # self._ic = trio.Event() + if axis == 1: self.chart._static_yrange = 'axis' @@ -641,6 +671,13 @@ class ChartView(ViewBox): self.sigRangeChangedManually.emit(self.state['mouseEnabled']) + if ev.isFinish(): + print('DRAG FINISH') + self.signal_ic() + # self._ic.set() + # self._ic = None + # self.chart.resume_all_feeds() + # WEIRD "RIGHT-CLICK CENTER ZOOM" MODE elif button & QtCore.Qt.RightButton: @@ -788,11 +825,13 @@ class ChartView(ViewBox): # iterate those. # - only register this when certain downsampleable graphics are # "added to scene". - vb.sigRangeChangedManually.connect(vb.maybe_downsample_graphics) + vb.sigXRangeChanged.connect(vb.maybe_downsample_graphics) # mouse wheel doesn't emit XRangeChanged vb.sigRangeChangedManually.connect(vb._set_yrange) - vb.sigResized.connect(vb._set_yrange) # splitter(s) resizing + + # splitter(s) resizing + vb.sigResized.connect(vb._set_yrange) def disable_auto_yrange( self, @@ -808,10 +847,27 @@ class ChartView(ViewBox): ''' for graphic in self._chart._graphics.values(): - # if isinstance(graphic, BarItems): - xpx = graphic.pixelVectors()[0].x() - if xpx: - return xpx + xvec = graphic.pixelVectors()[0] + if xvec: + xpx = xvec.x() + if xpx: + return xpx else: continue return 1.0 + + def maybe_downsample_graphics(self): + + # TODO: a faster single-loop-iterator way of doing this XD + chart = self._chart + # graphics = list(self._chart._graphics.values()) + + for name, graphics in chart._graphics.items(): + # pass in no array which will read and render from the last + # passed array (normally provided by the display loop.) + chart.update_graphics_from_array(name) + + # for graphic in graphics: + # ds_meth = getattr(graphic, 'maybe_downsample', None) + # if ds_meth: + # ds_meth() From 2c1daab9904a4de92772d2fa18616166c60d3fc8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 3 Apr 2022 23:34:55 -0400 Subject: [PATCH 56/84] Port to new `.update_graphics_from_array()`, pause quote updates on chart interaction --- piker/ui/_display.py | 23 +++++++++++++++++++---- piker/ui/_fsp.py | 5 +++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 70c4b49d..042b046f 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -53,6 +53,10 @@ from ._forms import ( mk_order_pane_layout, ) from .order_mode import open_order_mode +# from .._profile import ( +# pg_profile_enabled, +# ms_slower_then, +# ) from ..log import get_logger log = get_logger(__name__) @@ -284,6 +288,12 @@ async def graphics_update_loop( chart.pause_all_feeds() continue + ic = chart.view._ic + if ic: + chart.pause_all_feeds() + await ic.wait() + chart.resume_all_feeds() + # sync call to update all graphics/UX components. graphics_update_cycle(ds) @@ -300,8 +310,9 @@ def graphics_update_cycle( profiler = pg.debug.Profiler( disabled=True, # not pg_profile_enabled(), - delayed=False, + gt=1/12 * 1e3, ) + # unpack multi-referenced components chart = ds.chart vlm_chart = ds.vlm_chart @@ -421,8 +432,11 @@ def graphics_update_cycle( if ( (xpx < update_uppx or i_diff > 0) or trigger_all + and r >= i_step ): - vlm_chart.update_curve_from_array('volume', array) + # TODO: make it so this doesn't have to be called + # once the $vlm is up? + vlm_chart.update_graphics_from_array('volume', array) if ( mx_vlm_in_view != vars['last_mx_vlm'] @@ -495,7 +509,8 @@ def graphics_update_cycle( xpx < update_uppx or i_diff > 0 ): - chart.update_ohlc_from_array( + # chart.update_ohlc_from_array( + chart.update_graphics_from_array( chart.name, array, ) @@ -534,7 +549,7 @@ def graphics_update_cycle( if wap_in_history: # update vwap overlay line - chart.update_curve_from_array( + chart.update_graphics_from_array( 'bar_wap', array, ) diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 95823948..5c656a2d 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -89,7 +89,7 @@ def update_fsp_chart( # update graphics # NOTE: this does a length check internally which allows it # staying above the last row check below.. - chart.update_curve_from_array( + chart.update_graphics_from_array( graphics_name, array, array_key=array_key or graphics_name, @@ -425,6 +425,7 @@ class FspAdmin: ) as (ctx, last_index), ctx.open_stream() as stream, ): + # register output data self._registry[ (fqsn, ns_path) @@ -678,7 +679,7 @@ async def open_vlm_displays( last_val_sticky.update_from_data(-1, value) - vlm_curve = chart.update_curve_from_array( + vlm_curve = chart.update_graphics_from_array( 'volume', shm.array, ) From 82732e3f17dcb0a5829f3dce5e6294d33db728b9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 3 Apr 2022 23:36:30 -0400 Subject: [PATCH 57/84] TOQUASH: drop display loop old .update_ohlc_.. --- piker/ui/_display.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 042b046f..43597f27 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -509,7 +509,6 @@ def graphics_update_cycle( xpx < update_uppx or i_diff > 0 ): - # chart.update_ohlc_from_array( chart.update_graphics_from_array( chart.name, array, From fdd5aa33d201adfdba0706d7ba66764b98403d34 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 10:14:46 -0400 Subject: [PATCH 58/84] Fix view range array to include most recent (facepalm) --- piker/ui/_chart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 354faf50..d8e90ec3 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1221,7 +1221,7 @@ class ChartPlotWidget(pg.PlotWidget): # TODO: we could do it this way as well no? # to_draw = array[lbar - ifirst:(rbar - ifirst) + 1] - in_view = array[lbar_i: rbar_i] + in_view = array[lbar_i: rbar_i + 1] if not in_view.size: return graphics From b6f852e0adc412085e147eebf6d497f12a1cbd28 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 10:20:10 -0400 Subject: [PATCH 59/84] Make `FastAppendCurve` optionally view range aware As with the `BarItems` graphics, this makes it possible to pass in a "in view" range of array data that can be *only* rendered improving performance for large(r) data sets. All the other normal behaviour is kept (i.e a persistent, (pre/ap)pendable path can still be maintained) if a ``view_range`` is not provided. Further updates, - drop the `.should_ds_or_redraw()` and `.maybe_downsample()` predicates instead moving all that logic inside `.update_from_array()`. - disable the "cache flipping", which doesn't seem to be needed to avoid artifacts any more? - handle all redraw/dowsampling logic in `.update_from_array()`. - even more profiling. - drop path `.reserve()` stuff until we better figure out how it's supposed to work. --- piker/ui/_curve.py | 244 ++++++++++++++++++++++++--------------------- 1 file changed, 131 insertions(+), 113 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 2148eb3c..59dbc0e5 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -214,6 +214,9 @@ class FastAppendCurve(pg.GraphicsObject): vr = self.viewRect() l, r = int(vr.left()), int(vr.right()) + if not self._xrange: + return 0 + start, stop = self._xrange lbar = max(l, start) rbar = min(r, stop) @@ -222,45 +225,6 @@ class FastAppendCurve(pg.GraphicsObject): QLineF(lbar, 0, rbar, 0) ).length() - def should_ds_or_redraw( - self, - - ) -> tuple[bool, bool]: - - uppx = self.x_uppx() - px_width = self.px_width() - # uppx_diff = abs(uppx - self._last_uppx) - uppx_diff = (uppx - self._last_uppx) - self._last_uppx = uppx - - should_redraw: bool = False - should_ds: bool = False - - # print(uppx_diff) - - if ( - uppx <= 8 - ): - # trigger redraw or original non-downsampled data - if self._in_ds: - print('REVERTING BACK TO SRC DATA') - # clear downsampled curve(s) and expect - # refresh of path segments. - should_redraw = True - - elif ( - uppx_diff >= 4 - or uppx_diff <= -2 - or self._step_mode and abs(uppx_diff) >= 1 - ): - log.info( - f'{self._name} downsampler change: {self._last_uppx} -> {uppx}' - ) - should_ds = {'px_width': px_width, 'uppx': uppx} - should_redraw = True - - return should_ds, should_redraw - def downsample( self, x, @@ -289,23 +253,19 @@ class FastAppendCurve(pg.GraphicsObject): self._in_ds = True return x, y - def maybe_downsample( - self, - ) -> None: - ''' - Simple update call but with previously cached arrays data. - - ''' - # print('DS CALLED FROM INTERACTION?') - # presume this is a so called "interaction update", see - # ``ChartView.maybe_downsample_graphics()``. - self.update_from_array(self._x, self._y) - def update_from_array( self, + + # full array input history x: np.ndarray, y: np.ndarray, + # pre-sliced array data that's "in view" + x_iv: np.ndarray, + y_iv: np.ndarray, + + view_range: Optional[tuple[int, int]] = None, + ) -> QtGui.QPainterPath: ''' Update curve from input 2-d data. @@ -315,11 +275,11 @@ class FastAppendCurve(pg.GraphicsObject): ''' profiler = pg.debug.Profiler( - msg=f'{self._name}.update_from_array()', + msg=f'FastAppendCurve.update_from_array(): `{self._name}`', disabled=not pg_profile_enabled(), gt=ms_slower_then, ) - flip_cache = False + # flip_cache = False if self._xrange: istart, istop = self._xrange @@ -327,40 +287,110 @@ class FastAppendCurve(pg.GraphicsObject): self._xrange = istart, istop = x[0], x[-1] # print(f"xrange: {self._xrange}") - should_ds, should_redraw = self.should_ds_or_redraw() + # XXX: lol brutal, the internals of `CurvePoint` (inherited by + # our `LineDot`) required ``.getData()`` to work.. + self.xData = x + self.yData = y + self._x, self._y = x, y + + if view_range: + profiler(f'view range slice {view_range}') + + # downsampling incremental state checking + uppx = self.x_uppx() + px_width = self.px_width() + uppx_diff = (uppx - self._last_uppx) + + should_ds = False + should_redraw = False + + # if a view range is passed, plan to draw the + # source ouput that's "in view" of the chart. + if view_range and not self._in_ds: + # print(f'{self._name} vr: {view_range}') + + # by default we only pull data up to the last (current) index + x_out, y_out = x_iv[:-1], y_iv[:-1] + + # step mode: draw flat top discrete "step" + # over the index space for each datum. + if self._step_mode: + # TODO: numba this bish + x_out, y_out = step_path_arrays_from_1d( + x_out, + y_out + ) + profiler('generated step arrays') + + should_redraw = True + profiler('sliced in-view array history') + + # x_last = x_iv[-1] + # y_last = y_iv[-1] + self._last_vr = view_range + + # self.disable_cache() + # flip_cache = True + + else: + self._xrange = x[0], x[-1] + + x_last = x[-1] + y_last = y[-1] + + # check for downsampling conditions + if ( + # std m4 downsample conditions + uppx_diff >= 2 + or uppx_diff <= -2 + or self._step_mode and abs(uppx_diff) >= 2 + + ): + log.info( + f'{self._name} sampler change: {self._last_uppx} -> {uppx}' + ) + self._last_uppx = uppx + should_ds = {'px_width': px_width, 'uppx': uppx} + + elif ( + uppx <= 2 + and self._in_ds + ): + # we should de-downsample back to our original + # source data so we clear our path data in prep + # to generate a new one from original source data. + should_redraw = True + should_ds = False # compute the length diffs between the first/last index entry in # the input data and the last indexes we have on record from the # last time we updated the curve index. prepend_length = int(istart - x[0]) append_length = int(x[-1] - istop) - no_path_yet = self.path is None + # no_path_yet = self.path is None if ( - should_redraw or should_ds - or self.path is None + self.path is None + or should_redraw + or should_ds or prepend_length > 0 ): - # step mode: draw flat top discrete "step" - # over the index space for each datum. - if self._step_mode: - x_out, y_out = step_path_arrays_from_1d( - x[:-1], y[:-1] - ) - profiler('generated step arrays') - - else: + if ( + not view_range + or self._in_ds + ): # by default we only pull data up to the last (current) index x_out, y_out = x[:-1], y[:-1] - if should_ds: - x_out, y_out = self.downsample( - x_out, - y_out, - **should_ds, - ) - profiler(f'path downsample redraw={should_ds}') - self._in_ds = True + # step mode: draw flat top discrete "step" + # over the index space for each datum. + if self._step_mode: + x_out, y_out = step_path_arrays_from_1d( + x_out, + y_out, + ) + # TODO: numba this bish + profiler('generated step arrays') if should_redraw: profiler('path reversion to non-ds') @@ -371,10 +401,20 @@ class FastAppendCurve(pg.GraphicsObject): self.fast_path.clear() if should_redraw and not should_ds: - log.info(f'DEDOWN -> {self._name}') + if self._in_ds: + log.info(f'DEDOWN -> {self._name}') + self._in_ds = False - # else: + elif should_ds: + x_out, y_out = self.downsample( + x_out, + y_out, + **should_ds, + ) + profiler(f'FULL PATH downsample redraw={should_ds}') + self._in_ds = True + self.path = pg.functions.arrayToQPath( x_out, y_out, @@ -382,6 +422,9 @@ class FastAppendCurve(pg.GraphicsObject): finiteCheck=False, path=self.path, ) + profiler('generated fresh path') + # profiler(f'DRAW PATH IN VIEW -> {self._name}') + # reserve mem allocs see: # - https://doc.qt.io/qt-5/qpainterpath.html#reserve # - https://doc.qt.io/qt-5/qpainterpath.html#capacity @@ -389,17 +432,12 @@ class FastAppendCurve(pg.GraphicsObject): # XXX: right now this is based on had hoc checks on a # hidpi 3840x2160 4k monitor but we should optimize for # the target display(s) on the sys. - if no_path_yet: - self.path.reserve(int(500e3)) - - profiler('generated fresh path') - - # if self._step_mode: - # self.path.closeSubpath() + # if no_path_yet: + # self.path.reserve(int(500e3)) # TODO: get this piecewise prepend working - right now it's # giving heck on vwap... - # if prepend_length: + # elif prepend_length: # breakpoint() # prepend_path = pg.functions.arrayToQPath( @@ -416,11 +454,15 @@ class FastAppendCurve(pg.GraphicsObject): elif ( append_length > 0 + and not view_range ): + new_x = x[-append_length - 2:-1] + new_y = y[-append_length - 2:-1] + if self._step_mode: new_x, new_y = step_path_arrays_from_1d( - x[-append_length - 2:-1], - y[-append_length - 2:-1], + new_x, + new_y, ) # [1:] since we don't need the vertical line normally at # the beginning of the step curve taking the first (x, @@ -429,12 +471,6 @@ class FastAppendCurve(pg.GraphicsObject): new_x = new_x[1:] new_y = new_y[1:] - else: - # print(f"append_length: {append_length}") - new_x = x[-append_length - 2:-1] - new_y = y[-append_length - 2:-1] - # print((new_x, new_y)) - profiler('diffed append arrays') if should_ds: @@ -490,21 +526,6 @@ class FastAppendCurve(pg.GraphicsObject): # self.disable_cache() # flip_cache = True - # XXX: do we need this any more? - # if ( - # self._step_mode - # ): - # self.disable_cache() - # flip_cache = True - - # XXX: lol brutal, the internals of `CurvePoint` (inherited by - # our `LineDot`) required ``.getData()`` to work.. - self.xData = x - self.yData = y - - x0, x_last = self._xrange = x[0], x[-1] - y_last = y[-1] - # draw the "current" step graphic segment so it lines up with # the "middle" of the current (OHLC) sample. if self._step_mode: @@ -522,7 +543,6 @@ class FastAppendCurve(pg.GraphicsObject): # f"last rect br: {self._last_step_rect}", # ) else: - # print((x[-1], y_last)) self._last_line = QLineF( x[-2], y[-2], x[-1], y_last @@ -536,11 +556,9 @@ class FastAppendCurve(pg.GraphicsObject): self.update() profiler('.update()') - if flip_cache: - # XXX: seems to be needed to avoid artifacts (see above). - self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - - self._x, self._y = x, y + # if flip_cache: + # # XXX: seems to be needed to avoid artifacts (see above). + # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. From ce85031ef22e6d91de8ad7143a97d534a73bf69c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 10:31:01 -0400 Subject: [PATCH 60/84] Given in-view rendering, make bars downsample on uppx >= 8 --- piker/ui/_ohlc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 482d6135..5b593dcb 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -364,7 +364,7 @@ class BarItems(pg.GraphicsObject): flip_cache = False - x_gt = 2 + x_gt = 8 if self._ds_line: uppx = self._ds_line.x_uppx() else: @@ -404,9 +404,9 @@ class BarItems(pg.GraphicsObject): y=y, x_iv=x_iv, y_iv=y_iv, - view_range=view_range, # hack + view_range=None, # hack ) - profiler('udated ds line') + profiler('updated ds line') if not self._in_ds: # hide bars and show line @@ -426,6 +426,7 @@ class BarItems(pg.GraphicsObject): # stop here since we don't need to update bars path any more # as we delegate to the downsample line with updates. + profiler.finish() return elif ( From 9bbfa4be02f270e6fbbccb6e7d86615b48baa87e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 10:43:44 -0400 Subject: [PATCH 61/84] Guard against zero px width --- piker/ui/_curve.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 59dbc0e5..26016342 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -341,7 +341,8 @@ class FastAppendCurve(pg.GraphicsObject): # check for downsampling conditions if ( # std m4 downsample conditions - uppx_diff >= 2 + px_width + and uppx_diff >= 2 or uppx_diff <= -2 or self._step_mode and abs(uppx_diff) >= 2 @@ -350,7 +351,7 @@ class FastAppendCurve(pg.GraphicsObject): f'{self._name} sampler change: {self._last_uppx} -> {uppx}' ) self._last_uppx = uppx - should_ds = {'px_width': px_width, 'uppx': uppx} + should_ds = True elif ( uppx <= 2 @@ -410,7 +411,8 @@ class FastAppendCurve(pg.GraphicsObject): x_out, y_out = self.downsample( x_out, y_out, - **should_ds, + px_width, + uppx, ) profiler(f'FULL PATH downsample redraw={should_ds}') self._in_ds = True From eeca9eb4c7898ccfa642ace7bc8737269237bf7f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 13:47:34 -0400 Subject: [PATCH 62/84] Add `.update_graphics_from_array()` flags for setting view-range use and graphics rendering --- piker/ui/_chart.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index d8e90ec3..c09ac89b 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -958,6 +958,7 @@ class ChartPlotWidget(pg.PlotWidget): def increment_view( self, steps: int = 1, + vb: Optional[ChartView] = None, ) -> None: """ @@ -966,7 +967,8 @@ class ChartPlotWidget(pg.PlotWidget): """ l, r = self.view_range() - self.view.setXRange( + view = vb or self.view + view.setXRange( min=l + steps, max=r + steps, @@ -1043,6 +1045,7 @@ class ChartPlotWidget(pg.PlotWidget): ) pi.hideButtons() + # cv.enable_auto_yrange(self.view) cv.enable_auto_yrange() # compose this new plot's graphics with the current chart's @@ -1187,6 +1190,9 @@ class ChartPlotWidget(pg.PlotWidget): array: Optional[np.ndarray] = None, array_key: Optional[str] = None, + use_vr: bool = True, + render: bool = True, + **kwargs, ) -> pg.GraphicsObject: @@ -1223,14 +1229,17 @@ class ChartPlotWidget(pg.PlotWidget): # to_draw = array[lbar - ifirst:(rbar - ifirst) + 1] in_view = array[lbar_i: rbar_i + 1] - if not in_view.size: + if ( + not in_view.size + or not render + ): return graphics if isinstance(graphics, BarItems): graphics.update_from_array( array, in_view, - view_range=(lbar_i, rbar_i), + view_range=(lbar_i, rbar_i) if use_vr else None, **kwargs, ) @@ -1242,7 +1251,7 @@ class ChartPlotWidget(pg.PlotWidget): x_iv=in_view['index'], y_iv=in_view[data_key], - view_range=(lbar_i, rbar_i), + view_range=(lbar_i, rbar_i) if use_vr else None, **kwargs ) From 27e3d0ef80815c0fae475c71fdeea4556db2baff Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 14:43:13 -0400 Subject: [PATCH 63/84] Ensure we update the volume array, not graphics Ugh, turns out the wacky `ChartView.maxmin` callback stuff we did (for determining y-range sizings) currently requires that the volume array has a "bars in view" result.. so let's make that keep working without rendering the graphics for the curve (since we're disabling them once $vlm comes up). --- piker/ui/_display.py | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 43597f27..67e07d0a 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -75,6 +75,7 @@ _tick_groups = { def chart_maxmin( chart: ChartPlotWidget, + ohlcv_shm: ShmArray, vlm_chart: Optional[ChartPlotWidget] = None, ) -> tuple[ @@ -91,7 +92,8 @@ def chart_maxmin( # https://arxiv.org/abs/cs/0610046 # https://github.com/lemire/pythonmaxmin - array = chart._arrays[chart.name] + # array = chart._arrays[chart.name] + array = ohlcv_shm.array ifirst = array[0]['index'] last_bars_range = chart.bars_range() @@ -183,7 +185,12 @@ async def graphics_update_loop( if vlm_chart: vlm_sticky = vlm_chart._ysticks['volume'] - maxmin = partial(chart_maxmin, chart, vlm_chart) + maxmin = partial( + chart_maxmin, + chart, + ohlcv, + vlm_chart, + ) last_bars_range: tuple[float, float] ( last_bars_range, @@ -359,6 +366,7 @@ def graphics_update_cycle( # compute the first available graphic's x-units-per-pixel xpx = vlm_chart.view.xs_in_px() + # print(f'vlm xpx {xpx}') in_view = chart.in_view(ohlcv.array) @@ -436,14 +444,30 @@ def graphics_update_cycle( ): # TODO: make it so this doesn't have to be called # once the $vlm is up? - vlm_chart.update_graphics_from_array('volume', array) + vlm_chart.update_graphics_from_array( + 'volume', + array, + + # UGGGh, see ``maxmin()`` impl in `._fsp` for + # the overlayed plotitems... we need a better + # bay to invoke a maxmin per overlay.. + render=False, + # XXX: ^^^^ THIS IS SUPER IMPORTANT! ^^^^ + # without this, since we disable the + # 'volume' (units) chart after the $vlm starts + # up we need to be sure to enable this + # auto-ranging otherwise there will be no handler + # connected to update accompanying overlay + # graphics.. + ) if ( mx_vlm_in_view != vars['last_mx_vlm'] ): # print(f'mx vlm: {last_mx_vlm} -> {mx_vlm_in_view}') + yrange = (0, mx_vlm_in_view * 1.375) vlm_chart.view._set_yrange( - yrange=(0, mx_vlm_in_view * 1.375) + yrange=yrange, ) vars['last_mx_vlm'] = mx_vlm_in_view @@ -455,7 +479,10 @@ def graphics_update_cycle( array_key=curve_name, ) # is this even doing anything? - flow.plot.vb._set_yrange( + # (pretty sure it's the real-time + # resizing from last quote?) + fvb = flow.plot.vb + fvb._set_yrange( autoscale_linked_plots=False, name=curve_name, ) @@ -601,6 +628,9 @@ def graphics_update_cycle( # run synchronous update on all derived fsp subplots for name, subchart in ds.linked.subplots.items(): + if name == 'volume': + continue + update_fsp_chart( subchart, subchart._shm, From 082b02776cdd283144669f68b749ee5174111c6d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 14:45:29 -0400 Subject: [PATCH 64/84] Drop the unit-volume chart once $vlm is fully drawn --- piker/ui/_fsp.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 5c656a2d..42017752 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -756,20 +756,14 @@ async def open_vlm_displays( 'dark_trade_rate', ] - # add custom auto range handler - dvlm_pi.vb._maxmin = partial( + group_mxmn = partial( maxmin, # keep both regular and dark vlm in view names=fields + dvlm_rate_fields, ) - # TODO: is there a way to "sync" the dual axes such that only - # one curve is needed? - # hide the original vlm curve since the $vlm one is now - # displayed and the curves are effectively the same minus - # liquidity events (well at least on low OHLC periods - 1s). - vlm_curve.hide() - chart.removeItem(vlm_curve) + # add custom auto range handler + dvlm_pi.vb._maxmin = group_mxmn # use slightly less light (then bracket) gray # for volume from "main exchange" and a more "bluey" @@ -837,6 +831,14 @@ async def open_vlm_displays( fr_shm, ) + # TODO: is there a way to "sync" the dual axes such that only + # one curve is needed? + # hide the original vlm curve since the $vlm one is now + # displayed and the curves are effectively the same minus + # liquidity events (well at least on low OHLC periods - 1s). + vlm_curve.hide() + chart.removeItem(vlm_curve) + # Trade rate overlay # XXX: requires an additional overlay for # a trades-per-period (time) y-range. From 2284e61eda7f3455e4f7495b9421a8af0b091362 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 15:58:06 -0400 Subject: [PATCH 65/84] Only pass vr for bars, allow source vb in autorange --- piker/ui/_interaction.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index adc551dc..93d4676d 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -37,7 +37,7 @@ from ..log import get_logger from ._style import _min_points_to_show from ._editors import SelectRect from . import _event -# from ._ohlc import BarItems +from ._ohlc import BarItems log = get_logger(__name__) @@ -809,7 +809,8 @@ class ChartView(ViewBox): self.setYRange(ylow, yhigh) def enable_auto_yrange( - vb: ChartView, + self, + src_vb: Optional[ChartView] = None, ) -> None: ''' @@ -817,7 +818,10 @@ class ChartView(ViewBox): based on data contents and ``ViewBox`` state. ''' - vb.sigXRangeChanged.connect(vb._set_yrange) + if src_vb is None: + src_vb = self + + src_vb.sigXRangeChanged.connect(self._set_yrange) # TODO: a smarter way to avoid calling this needlessly? # 2 things i can think of: @@ -825,13 +829,13 @@ class ChartView(ViewBox): # iterate those. # - only register this when certain downsampleable graphics are # "added to scene". - vb.sigXRangeChanged.connect(vb.maybe_downsample_graphics) + src_vb.sigXRangeChanged.connect(self.maybe_downsample_graphics) # mouse wheel doesn't emit XRangeChanged - vb.sigRangeChangedManually.connect(vb._set_yrange) + src_vb.sigRangeChangedManually.connect(self._set_yrange) # splitter(s) resizing - vb.sigResized.connect(vb._set_yrange) + src_vb.sigResized.connect(self._set_yrange) def disable_auto_yrange( self, @@ -863,9 +867,21 @@ class ChartView(ViewBox): # graphics = list(self._chart._graphics.values()) for name, graphics in chart._graphics.items(): + + # TODO: make it so we don't have to do this XD + # if name == 'volume': + # continue + + use_vr = False + if isinstance(graphics, BarItems): + use_vr = True + # pass in no array which will read and render from the last # passed array (normally provided by the display loop.) - chart.update_graphics_from_array(name) + chart.update_graphics_from_array( + name, + use_vr=use_vr, + ) # for graphic in graphics: # ds_meth = getattr(graphic, 'maybe_downsample', None) From 91de281b7e3885e3da9440978a3cf21a6939a7bf Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 17:28:29 -0400 Subject: [PATCH 66/84] Downsample curves even less frequently --- piker/ui/_curve.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 26016342..df4d64e5 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -342,9 +342,9 @@ class FastAppendCurve(pg.GraphicsObject): if ( # std m4 downsample conditions px_width - and uppx_diff >= 2 - or uppx_diff <= -2 - or self._step_mode and abs(uppx_diff) >= 2 + and uppx_diff >= 4 + or uppx_diff <= -3 + or self._step_mode and abs(uppx_diff) >= 4 ): log.info( From f95d22bfd302fb8b99698c6031cd08e1b4fcc2ee Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 17:28:56 -0400 Subject: [PATCH 67/84] Delegate `BarItems.x_uppx()` to internal ds curve --- piker/ui/_ohlc.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 5b593dcb..6507c35b 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -318,6 +318,13 @@ class BarItems(pg.GraphicsObject): return self.path + + def x_uppx(self) -> int: + if self._ds_line: + return self._ds_line.x_uppx() + else: + return 0 + def update_from_array( self, @@ -346,6 +353,7 @@ class BarItems(pg.GraphicsObject): profiler = pg.debug.Profiler( disabled=not pg_profile_enabled(), gt=ms_slower_then, + delayed=True, ) # index = self.start_index @@ -364,7 +372,7 @@ class BarItems(pg.GraphicsObject): flip_cache = False - x_gt = 8 + x_gt = 16 if self._ds_line: uppx = self._ds_line.x_uppx() else: @@ -383,6 +391,8 @@ class BarItems(pg.GraphicsObject): ): should_line = True + profiler('ds logic complete') + if ( should_line ): @@ -391,6 +401,7 @@ class BarItems(pg.GraphicsObject): x, y = self._ds_line_xy = ohlc_flatten(ohlc) x_iv, y_iv = self._ds_line_xy = ohlc_flatten(ohlc_iv) profiler('flattening bars to line') + print(f'rendering linesc') # TODO: we should be diffing the amount of new data which # needs to be downsampled. Ideally we actually are just @@ -427,6 +438,7 @@ class BarItems(pg.GraphicsObject): # stop here since we don't need to update bars path any more # as we delegate to the downsample line with updates. profiler.finish() + print('terminating early') return elif ( @@ -598,6 +610,8 @@ class BarItems(pg.GraphicsObject): if flip_cache: self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + profiler.finish() + def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect From b524929cb6781c8d1d416f22bc8721b43c943e52 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 4 Apr 2022 17:29:33 -0400 Subject: [PATCH 68/84] Only bail up pan updates if uppx > 16 --- piker/ui/_interaction.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 93d4676d..751240e1 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -34,6 +34,7 @@ import numpy as np import trio from ..log import get_logger +from .._profile import pg_profile_enabled, ms_slower_then from ._style import _min_points_to_show from ._editors import SelectRect from . import _event @@ -862,15 +863,30 @@ class ChartView(ViewBox): def maybe_downsample_graphics(self): + profiler = pg.debug.Profiler( + disabled=not pg_profile_enabled(), + gt=3, + delayed=True, + ) + # TODO: a faster single-loop-iterator way of doing this XD chart = self._chart - # graphics = list(self._chart._graphics.values()) for name, graphics in chart._graphics.items(): # TODO: make it so we don't have to do this XD # if name == 'volume': # continue + uppx = graphics.x_uppx() + if ( + uppx and uppx > 16 + and self._ic is not None + ): + # don't bother updating since we're zoomed out bigly and + # in a pan-interaction, in which case we shouldn't be + # doing view-range based rendering (at least not yet). + # print(f'{uppx} exiting early!') + return use_vr = False if isinstance(graphics, BarItems): @@ -882,7 +898,9 @@ class ChartView(ViewBox): name, use_vr=use_vr, ) + profiler(f'updated {name}') + profiler.finish() # for graphic in graphics: # ds_meth = getattr(graphic, 'maybe_downsample', None) # if ds_meth: From 4bc2bbda6981663c520a5f5227882e9e79358a05 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 5 Apr 2022 09:18:19 -0400 Subject: [PATCH 69/84] Allow passing "ms slower then" value on cli to `--profile` --- piker/_profile.py | 2 +- piker/ui/cli.py | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/piker/_profile.py b/piker/_profile.py index 2e726e80..697c3c3b 100644 --- a/piker/_profile.py +++ b/piker/_profile.py @@ -24,7 +24,7 @@ from functools import wraps # NOTE: you can pass a flag to enable this: # ``piker chart --profile``. _pg_profile: bool = False -ms_slower_then: float = 10 +ms_slower_then: float = 0 def pg_profile_enabled() -> bool: diff --git a/piker/ui/cli.py b/piker/ui/cli.py index a8bd8e9f..02515fc0 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -122,7 +122,8 @@ def optschain(config, symbol, date, rate, test): @cli.command() @click.option( '--profile', - is_flag=True, + '-p', + default=None, help='Enable pyqtgraph profiling' ) @click.option( @@ -133,9 +134,16 @@ def optschain(config, symbol, date, rate, test): @click.argument('symbol', required=True) @click.pass_obj def chart(config, symbol, profile, pdb): - """Start a real-time chartng UI - """ - from .. import _profile + ''' + Start a real-time chartng UI + + ''' + # eg. ``--profile 3`` reports profiling for anything slower then 3 ms. + if profile is not None: + from .. import _profile + _profile._pg_profile = True + _profile.ms_slower_then = float(profile) + from ._app import _main if '.' not in symbol: @@ -145,8 +153,6 @@ def chart(config, symbol, profile, pdb): )) return - # toggle to enable profiling - _profile._pg_profile = profile # global opts brokernames = config['brokers'] From b20e9e58eec84dad114dd28ffe483c5320c6acee Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 5 Apr 2022 11:04:45 -0400 Subject: [PATCH 70/84] Use HL tracer by default, seems to be faster? --- piker/ui/_compression.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index 1e178036..cdc8bb2e 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -155,7 +155,7 @@ def downsample( def ohlc_flatten( ohlc: np.ndarray, - use_mxmn: bool = False, + use_mxmn: bool = True, ) -> tuple[np.ndarray, np.ndarray]: ''' @@ -167,7 +167,11 @@ def ohlc_flatten( index = ohlc['index'] if use_mxmn: + # traces a line optimally over highs to lows + # using numba. NOTE: pretty sure this is faster + # and looks about the same as the below output. flat, x = hl2mxmn(ohlc) + else: flat = rfn.structured_to_unstructured( ohlc[['open', 'high', 'low', 'close']] From 1a95712680c3d5df65d291997c7378dba010c98e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 6 Apr 2022 09:09:44 -0400 Subject: [PATCH 71/84] Don't return early on ds line render to avoid breaking profiling The pg profiler seems to have trouble with early `return`s in function calls (likely muckery with the GC/`.__delete__()`) so let's just try to avoid it for now until we either fix it (probably by implementing as a ctx mngr) or use diff one. --- piker/ui/_curve.py | 3 +- piker/ui/_ohlc.py | 280 ++++++++++++++++++++++----------------------- 2 files changed, 140 insertions(+), 143 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index df4d64e5..0e6f8211 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -265,6 +265,7 @@ class FastAppendCurve(pg.GraphicsObject): y_iv: np.ndarray, view_range: Optional[tuple[int, int]] = None, + profiler: Optional[pg.debug.Profiler] = None, ) -> QtGui.QPainterPath: ''' @@ -274,7 +275,7 @@ class FastAppendCurve(pg.GraphicsObject): a length diff. ''' - profiler = pg.debug.Profiler( + profiler = profiler or pg.debug.Profiler( msg=f'FastAppendCurve.update_from_array(): `{self._name}`', disabled=not pg_profile_enabled(), gt=ms_slower_then, diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 6507c35b..44dbb0c2 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -318,7 +318,6 @@ class BarItems(pg.GraphicsObject): return self.path - def x_uppx(self) -> int: if self._ds_line: return self._ds_line.x_uppx() @@ -335,6 +334,7 @@ class BarItems(pg.GraphicsObject): ohlc_iv: np.ndarray, view_range: Optional[tuple[int, int]] = None, + profiler: Optional[pg.debug.Profiler] = None, ) -> None: ''' @@ -350,7 +350,7 @@ class BarItems(pg.GraphicsObject): This routine should be made (transitively) as fast as possible. ''' - profiler = pg.debug.Profiler( + profiler = profiler or pg.debug.Profiler( disabled=not pg_profile_enabled(), gt=ms_slower_then, delayed=True, @@ -393,15 +393,12 @@ class BarItems(pg.GraphicsObject): profiler('ds logic complete') - if ( - should_line - ): + if should_line: # update the line graphic # x, y = self._ds_line_xy = ohlc_flatten(ohlc_iv) x, y = self._ds_line_xy = ohlc_flatten(ohlc) x_iv, y_iv = self._ds_line_xy = ohlc_flatten(ohlc_iv) profiler('flattening bars to line') - print(f'rendering linesc') # TODO: we should be diffing the amount of new data which # needs to be downsampled. Ideally we actually are just @@ -416,6 +413,7 @@ class BarItems(pg.GraphicsObject): x_iv=x_iv, y_iv=y_iv, view_range=None, # hack + profiler=profiler, ) profiler('updated ds line') @@ -438,179 +436,177 @@ class BarItems(pg.GraphicsObject): # stop here since we don't need to update bars path any more # as we delegate to the downsample line with updates. profiler.finish() - print('terminating early') + # print('terminating early') return - elif ( - not should_line - and self._in_ds - ): - # flip back to bars graphics and hide the downsample line. - log.info(f'showing bars graphic {self._name}') + else: + # we should be in bars mode - curve = self._ds_line - curve.hide() - # self._pi.removeItem(curve) + if self._in_ds: + # flip back to bars graphics and hide the downsample line. + log.info(f'showing bars graphic {self._name}') - # XXX: is this actually any faster? - # self._pi.addItem(self) - self.show() - self._in_ds = False + curve = self._ds_line + curve.hide() + # self._pi.removeItem(curve) - # generate in_view path - self.path = gen_qpath( - ohlc_iv, - 0, - self.w, - # path=self.path, - ) + # XXX: is this actually any faster? + # self._pi.addItem(self) + self.show() + self._in_ds = False - # TODO: to make the downsampling faster - # - allow mapping only a range of lines thus only drawing as - # many bars as exactly specified. - # - move ohlc "flattening" to a shmarr - # - maybe move all this embedded logic to a higher - # level type? + # generate in_view path + self.path = gen_qpath( + ohlc_iv, + 0, + self.w, + # path=self.path, + ) - # ohlc = in_view + # TODO: to make the downsampling faster + # - allow mapping only a range of lines thus only drawing as + # many bars as exactly specified. + # - move ohlc "flattening" to a shmarr + # - maybe move all this embedded logic to a higher + # level type? - # if prepend_length: - # # new history was added and we need to render a new path - # prepend_bars = ohlc[:prepend_length] + # if prepend_length: + # # new history was added and we need to render a new path + # prepend_bars = ohlc[:prepend_length] - # if ds_prepend_length: - # ds_prepend_bars = ohlc[:ds_prepend_length] - # pre_x, pre_y = ohlc_flatten(ds_prepend_bars) - # fx = np.concatenate((pre_x, fx)) - # fy = np.concatenate((pre_y, fy)) - # profiler('ds line prepend diff complete') + # if ds_prepend_length: + # ds_prepend_bars = ohlc[:ds_prepend_length] + # pre_x, pre_y = ohlc_flatten(ds_prepend_bars) + # fx = np.concatenate((pre_x, fx)) + # fy = np.concatenate((pre_y, fy)) + # profiler('ds line prepend diff complete') - # if append_length: - # # generate new graphics to match provided array - # # path appending logic: - # # we need to get the previous "current bar(s)" for the time step - # # and convert it to a sub-path to append to the historical set - # # new_bars = ohlc[istop - 1:istop + append_length - 1] - # append_bars = ohlc[-append_length - 1:-1] - # # print(f'ohlc bars to append size: {append_bars.size}\n') + # if append_length: + # # generate new graphics to match provided array + # # path appending logic: + # # we need to get the previous "current bar(s)" for the time step + # # and convert it to a sub-path to append to the historical set + # # new_bars = ohlc[istop - 1:istop + append_length - 1] + # append_bars = ohlc[-append_length - 1:-1] + # # print(f'ohlc bars to append size: {append_bars.size}\n') - # if ds_append_length: - # ds_append_bars = ohlc[-ds_append_length - 1:-1] - # post_x, post_y = ohlc_flatten(ds_append_bars) - # print( - # f'ds curve to append sizes: {(post_x.size, post_y.size)}' - # ) - # fx = np.concatenate((fx, post_x)) - # fy = np.concatenate((fy, post_y)) + # if ds_append_length: + # ds_append_bars = ohlc[-ds_append_length - 1:-1] + # post_x, post_y = ohlc_flatten(ds_append_bars) + # print( + # f'ds curve to append sizes: {(post_x.size, post_y.size)}' + # ) + # fx = np.concatenate((fx, post_x)) + # fy = np.concatenate((fy, post_y)) - # profiler('ds line append diff complete') + # profiler('ds line append diff complete') - profiler('array diffs complete') + profiler('array diffs complete') - # does this work? - last = ohlc[-1] - # fy[-1] = last['close'] + # does this work? + last = ohlc[-1] + # fy[-1] = last['close'] - # # incremental update and cache line datums - # self._ds_line_xy = fx, fy + # # incremental update and cache line datums + # self._ds_line_xy = fx, fy - # maybe downsample to line - # ds = self.maybe_downsample() - # if ds: - # # if we downsample to a line don't bother with - # # any more path generation / updates - # self._ds_xrange = first_index, last_index - # profiler('downsampled to line') - # return + # maybe downsample to line + # ds = self.maybe_downsample() + # if ds: + # # if we downsample to a line don't bother with + # # any more path generation / updates + # self._ds_xrange = first_index, last_index + # profiler('downsampled to line') + # return - # print(in_view.size) + # print(in_view.size) - # if self.path: - # self.path = path - # self.path.reserve(path.capacity()) - # self.path.swap(path) + # if self.path: + # self.path = path + # self.path.reserve(path.capacity()) + # self.path.swap(path) - # path updates - # if prepend_length: - # # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path - # # y value not matching the first value from - # # ohlc[prepend_length + 1] ??? - # prepend_path = gen_qpath(prepend_bars, 0, self.w) - # old_path = self.path - # self.path = prepend_path - # self.path.addPath(old_path) - # profiler('path PREPEND') + # path updates + # if prepend_length: + # # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path + # # y value not matching the first value from + # # ohlc[prepend_length + 1] ??? + # prepend_path = gen_qpath(prepend_bars, 0, self.w) + # old_path = self.path + # self.path = prepend_path + # self.path.addPath(old_path) + # profiler('path PREPEND') - # if append_length: - # append_path = gen_qpath(append_bars, 0, self.w) + # if append_length: + # append_path = gen_qpath(append_bars, 0, self.w) - # self.path.moveTo( - # float(istop - self.w), - # float(append_bars[0]['open']) - # ) - # self.path.addPath(append_path) + # self.path.moveTo( + # float(istop - self.w), + # float(append_bars[0]['open']) + # ) + # self.path.addPath(append_path) - # profiler('path APPEND') - # fp = self.fast_path - # if fp is None: - # self.fast_path = append_path + # profiler('path APPEND') + # fp = self.fast_path + # if fp is None: + # self.fast_path = append_path - # else: - # fp.moveTo( - # float(istop - self.w), float(new_bars[0]['open']) - # ) - # fp.addPath(append_path) + # else: + # fp.moveTo( + # float(istop - self.w), float(new_bars[0]['open']) + # ) + # fp.addPath(append_path) - # self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # flip_cache = True + # self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) + # flip_cache = True - self._xrange = first_index, last_index + self._xrange = first_index, last_index - # trigger redraw despite caching - self.prepareGeometryChange() + # trigger redraw despite caching + self.prepareGeometryChange() - # generate new lines objects for updatable "current bar" - self._last_bar_lines = bar_from_ohlc_row(last, self.w) + # generate new lines objects for updatable "current bar" + self._last_bar_lines = bar_from_ohlc_row(last, self.w) - # last bar update - i, o, h, l, last, v = last[ - ['index', 'open', 'high', 'low', 'close', 'volume'] - ] - # assert i == self.start_index - 1 - # assert i == last_index - body, larm, rarm = self._last_bar_lines + # last bar update + i, o, h, l, last, v = last[ + ['index', 'open', 'high', 'low', 'close', 'volume'] + ] + # assert i == self.start_index - 1 + # assert i == last_index + body, larm, rarm = self._last_bar_lines - # XXX: is there a faster way to modify this? - rarm.setLine(rarm.x1(), last, rarm.x2(), last) + # XXX: is there a faster way to modify this? + rarm.setLine(rarm.x1(), last, rarm.x2(), last) - # writer is responsible for changing open on "first" volume of bar - larm.setLine(larm.x1(), o, larm.x2(), o) + # writer is responsible for changing open on "first" volume of bar + larm.setLine(larm.x1(), o, larm.x2(), o) - if l != h: # noqa + if l != h: # noqa - if body is None: - body = self._last_bar_lines[0] = QLineF(i, l, i, h) - else: - # update body - body.setLine(i, l, i, h) + if body is None: + body = self._last_bar_lines[0] = QLineF(i, l, i, h) + else: + # update body + body.setLine(i, l, i, h) - # XXX: pretty sure this is causing an issue where the bar has - # a large upward move right before the next sample and the body - # is getting set to None since the next bar is flat but the shm - # array index update wasn't read by the time this code runs. Iow - # we're doing this removal of the body for a bar index that is - # now out of date / from some previous sample. It's weird - # though because i've seen it do this to bars i - 3 back? + # XXX: pretty sure this is causing an issue where the bar has + # a large upward move right before the next sample and the body + # is getting set to None since the next bar is flat but the shm + # array index update wasn't read by the time this code runs. Iow + # we're doing this removal of the body for a bar index that is + # now out of date / from some previous sample. It's weird + # though because i've seen it do this to bars i - 3 back? - profiler('last bar set') + profiler('last bar set') - self.update() - profiler('.update()') + self.update() + profiler('.update()') - if flip_cache: - self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + if flip_cache: + self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - profiler.finish() + profiler.finish() def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect From b5f2558cecaa97bb54c2d408ac69174129d28954 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 6 Apr 2022 11:11:28 -0400 Subject: [PATCH 72/84] Only `.maybe_downsample_graphics()` on manual changes We don't need update graphics on every x-range change since that's what the display loop does. Instead, only on manual changes do we make manual calls into `.update_graphics_from_array()` and be sure to iterate all linked subplots and all their embedded graphics. --- piker/ui/_interaction.py | 96 +++++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 751240e1..069ab3e9 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -360,6 +360,7 @@ class ChartView(ViewBox): ): super().__init__( parent=parent, + name=name, # TODO: look into the default view padding # support that might replace somem of our # ``ChartPlotWidget._set_yrange()` @@ -742,6 +743,11 @@ class ChartView(ViewBox): data set. ''' + profiler = pg.debug.Profiler( + disabled=not pg_profile_enabled(), + gt=ms_slower_then, + delayed=True, + ) set_range = True chart = self._chart @@ -769,9 +775,11 @@ class ChartView(ViewBox): # Make sure min bars/datums on screen is adhered. else: br = bars_range or chart.bars_range() + profiler(f'got bars range: {br}') # TODO: maybe should be a method on the # chart widget/item? + # if False: if autoscale_linked_plots: # avoid recursion by sibling plots linked = self.linkedsplits @@ -786,6 +794,7 @@ class ChartView(ViewBox): bars_range=br, autoscale_linked_plots=False, ) + profiler('autoscaled linked plots') if set_range: @@ -795,6 +804,8 @@ class ChartView(ViewBox): ylow, yhigh = yrange + profiler(f'maxmin(): {yrange}') + # view margins: stay within a % of the "true range" diff = yhigh - ylow ylow = ylow - (diff * range_margin) @@ -808,6 +819,7 @@ class ChartView(ViewBox): yMax=yhigh, ) self.setYRange(ylow, yhigh) + profiler(f'set limits: {(ylow, yhigh)}') def enable_auto_yrange( self, @@ -830,7 +842,9 @@ class ChartView(ViewBox): # iterate those. # - only register this when certain downsampleable graphics are # "added to scene". - src_vb.sigXRangeChanged.connect(self.maybe_downsample_graphics) + src_vb.sigRangeChangedManually.connect( + self.maybe_downsample_graphics + ) # mouse wheel doesn't emit XRangeChanged src_vb.sigRangeChangedManually.connect(self._set_yrange) @@ -844,25 +858,39 @@ class ChartView(ViewBox): self._chart._static_yrange = 'axis' - def xs_in_px(self) -> float: + def x_uppx(self) -> float: ''' Return the "number of x units" within a single pixel currently being displayed for relevant graphics items which are our children. ''' - for graphic in self._chart._graphics.values(): - xvec = graphic.pixelVectors()[0] - if xvec: - xpx = xvec.x() - if xpx: - return xpx - else: - continue - return 1.0 + graphics = list(self._chart._graphics.values()) + if not graphics: + return 0 + + graphic = graphics[0] + xvec = graphic.pixelVectors()[0] + if xvec: + return xvec.x() + else: + return 0 def maybe_downsample_graphics(self): + uppx = self.x_uppx() + if ( + # we probably want to drop this once we are "drawing in + # view" for downsampled flows.. + uppx and uppx > 16 + and self._ic is not None + ): + # don't bother updating since we're zoomed out bigly and + # in a pan-interaction, in which case we shouldn't be + # doing view-range based rendering (at least not yet). + # print(f'{uppx} exiting early!') + return + profiler = pg.debug.Profiler( disabled=not pg_profile_enabled(), gt=3, @@ -871,37 +899,23 @@ class ChartView(ViewBox): # TODO: a faster single-loop-iterator way of doing this XD chart = self._chart + linked = self.linkedsplits + plots = linked.subplots | {chart.name: chart} + for chart_name, chart in plots.items(): + for name, graphics in chart._graphics.items(): + # print(f'maybe ds chart:{name} graphic:{name}') - for name, graphics in chart._graphics.items(): + use_vr = False + if isinstance(graphics, BarItems): + use_vr = True - # TODO: make it so we don't have to do this XD - # if name == 'volume': - # continue - uppx = graphics.x_uppx() - if ( - uppx and uppx > 16 - and self._ic is not None - ): - # don't bother updating since we're zoomed out bigly and - # in a pan-interaction, in which case we shouldn't be - # doing view-range based rendering (at least not yet). - # print(f'{uppx} exiting early!') - return - - use_vr = False - if isinstance(graphics, BarItems): - use_vr = True - - # pass in no array which will read and render from the last - # passed array (normally provided by the display loop.) - chart.update_graphics_from_array( - name, - use_vr=use_vr, - ) - profiler(f'updated {name}') + # pass in no array which will read and render from the last + # passed array (normally provided by the display loop.) + chart.update_graphics_from_array( + name, + use_vr=use_vr, + profiler=profiler, + ) + profiler(f'range change updated {chart_name}:{name}') profiler.finish() - # for graphic in graphics: - # ds_meth = getattr(graphic, 'maybe_downsample', None) - # if ds_meth: - # ds_meth() From a1de097a4342076dec8ad37797b0875f17033a32 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 6 Apr 2022 12:13:05 -0400 Subject: [PATCH 73/84] Loop for first graphic with xvec --- piker/ui/_interaction.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 069ab3e9..6a833b1f 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -869,10 +869,10 @@ class ChartView(ViewBox): if not graphics: return 0 - graphic = graphics[0] - xvec = graphic.pixelVectors()[0] - if xvec: - return xvec.x() + for graphic in graphics: + xvec = graphic.pixelVectors()[0] + if xvec: + return xvec.x() else: return 0 From eec329a221bbe60fdb00173f7233ca103bdcf598 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 6 Apr 2022 17:05:57 -0400 Subject: [PATCH 74/84] Add `Flow` type with a real chitty mxmn cacheing method This new type wraps a shm data flow and will eventually include things like incremental path-graphics updates and serialization + bg downsampling techniques. The main immediate motivation was to get a cached y-range max/min calc going since profiling revealed the `numpy` equivalents were actually quite slow as the data set grows large. Likely we can use all this to drive a streaming mx/mn routine that's always launched as part of each on-host flow. This is our official foray into use of `msgspec.Struct` B) and I have to say, pretty impressed; we'll likely completely ditch `pydantic` from here on out. --- piker/ui/_chart.py | 160 ++++++++++++++++++++++++++++++++------------- 1 file changed, 116 insertions(+), 44 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index c09ac89b..80b88b0c 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -34,10 +34,11 @@ from PyQt5.QtWidgets import ( QVBoxLayout, QSplitter, ) +import msgspec import numpy as np +# from pydantic import BaseModel import pyqtgraph as pg import trio -from pydantic import BaseModel from ._axes import ( DynamicDateAxis, @@ -59,10 +60,14 @@ from ._style import ( ) from ..data.feed import Feed from ..data._source import Symbol -from ..data._sharedmem import ShmArray +from ..data._sharedmem import ( + ShmArray, + # _Token, +) from ..log import get_logger from ._interaction import ChartView from ._forms import FieldsForm +from .._profile import pg_profile_enabled, ms_slower_then from ._overlay import PlotItemOverlay if TYPE_CHECKING: @@ -663,19 +668,90 @@ class LinkedSplits(QWidget): # flows: dict[str, np.ndarray] = {} -class Flow(BaseModel): +class Flow(msgspec.Struct): # , frozen=True): ''' (FinancialSignal-)Flow compound type which wraps a real-time graphics (curve) and its backing data stream together for high level access and control. - ''' - class Config: - arbitrary_types_allowed = True + The intention is for this type to eventually be capable of shm-passing + of incrementally updated graphics stream data between actors. + ''' name: str plot: pg.PlotItem - shm: Optional[ShmArray] = None # may be filled in "later" + is_ohlc: bool = False + + # TODO: hackery to be able to set a shm later + # but whilst also allowing this type to hashable, + # likely will require serializable token that is used to attach + # to the underlying shm ref after startup? + _shm: Optional[ShmArray] = None # currently, may be filled in "later" + + # cache of y-range values per x-range input. + _mxmns: dict[tuple[int, int], tuple[float, float]] = {} + + @property + def shm(self) -> ShmArray: + return self._shm + + @shm.setter + def shm(self, shm: ShmArray) -> ShmArray: + self._shm = shm + + def maxmin( + self, + lbar, + rbar, + + ) -> tuple[float, float]: + ''' + Compute the cached max and min y-range values for a given + x-range determined by ``lbar`` and ``rbar``. + + ''' + rkey = (lbar, rbar) + result = self._mxmns.get(rkey) + if result: + return result + + shm = self.shm + if shm is None: + # print(f'no shm {self.name}?') + return 0, 0 + + arr = shm.array + + # build relative indexes into shm array + # TODO: should we just add/use a method + # on the shm to do this? + ifirst = arr[0]['index'] + slice_view = arr[ + lbar - ifirst:(rbar - ifirst) + 1] + + if not slice_view.size: + # print(f'no data in view {self.name}?') + return 0, 0 + + if self.is_ohlc: + ylow = np.min(slice_view['low']) + yhigh = np.max(slice_view['high']) + + else: + view = slice_view[self.name] + ylow = np.min(view) + yhigh = np.max(view) + # else: + # ylow, yhigh = 0, 0 + + result = ylow, yhigh + + if result != (0, 0): + self._mxmns[rkey] = result + + if self.name == 'drk_vlm': + print(f'{self.name} mxmn @ {rkey} -> {result}') + return result class ChartPlotWidget(pg.PlotWidget): @@ -1005,6 +1081,13 @@ class ChartPlotWidget(pg.PlotWidget): data_key = array_key or name self._graphics[data_key] = graphics + + self._flows[data_key] = Flow( + name=name, + plot=self.plotItem, + is_ohlc=True, + ) + self._add_sticky(name, bg_color='davies') return graphics, data_key @@ -1122,6 +1205,12 @@ class ChartPlotWidget(pg.PlotWidget): pi = self.plotItem + self._flows[data_key] = Flow( + name=name, + plot=pi, + is_ohlc=False, + ) + # TODO: this probably needs its own method? if overlay: if isinstance(overlay, pg.PlotItem): @@ -1130,10 +1219,6 @@ class ChartPlotWidget(pg.PlotWidget): f'{overlay} must be from `.plotitem_overlay()`' ) pi = overlay - - # anchor_at = ('bottom', 'left') - self._flows[name] = Flow(name=name, plot=pi) - else: # anchor_at = ('top', 'left') @@ -1342,46 +1427,33 @@ class ChartPlotWidget(pg.PlotWidget): If ``bars_range`` is provided use that range. ''' - l, lbar, rbar, r = bars_range or self.bars_range() - # TODO: logic to check if end of bars in view - # extra = view_len - _min_points_to_show - # begin = self._arrays['ohlc'][0]['index'] - extra - # # end = len(self._arrays['ohlc']) - 1 + extra - # end = self._arrays['ohlc'][-1]['index'] - 1 + extra + profiler = pg.debug.Profiler( + disabled=not pg_profile_enabled(), + gt=ms_slower_then, + delayed=True, + ) - # bars_len = rbar - lbar - # log.debug( - # f"\nl: {l}, lbar: {lbar}, rbar: {rbar}, r: {r}\n" - # f"view_len: {view_len}, bars_len: {bars_len}\n" - # f"begin: {begin}, end: {end}, extra: {extra}" - # ) + l, lbar, rbar, r = bars_range or self.bars_range() + profiler(f'{self.name} got bars range') # TODO: here we should instead look up the ``Flow.shm.array`` # and read directly from shm to avoid copying to memory first # and then reading it again here. - a = self._arrays.get(name or self.name) - if a is None: - return None - - ifirst = a[0]['index'] - bars = a[lbar - ifirst:(rbar - ifirst) + 1] - - if not len(bars): - # likely no data loaded yet or extreme scrolling? - log.error(f"WTF bars_range = {lbar}:{rbar}") - return - + flow_key = name or self.name + flow = self._flows.get(flow_key) if ( - self.data_key == self.linked.symbol.key + flow is None ): - # ohlc sampled bars hi/lo lookup - ylow = np.nanmin(bars['low']) - yhigh = np.nanmax(bars['high']) + print(f"flow {flow_key} doesn't exist in chart {self.name}") + return 0, 0 else: - view = bars[name or self.data_key] - ylow = np.nanmin(view) - yhigh = np.nanmax(view) + key = round(lbar), round(rbar) + res = flow.maxmin(*key) + profiler(f'{key} max-min {res}') + if res == (0, 0): + log.error( + f"{flow_key} -> (0, 0) for bars_range = {key}" + ) - # print(f'{(ylow, yhigh)}') - return ylow, yhigh + return res From d4eddbdb250099e42c34b05df42c6d7d837388a6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 6 Apr 2022 17:10:19 -0400 Subject: [PATCH 75/84] Guard against zero px width --- piker/ui/_curve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 0e6f8211..00a4ca7a 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -408,7 +408,7 @@ class FastAppendCurve(pg.GraphicsObject): self._in_ds = False - elif should_ds: + elif should_ds and px_width: x_out, y_out = self.downsample( x_out, y_out, From ca283660dea898a2b35adcaebc21bfd62c85417d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 6 Apr 2022 17:11:15 -0400 Subject: [PATCH 76/84] Fix bug where if `yrange` was passed the mxmin callback was still used.. --- piker/ui/_interaction.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 6a833b1f..f76bac00 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -798,9 +798,14 @@ class ChartView(ViewBox): if set_range: - yrange = self._maxmin() - if yrange is None: - return + if not yrange: + # XXX: only compute the mxmn range + # if none is provided as input! + yrange = self._maxmin() + + if yrange is None: + log.warning(f'No yrange provided for {self.name}!?') + return ylow, yhigh = yrange From 454cd7920df49d866b0581b4986ef2a6adc18fd1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 7 Apr 2022 10:58:09 -0400 Subject: [PATCH 77/84] Disconnect signals in `ChartView.disable_auto_yrange()` Allows for removing resize callbacks for a flow/overlay that you wish to remove from view (eg. unit volume after dollar volume is up) and thus less general interaction callback overhead for any plot you don't wish to show or resize. Further, - drop the `autoscale_linked_plots` block for now since with multi-view-box overlays each register their own vb resize slots - pull the graphics object from the chart's `Flow` map inside `.maybe_downsample_graphics()` --- piker/ui/_interaction.py | 62 +++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index f76bac00..c6411caf 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -780,21 +780,21 @@ class ChartView(ViewBox): # TODO: maybe should be a method on the # chart widget/item? # if False: - if autoscale_linked_plots: - # avoid recursion by sibling plots - linked = self.linkedsplits - plots = list(linked.subplots.copy().values()) - main = linked.chart - if main: - plots.append(main) + # if autoscale_linked_plots: + # # avoid recursion by sibling plots + # linked = self.linkedsplits + # plots = list(linked.subplots.copy().values()) + # main = linked.chart + # if main: + # plots.append(main) - for chart in plots: - if chart and not chart._static_yrange: - chart.cv._set_yrange( - bars_range=br, - autoscale_linked_plots=False, - ) - profiler('autoscaled linked plots') + # for chart in plots: + # if chart and not chart._static_yrange: + # chart.cv._set_yrange( + # bars_range=br, + # autoscale_linked_plots=False, + # ) + # profiler('autoscaled linked plots') if set_range: @@ -839,8 +839,17 @@ class ChartView(ViewBox): if src_vb is None: src_vb = self + # such that when a linked chart changes its range + # this local view is also automatically changed and + # resized to data. src_vb.sigXRangeChanged.connect(self._set_yrange) + # splitter(s) resizing + src_vb.sigResized.connect(self._set_yrange) + + # mouse wheel doesn't emit XRangeChanged + src_vb.sigRangeChangedManually.connect(self._set_yrange) + # TODO: a smarter way to avoid calling this needlessly? # 2 things i can think of: # - register downsample-able graphics specially and only @@ -851,17 +860,24 @@ class ChartView(ViewBox): self.maybe_downsample_graphics ) - # mouse wheel doesn't emit XRangeChanged - src_vb.sigRangeChangedManually.connect(self._set_yrange) - - # splitter(s) resizing - src_vb.sigResized.connect(self._set_yrange) - def disable_auto_yrange( self, ) -> None: - self._chart._static_yrange = 'axis' + # self._chart._static_yrange = 'axis' + + self.sigXRangeChanged.disconnect( + self._set_yrange, + ) + self.sigResized.disconnect( + self._set_yrange, + ) + self.sigRangeChangedManually.disconnect( + self.maybe_downsample_graphics + ) + self.sigRangeChangedManually.disconnect( + self._set_yrange, + ) def x_uppx(self) -> float: ''' @@ -907,8 +923,8 @@ class ChartView(ViewBox): linked = self.linkedsplits plots = linked.subplots | {chart.name: chart} for chart_name, chart in plots.items(): - for name, graphics in chart._graphics.items(): - # print(f'maybe ds chart:{name} graphic:{name}') + for name, flow in chart._flows.items(): + graphics = flow.graphics use_vr = False if isinstance(graphics, BarItems): From b8374dbe9ad94204040277b0d1be41e338199239 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 7 Apr 2022 11:04:47 -0400 Subject: [PATCH 78/84] Fsp UI initialization updates - set shm refs on `Flow` entries. - don't run a graphics cycle on 'update' msgs from the engine if the containing chart is hidden. - drop `volume` from flows map and disable auto-yranging once $vlm comes up. --- piker/ui/_fsp.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 42017752..52763375 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -72,12 +72,16 @@ def has_vlm(ohlcv: ShmArray) -> bool: def update_fsp_chart( chart: ChartPlotWidget, - shm: ShmArray, + flow, graphics_name: str, array_key: Optional[str], ) -> None: + shm = flow.shm + if not shm: + return + array = shm.array last_row = try_read(array) @@ -271,6 +275,7 @@ async def run_fsp_ui( # data looked up from the chart's internal array set. # TODO: we must get a data view api going STAT!! chart._shm = shm + chart._flows[chart.data_key].shm = shm # should **not** be the same sub-chart widget assert chart.name != linkedsplits.chart.name @@ -282,7 +287,7 @@ async def run_fsp_ui( # first UI update, usually from shm pushed history update_fsp_chart( chart, - shm, + chart._flows[array_key], name, array_key=array_key, ) @@ -634,6 +639,7 @@ async def open_vlm_displays( # the curve item internals are pretty convoluted. style='step', ) + chart._flows['volume'].shm = ohlcv # force 0 to always be in view def maxmin( @@ -797,13 +803,16 @@ async def open_vlm_displays( color=color, step_mode=step_mode, style=style, + pi=pi, ) # TODO: we need a better API to do this.. # specially store ref to shm for lookup in display loop # since only a placeholder of `None` is entered in # ``.draw_curve()``. - chart._flows[name].shm = shm + flow = chart._flows[name] + assert flow.plot is pi + flow.shm = shm chart_curves( fields, @@ -838,6 +847,9 @@ async def open_vlm_displays( # liquidity events (well at least on low OHLC periods - 1s). vlm_curve.hide() chart.removeItem(vlm_curve) + chart._flows.pop('volume') + # avoid range sorting on volume once disabled + chart.view.disable_auto_yrange() # Trade rate overlay # XXX: requires an additional overlay for @@ -878,7 +890,10 @@ async def open_vlm_displays( style='dash', ) - for pi in (dvlm_pi, tr_pi): + for pi in ( + dvlm_pi, + tr_pi, + ): for name, axis_info in pi.axes.items(): # lol this sux XD axis = axis_info['item'] From 7c615a403b886593a16b25d634cd09b82cd73dec Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 7 Apr 2022 11:13:02 -0400 Subject: [PATCH 79/84] Allow passing a `plotItem` to `.draw_curve()` If manually managing an overlay you'll likely call `.overlay_plotitem()` and then a plotting method so we need to accept a plot item input so that the chart's pi doesn't get assigned incorrectly in the `Flow` entry (though it is by default if no input is provided). More, - add a `Flow.graphics` field and set it to the `pg.GraphicsObject`. - make `Flow.maxmin()` return `None` in the "can't calculate" cases. --- piker/ui/_chart.py | 209 +++++++++++++++++++++++---------------------- 1 file changed, 107 insertions(+), 102 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 80b88b0c..a3a97164 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -658,102 +658,6 @@ class LinkedSplits(QWidget): cpw.sidepane.setMaximumWidth(sp_w) -# class FlowsTable(pydantic.BaseModel): -# ''' -# Data-AGGRegate: high level API onto multiple (categorized) -# ``Flow``s with high level processing routines for -# multi-graphics computations and display. - -# ''' -# flows: dict[str, np.ndarray] = {} - - -class Flow(msgspec.Struct): # , frozen=True): - ''' - (FinancialSignal-)Flow compound type which wraps a real-time - graphics (curve) and its backing data stream together for high level - access and control. - - The intention is for this type to eventually be capable of shm-passing - of incrementally updated graphics stream data between actors. - - ''' - name: str - plot: pg.PlotItem - is_ohlc: bool = False - - # TODO: hackery to be able to set a shm later - # but whilst also allowing this type to hashable, - # likely will require serializable token that is used to attach - # to the underlying shm ref after startup? - _shm: Optional[ShmArray] = None # currently, may be filled in "later" - - # cache of y-range values per x-range input. - _mxmns: dict[tuple[int, int], tuple[float, float]] = {} - - @property - def shm(self) -> ShmArray: - return self._shm - - @shm.setter - def shm(self, shm: ShmArray) -> ShmArray: - self._shm = shm - - def maxmin( - self, - lbar, - rbar, - - ) -> tuple[float, float]: - ''' - Compute the cached max and min y-range values for a given - x-range determined by ``lbar`` and ``rbar``. - - ''' - rkey = (lbar, rbar) - result = self._mxmns.get(rkey) - if result: - return result - - shm = self.shm - if shm is None: - # print(f'no shm {self.name}?') - return 0, 0 - - arr = shm.array - - # build relative indexes into shm array - # TODO: should we just add/use a method - # on the shm to do this? - ifirst = arr[0]['index'] - slice_view = arr[ - lbar - ifirst:(rbar - ifirst) + 1] - - if not slice_view.size: - # print(f'no data in view {self.name}?') - return 0, 0 - - if self.is_ohlc: - ylow = np.min(slice_view['low']) - yhigh = np.max(slice_view['high']) - - else: - view = slice_view[self.name] - ylow = np.min(view) - yhigh = np.max(view) - # else: - # ylow, yhigh = 0, 0 - - result = ylow, yhigh - - if result != (0, 0): - self._mxmns[rkey] = result - - if self.name == 'drk_vlm': - print(f'{self.name} mxmn @ {rkey} -> {result}') - return result - - class ChartPlotWidget(pg.PlotWidget): ''' ``GraphicsView`` subtype containing a single ``PlotItem``. @@ -1086,6 +990,7 @@ class ChartPlotWidget(pg.PlotWidget): name=name, plot=self.plotItem, is_ohlc=True, + graphics=graphics, ) self._add_sticky(name, bg_color='davies') @@ -1159,6 +1064,7 @@ class ChartPlotWidget(pg.PlotWidget): overlay: bool = False, color: Optional[str] = None, add_label: bool = True, + pi: Optional[pg.PlotItem] = None, **pdi_kwargs, @@ -1203,12 +1109,13 @@ class ChartPlotWidget(pg.PlotWidget): self._graphics[name] = curve self._arrays[data_key] = data - pi = self.plotItem + pi = pi or self.plotItem self._flows[data_key] = Flow( name=name, plot=pi, is_ohlc=False, + graphics=curve, ) # TODO: this probably needs its own method? @@ -1428,6 +1335,7 @@ class ChartPlotWidget(pg.PlotWidget): ''' profiler = pg.debug.Profiler( + msg=f'`{str(self)}.maxmin()` loop cycle for: `{self.name}`', disabled=not pg_profile_enabled(), gt=ms_slower_then, delayed=True, @@ -1444,16 +1352,113 @@ class ChartPlotWidget(pg.PlotWidget): if ( flow is None ): - print(f"flow {flow_key} doesn't exist in chart {self.name}") - return 0, 0 + log.error(f"flow {flow_key} doesn't exist in chart {self.name} !?") + res = 0, 0 else: key = round(lbar), round(rbar) res = flow.maxmin(*key) - profiler(f'{key} max-min {res}') - if res == (0, 0): + profiler(f'yrange mxmn: {key} -> {res}') + if res == (None, None): log.error( - f"{flow_key} -> (0, 0) for bars_range = {key}" + f"{flow_key} no mxmn for bars_range => {key} !?" ) + res = 0, 0 return res + + +# class FlowsTable(pydantic.BaseModel): +# ''' +# Data-AGGRegate: high level API onto multiple (categorized) +# ``Flow``s with high level processing routines for +# multi-graphics computations and display. + +# ''' +# flows: dict[str, np.ndarray] = {} + + +class Flow(msgspec.Struct): # , frozen=True): + ''' + (FinancialSignal-)Flow compound type which wraps a real-time + graphics (curve) and its backing data stream together for high level + access and control. + + The intention is for this type to eventually be capable of shm-passing + of incrementally updated graphics stream data between actors. + + ''' + name: str + plot: pg.PlotItem + is_ohlc: bool = False + graphics: pg.GraphicsObject + + # TODO: hackery to be able to set a shm later + # but whilst also allowing this type to hashable, + # likely will require serializable token that is used to attach + # to the underlying shm ref after startup? + _shm: Optional[ShmArray] = None # currently, may be filled in "later" + + # cache of y-range values per x-range input. + _mxmns: dict[tuple[int, int], tuple[float, float]] = {} + + @property + def shm(self) -> ShmArray: + return self._shm + + @shm.setter + def shm(self, shm: ShmArray) -> ShmArray: + self._shm = shm + + def maxmin( + self, + lbar, + rbar, + + ) -> tuple[float, float]: + ''' + Compute the cached max and min y-range values for a given + x-range determined by ``lbar`` and ``rbar``. + + ''' + rkey = (lbar, rbar) + cached_result = self._mxmns.get(rkey) + if cached_result: + return cached_result + + shm = self.shm + if shm is None: + mxmn = None + + else: # new block for profiling?.. + arr = shm.array + + # build relative indexes into shm array + # TODO: should we just add/use a method + # on the shm to do this? + ifirst = arr[0]['index'] + slice_view = arr[ + lbar - ifirst: + (rbar - ifirst) + 1 + ] + + if not slice_view.size: + mxmn = None + + else: + if self.is_ohlc: + ylow = np.min(slice_view['low']) + yhigh = np.max(slice_view['high']) + + else: + view = slice_view[self.name] + ylow = np.min(view) + yhigh = np.max(view) + + mxmn = ylow, yhigh + + if mxmn is not None: + # cache new mxmn result + self._mxmns[rkey] = mxmn + + return mxmn From ee831baeb3e5d21fdaf0e4de94f193263bec6822 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 7 Apr 2022 14:11:01 -0400 Subject: [PATCH 80/84] Display loop mega-cleanup The most important changes include: - iterating the new `Flow` type and updating graphics - adding detailed profiling - increasing the min uppx before graphics updates are throttled - including the L1 spread in y-range calcs so that you never have the bid/ask go "out of view".. - pass around `Flow`s instead of shms - drop all the old prototyped downsampling code --- piker/ui/_display.py | 167 +++++++++++++++---------------------------- 1 file changed, 59 insertions(+), 108 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 67e07d0a..27fcc02e 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -30,7 +30,6 @@ import numpy as np import tractor import trio import pyqtgraph as pg -from PyQt5.QtCore import QLineF from .. import brokers from ..data.feed import open_feed @@ -73,13 +72,20 @@ _tick_groups = { } +# TODO: delegate this to each `Flow.maxmin()` which includes +# caching and further we should implement the following stream based +# approach, likely with ``numba``: +# https://arxiv.org/abs/cs/0610046 +# https://github.com/lemire/pythonmaxmin def chart_maxmin( chart: ChartPlotWidget, ohlcv_shm: ShmArray, vlm_chart: Optional[ChartPlotWidget] = None, ) -> tuple[ + tuple[int, int, int, int], + float, float, float, @@ -88,11 +94,6 @@ def chart_maxmin( Compute max and min datums "in view" for range limits. ''' - # TODO: implement this - # https://arxiv.org/abs/cs/0610046 - # https://github.com/lemire/pythonmaxmin - - # array = chart._arrays[chart.name] array = ohlcv_shm.array ifirst = array[0]['index'] @@ -105,18 +106,23 @@ def chart_maxmin( chart.default_view() return (last_bars_range, 0, 0, 0) - mx, mn = np.nanmax(in_view['high']), np.nanmin(in_view['low']) - - # TODO: when we start using line charts, probably want to make - # this an overloaded call on our `DataView - # sym = chart.name - # mx, mn = np.nanmax(in_view[sym]), np.nanmin(in_view[sym]) + mx, mn = ( + np.nanmax(in_view['high']), + np.nanmin(in_view['low'],) + ) mx_vlm_in_view = 0 if vlm_chart: - mx_vlm_in_view = np.max(in_view['volume']) + mx_vlm_in_view = np.max( + in_view['volume'] + ) - return last_bars_range, mx, max(mn, 0), mx_vlm_in_view + return ( + last_bars_range, + mx, + max(mn, 0), # presuming price can't be negative? + mx_vlm_in_view, + ) @dataclass @@ -272,8 +278,9 @@ async def graphics_update_loop( chart.default_view() - # main loop + # main real-time quotes update loop async for quotes in stream: + ds.quotes = quotes quote_period = time.time() - last_quote quote_rate = round( @@ -311,36 +318,40 @@ def graphics_update_cycle( trigger_all: bool = False, # flag used by prepend history updates ) -> None: - # TODO: eventually optimize this whole graphics stack with ``numba`` # hopefully XD + chart = ds.chart + profiler = pg.debug.Profiler( + msg=f'Graphics loop cycle for: `{chart.name}`', disabled=True, # not pg_profile_enabled(), gt=1/12 * 1e3, + # gt=ms_slower_then, ) # unpack multi-referenced components - chart = ds.chart vlm_chart = ds.vlm_chart l1 = ds.l1 - ohlcv = ds.ohlcv array = ohlcv.array vars = ds.vars tick_margin = vars['tick_margin'] - update_uppx = 5 + update_uppx = 6 for sym, quote in ds.quotes.items(): + # compute the first available graphic's x-units-per-pixel + xpx = vlm_chart.view.x_uppx() + # NOTE: vlm may be written by the ``brokerd`` backend # event though a tick sample is not emitted. # TODO: show dark trades differently # https://github.com/pikers/piker/issues/116 # NOTE: this used to be implemented in a dedicated - # "increment tas": ``check_for_new_bars()`` but it doesn't + # "increment task": ``check_for_new_bars()`` but it doesn't # make sense to do a whole task switch when we can just do # this simple index-diff and all the fsp sub-curve graphics # are diffed on each draw cycle anyway; so updates to the @@ -364,60 +375,6 @@ def graphics_update_cycle( profiler('maxmin call') liv = r > i_step # the last datum is in view - # compute the first available graphic's x-units-per-pixel - xpx = vlm_chart.view.xs_in_px() - # print(f'vlm xpx {xpx}') - - in_view = chart.in_view(ohlcv.array) - - if lbar != rbar: - # view box width in pxs - w = chart.view.boundingRect().width() - - # TODO: a better way to get this? - # i would guess the esiest way is to just - # get the ``.boundingRect()`` of the curve - # in view but maybe there's something smarter? - # Currently we're just mapping the rbar, lbar to - # pixels via: - cw = chart.view.mapViewToDevice(QLineF(lbar, 0, rbar, 0)).length() - # is this faster? - # cw = chart.mapFromView(QLineF(lbar, 0 , rbar, 0)).length() - - profiler( - f'view width pxs: {w}\n' - f'curve width pxs: {cw}\n' - f'sliced in view: {in_view.size}' - ) - - # compress bars to m4 line(s) if uppx is high enough - # if in_view.size > cw: - # from ._compression import ds_m4, hl2mxmn - - # mxmn, x = hl2mxmn(in_view) - # profiler('hl tracer') - - # nb, x, y = ds_m4( - # x=x, - # y=mxmn, - # # TODO: this needs to actually be the width - # # in pixels of the visible curve since we don't - # # want to downsample any 'zeros' around the curve, - # # just the values that make up the curve graphic, - # # i think? - # px_width=cw, - # ) - # profiler( - # 'm4 downsampled\n' - # f' ds bins: {nb}\n' - # f' x.shape: {x.shape}\n' - # f' y.shape: {y.shape}\n' - # f' x: {x}\n' - # f' y: {y}\n' - # ) - - # assert y.size == mxmn.size - # don't real-time "shift" the curve to the # left unless we get one of the following: if ( @@ -435,7 +392,9 @@ def graphics_update_cycle( if vlm_chart: # always update y-label - ds.vlm_sticky.update_from_data(*array[-1][['index', 'volume']]) + ds.vlm_sticky.update_from_data( + *array[-1][['index', 'volume']] + ) if ( (xpx < update_uppx or i_diff > 0) @@ -464,17 +423,17 @@ def graphics_update_cycle( if ( mx_vlm_in_view != vars['last_mx_vlm'] ): - # print(f'mx vlm: {last_mx_vlm} -> {mx_vlm_in_view}') yrange = (0, mx_vlm_in_view * 1.375) vlm_chart.view._set_yrange( yrange=yrange, ) + # print(f'mx vlm: {last_mx_vlm} -> {mx_vlm_in_view}') vars['last_mx_vlm'] = mx_vlm_in_view for curve_name, flow in vlm_chart._flows.items(): update_fsp_chart( vlm_chart, - flow.shm, + flow, curve_name, array_key=curve_name, ) @@ -529,7 +488,6 @@ def graphics_update_cycle( # current) tick first order as an optimization where we only # update from the last tick from each type class. # last_clear_updated: bool = False - # for typ, tick in reversed(lasts.items()): # update ohlc sampled price bars if ( @@ -541,7 +499,7 @@ def graphics_update_cycle( array, ) - # iterate in FIFO order per frame + # iterate in FIFO order per tick-frame for typ, tick in lasts.items(): price = tick.get('price') @@ -612,42 +570,34 @@ def graphics_update_cycle( if ( (mx > vars['last_mx']) or (mn < vars['last_mn']) and not chart._static_yrange == 'axis' + and r > i_step # the last datum is in view ): - # print(f'new y range: {(mn, mx)}') - chart.view._set_yrange( - yrange=(mn, mx), - # TODO: we should probably scale - # the view margin based on the size - # of the true range? This way you can - # slap in orders outside the current - # L1 (only) book range. - # range_margin=0.1, - ) + main_vb = chart.view + if ( + main_vb._ic is None + or not main_vb._ic.is_set() + ): + main_vb._set_yrange( + # TODO: we should probably scale + # the view margin based on the size + # of the true range? This way you can + # slap in orders outside the current + # L1 (only) book range. + # range_margin=0.1, + yrange=(mn, mx), + ) vars['last_mx'], vars['last_mn'] = mx, mn - # run synchronous update on all derived fsp subplots - for name, subchart in ds.linked.subplots.items(): - if name == 'volume': + # run synchronous update on all linked flows + for curve_name, flow in chart._flows.items(): + # TODO: should the "main" (aka source) flow be special? + if curve_name == chart.data_key: continue - update_fsp_chart( - subchart, - subchart._shm, - - # XXX: do we really needs seperate names here? - name, - array_key=name, - ) - subchart.cv._set_yrange() - - # TODO: all overlays on all subplots.. - - # run synchronous update on all derived overlays - for curve_name, flow in chart._flows.items(): update_fsp_chart( chart, - flow.shm, + flow, curve_name, array_key=curve_name, ) @@ -743,6 +693,7 @@ async def display_symbol_data( # TODO: a data view api that makes this less shit chart._shm = ohlcv + chart._flows[chart.data_key].shm = ohlcv # NOTE: we must immediately tell Qt to show the OHLC chart # to avoid a race where the subplots get added/shown to @@ -799,7 +750,7 @@ async def display_symbol_data( # that it isn't double rendered in the display loop # above since we do a maxmin calc on the volume data to # determine if auto-range adjustements should be made. - linked.subplots.pop('volume', None) + # linked.subplots.pop('volume', None) # TODO: make this not so shit XD # close group status From f2f00dcc52bee467101a395e1fbf5b89b46b4a1e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 7 Apr 2022 14:15:18 -0400 Subject: [PATCH 81/84] Drop `._ic` debugging prints --- piker/ui/_interaction.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index c6411caf..4872f595 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -393,6 +393,11 @@ class ChartView(ViewBox): def start_ic( self, ) -> None: + ''' + Signal the beginning of a click-drag interaction + to any interested task waiters. + + ''' if self._ic is None: self.chart.pause_all_feeds() self._ic = trio.Event() @@ -400,13 +405,13 @@ class ChartView(ViewBox): def signal_ic( self, *args, - # ev = None, - ) -> None: - if args: - print(f'range change dun: {args}') - else: - print('proxy called') + ) -> None: + ''' + Signal the end of a click-drag interaction + to any waiters. + + ''' if self._ic: self._ic.set() self._ic = None @@ -674,7 +679,6 @@ class ChartView(ViewBox): self.sigRangeChangedManually.emit(self.state['mouseEnabled']) if ev.isFinish(): - print('DRAG FINISH') self.signal_ic() # self._ic.set() # self._ic = None From 62d08eaf85a76b3e6f6a3a7eef7852082abc8688 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 7 Apr 2022 14:20:19 -0400 Subject: [PATCH 82/84] Tweak log-scaler for more detail --- piker/ui/_compression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index cdc8bb2e..986bc7db 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -260,12 +260,13 @@ def ds_m4( if log_scale: assert uppx, 'You must provide a `uppx` value to use log scaling!' + # scaler = 2**7 / (1 + math.log(uppx, 2)) scaler = round( max( # NOTE: found that a 16x px width brought greater # detail, likely due to dpi scaling? # px_width=px_width * 16, - 2**6 / (1 + math.log(uppx, 2)), + 2**7 / (1 + math.log(uppx, 2)), 1 ) ) From cdc882657a98485c32b5c15980bd9329c860d42c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 3 Apr 2022 23:35:32 -0400 Subject: [PATCH 83/84] Drop old `pyqtgraph` downsample code --- piker/ui/_compression.py | 47 ---------------------------------------- 1 file changed, 47 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index 986bc7db..adb42251 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -106,53 +106,6 @@ def trace_hl( return out -def downsample( - x: np.ndarray, - y: np.ndarray, - bins: int = 2, - - method: str = 'peak', - - **kwargs, - -) -> tuple[np.ndarray, np.ndarray]: - ''' - Downsample x/y data for lesser curve graphics gen. - - The "peak" method is originally copied verbatim from - ``pyqtgraph.PlotDataItem.getDisplayDataset()`` which gets - all credit, though we will likely drop this in favor of the M4 - algo below. - - ''' - # py3.10 syntax - match method: - case 'peak': - if bins < 2: - log.warning('No downsampling taking place?') - - ds = bins - n = len(x) // ds - x1 = np.empty((n, 2)) - - # start of x-values; try to select a somewhat centered point - stx = ds // 2 - x1[:] = x[stx:stx+n*ds:ds, np.newaxis] - x = x1.reshape(n*2) - - y1 = np.empty((n, 2)) - y2 = y[:n*ds].reshape((n, ds)) - - y1[:, 0] = y2.max(axis=1) - y1[:, 1] = y2.min(axis=1) - y = y1.reshape(n*2) - - return x, y - - case 'm4': - return ds_m4(x, y, kwargs['px_width']) - - def ohlc_flatten( ohlc: np.ndarray, use_mxmn: bool = True, From 5921d18d66fb16a2f34e96c3ebcdd316163a920a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 12 Apr 2022 11:40:48 -0400 Subject: [PATCH 84/84] Only update y-range from L1 mxmn when last index in view We still have to always keep track of the last max and min though. --- piker/ui/_display.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 27fcc02e..82f12196 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -382,6 +382,7 @@ def graphics_update_cycle( i_diff > 0 # no new sample step and xpx < 4 # chart is zoomed out very far and r >= i_step # the last datum isn't in view + and liv ) or trigger_all ): @@ -399,7 +400,7 @@ def graphics_update_cycle( if ( (xpx < update_uppx or i_diff > 0) or trigger_all - and r >= i_step + and liv ): # TODO: make it so this doesn't have to be called # once the $vlm is up? @@ -549,7 +550,10 @@ def graphics_update_cycle( l1.bid_label.fields['level']: l1.bid_label, }.get(price) - if label is not None: + if ( + label is not None + # and liv + ): label.update_fields( {'level': price, 'size': size} ) @@ -558,19 +562,27 @@ def graphics_update_cycle( # the relevant L1 queue? # label.size -= size - # elif ticktype in ('ask', 'asize'): - elif typ in _tick_groups['asks']: + elif ( + typ in _tick_groups['asks'] + # TODO: instead we could check if the price is in the + # y-view-range? + # and liv + ): l1.ask_label.update_fields({'level': price, 'size': size}) - # elif ticktype in ('bid', 'bsize'): - elif typ in _tick_groups['bids']: + elif ( + typ in _tick_groups['bids'] + # TODO: instead we could check if the price is in the + # y-view-range? + # and liv + ): l1.bid_label.update_fields({'level': price, 'size': size}) # check for y-range re-size if ( (mx > vars['last_mx']) or (mn < vars['last_mn']) and not chart._static_yrange == 'axis' - and r > i_step # the last datum is in view + and liv ): main_vb = chart.view if (