From c097016fd2a0489fdd77387bb8f087316af1dfc0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 14 Apr 2022 09:38:25 -0400 Subject: [PATCH 001/113] Add new `ui._flows` module This begins the removal of data processing / analysis methods from the chart widget and instead moving them to our new `Flow` API (in the new module introduce here) and delegating the old chart methods to the respective internal flow. Most importantly is no longer storing the "last read" of an array from shm in an internal chart table (was `._arrays`) and instead the `ShmArray` instance is passed as input and stored in the `Flow` instance. This greatly simplifies lookup logic such that the display loop now doesn't have to worry about reading shm, it can be done by internal graphics logic as desired. Generally speaking, all previous `._arrays`/`._graphics` lookups are now delegated to the entries in the chart's `._flows` table. The new `Flow` methods are generally better factored and provide more detailed output regarding data-stream <-> graphics inter-relations for the future purpose of allowing much more efficient update calls in the display loop as well as supporting low latency interaction UX. The concept here is that we're introducing an intermediary layer that ties together graphics and real-time data flows such that widget code is oriented around plot layout and the flow apis are oriented around real-time low latency updates and providing an efficient high level metric layer for the UX. The summary api transition is something like: - `update_graphics_from_array()` -> `.update_graphics_from_flow()` - `.bars_range()` -> `Flow.datums_range()` - `.bars_range()` -> `Flow.datums_range()` --- piker/ui/_chart.py | 277 +++++++++-------------------------------- piker/ui/_flows.py | 303 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 360 insertions(+), 220 deletions(-) create mode 100644 piker/ui/_flows.py diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index a3a97164..8aa10091 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -34,9 +34,7 @@ from PyQt5.QtWidgets import ( QVBoxLayout, QSplitter, ) -import msgspec import numpy as np -# from pydantic import BaseModel import pyqtgraph as pg import trio @@ -49,6 +47,7 @@ from ._cursor import ( Cursor, ContentsLabel, ) +from ..data._sharedmem import ShmArray from ._l1 import L1Labels from ._ohlc import BarItems from ._curve import FastAppendCurve @@ -60,15 +59,12 @@ from ._style import ( ) from ..data.feed import Feed from ..data._source import Symbol -from ..data._sharedmem import ( - ShmArray, - # _Token, -) from ..log import get_logger from ._interaction import ChartView from ._forms import FieldsForm from .._profile import pg_profile_enabled, ms_slower_then from ._overlay import PlotItemOverlay +from ._flows import Flow if TYPE_CHECKING: from ._display import DisplayState @@ -419,7 +415,7 @@ class LinkedSplits(QWidget): self, symbol: Symbol, - array: np.ndarray, + shm: ShmArray, sidepane: FieldsForm, style: str = 'bar', @@ -444,7 +440,7 @@ class LinkedSplits(QWidget): self.chart = self.add_plot( name=symbol.key, - array=array, + shm=shm, style=style, _is_main=True, @@ -472,7 +468,7 @@ class LinkedSplits(QWidget): self, name: str, - array: np.ndarray, + shm: ShmArray, array_key: Optional[str] = None, style: str = 'line', @@ -516,7 +512,6 @@ class LinkedSplits(QWidget): name=name, data_key=array_key or name, - array=array, parent=qframe, linkedsplits=self, axisItems=axes, @@ -580,7 +575,7 @@ class LinkedSplits(QWidget): graphics, data_key = cpw.draw_ohlc( name, - array, + shm, array_key=array_key ) self.cursor.contents_labels.add_label( @@ -594,7 +589,7 @@ class LinkedSplits(QWidget): add_label = True graphics, data_key = cpw.draw_curve( name, - array, + shm, array_key=array_key, color='default_light', ) @@ -603,7 +598,7 @@ class LinkedSplits(QWidget): add_label = True graphics, data_key = cpw.draw_curve( name, - array, + shm, array_key=array_key, step_mode=True, color='davies', @@ -691,7 +686,6 @@ class ChartPlotWidget(pg.PlotWidget): # the "data view" we generate graphics from name: str, - array: np.ndarray, data_key: str, linkedsplits: LinkedSplits, @@ -744,14 +738,6 @@ class ChartPlotWidget(pg.PlotWidget): self._max_l1_line_len: float = 0 # self.setViewportMargins(0, 0, 0, 0) - # self._ohlc = array # readonly view of ohlc data - - # TODO: move to Aggr above XD - # readonly view of data arrays - self._arrays = { - self.data_key: array, - } - self._graphics = {} # registry of underlying graphics # registry of overlay curve names self._flows: dict[str, Flow] = {} @@ -767,7 +753,6 @@ class ChartPlotWidget(pg.PlotWidget): # show background grid self.showGrid(x=False, y=True, alpha=0.3) - self.default_view() self.cv.enable_auto_yrange() self.pi_overlay: PlotItemOverlay = PlotItemOverlay(self.plotItem) @@ -816,14 +801,8 @@ class ChartPlotWidget(pg.PlotWidget): Return a range tuple for the bars present in view. ''' - l, r = self.view_range() - array = self._arrays[self.name] - start, stop = self._xrange = ( - array[0]['index'], - array[-1]['index'], - ) - lbar = max(l, start) - rbar = min(r, stop) + main_flow = self._flows[self.name] + ifirst, l, lbar, rbar, r, ilast = main_flow.datums_range() return l, lbar, rbar, r def curve_width_pxs( @@ -877,40 +856,51 @@ class ChartPlotWidget(pg.PlotWidget): def default_view( self, - steps_on_screen: Optional[int] = None + bars_from_y: int = 5000, ) -> None: ''' Set the view box to the "default" startup view of the scene. ''' - try: - index = self._arrays[self.name]['index'] - except IndexError: - log.warning(f'array for {self.name} not loaded yet?') + flow = self._flows.get(self.name) + if not flow: + log.warning(f'`Flow` for {self.name} not loaded yet?') return + index = flow.shm.array['index'] xfirst, xlast = index[0], index[-1] l, lbar, rbar, r = self.bars_range() - - marker_pos, l1_len = self.pre_l1_xs() - end = xlast + l1_len + 1 + view = self.view if ( rbar < 0 or l < xfirst + or l < 0 or (rbar - lbar) < 6 ): - # set fixed bars count on screen that approx includes as + # TODO: set fixed bars count on screen that approx includes as # many bars as possible before a downsample line is shown. - begin = xlast - round(6116 / 6) + begin = xlast - bars_from_y + view.setXRange( + min=begin, + max=xlast, + padding=0, + ) + # re-get range + l, lbar, rbar, r = self.bars_range() - else: - begin = end - (r - l) + # we get the L1 spread label "length" in view coords + # terms now that we've scaled either by user control + # or to the default set of bars as per the immediate block + # above. + marker_pos, l1_len = self.pre_l1_xs() + end = xlast + l1_len + 1 + begin = end - (r - l) # for debugging # print( - # f'bars range: {brange}\n' + # # f'bars range: {brange}\n' # f'xlast: {xlast}\n' # f'marker pos: {marker_pos}\n' # f'l1 len: {l1_len}\n' @@ -922,14 +912,13 @@ class ChartPlotWidget(pg.PlotWidget): if self._static_yrange == 'axis': self._static_yrange = None - view = self.view view.setXRange( min=begin, max=end, padding=0, ) - view._set_yrange() self.view.maybe_downsample_graphics() + view._set_yrange() try: self.linked.graphics_cycle() except IndexError: @@ -960,7 +949,7 @@ class ChartPlotWidget(pg.PlotWidget): def draw_ohlc( self, name: str, - data: np.ndarray, + shm: ShmArray, array_key: Optional[str] = None, @@ -980,19 +969,21 @@ class ChartPlotWidget(pg.PlotWidget): # the np array buffer to be drawn on next render cycle self.plotItem.addItem(graphics) - # draw after to allow self.scene() to work... - graphics.draw_from_data(data) - data_key = array_key or name - self._graphics[data_key] = graphics self._flows[data_key] = Flow( name=name, plot=self.plotItem, + _shm=shm, is_ohlc=True, graphics=graphics, ) + # TODO: i think we can eventually remove this if + # we write the ``Flow.update_graphics()`` method right? + # draw after to allow self.scene() to work... + graphics.draw_from_data(shm.array) + self._add_sticky(name, bg_color='davies') return graphics, data_key @@ -1058,7 +1049,7 @@ class ChartPlotWidget(pg.PlotWidget): self, name: str, - data: np.ndarray, + shm: ShmArray, array_key: Optional[str] = None, overlay: bool = False, @@ -1071,7 +1062,7 @@ class ChartPlotWidget(pg.PlotWidget): ) -> (pg.PlotDataItem, str): ''' Draw a "curve" (line plot graphics) for the provided data in - the input array ``data``. + the input shm array ``shm``. ''' color = color or self.pen_color or 'default_light' @@ -1082,6 +1073,7 @@ class ChartPlotWidget(pg.PlotWidget): data_key = array_key or name # yah, we wrote our own B) + data = shm.array curve = FastAppendCurve( y=data[data_key], x=data['index'], @@ -1105,16 +1097,14 @@ class ChartPlotWidget(pg.PlotWidget): # and is disastrous for performance. # curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache) - # register curve graphics and backing array for name - self._graphics[name] = curve - self._arrays[data_key] = data - pi = pi or self.plotItem self._flows[data_key] = Flow( name=name, plot=pi, + _shm=shm, is_ohlc=False, + # register curve graphics with this flow graphics=curve, ) @@ -1175,16 +1165,11 @@ class ChartPlotWidget(pg.PlotWidget): ) return last - def update_graphics_from_array( + def update_graphics_from_flow( self, graphics_name: str, - - array: Optional[np.ndarray] = None, array_key: Optional[str] = None, - use_vr: bool = True, - render: bool = True, - **kwargs, ) -> pg.GraphicsObject: @@ -1192,63 +1177,11 @@ class ChartPlotWidget(pg.PlotWidget): Update the named internal graphics from ``array``. ''' - if array is not None: - assert len(array) - - data_key = array_key or graphics_name - if graphics_name not in self._flows: - data_key = self.name - - if array is not None: - # write array to internal graphics table - self._arrays[data_key] = array - else: - array = self._arrays[data_key] - - # array key and graphics "name" might be different.. - graphics = self._graphics[graphics_name] - - # compute "in-view" indices - l, lbar, rbar, r = self.bars_range() - indexes = array['index'] - ifirst = indexes[0] - ilast = indexes[-1] - - lbar_i = max(l, ifirst) - ifirst - rbar_i = min(r, ilast) - ifirst - - # TODO: we could do it this way as well no? - # to_draw = array[lbar - ifirst:(rbar - ifirst) + 1] - in_view = array[lbar_i: rbar_i + 1] - - if ( - not in_view.size - or not render - ): - return graphics - - if isinstance(graphics, BarItems): - graphics.update_from_array( - array, - in_view, - view_range=(lbar_i, rbar_i) if use_vr else None, - - **kwargs, - ) - - else: - graphics.update_from_array( - x=array['index'], - y=array[data_key], - - x_iv=in_view['index'], - y_iv=in_view[data_key], - view_range=(lbar_i, rbar_i) if use_vr else None, - - **kwargs - ) - - return graphics + flow = self._flows[array_key or graphics_name] + return flow.update_graphics( + array_key=array_key, + **kwargs, + ) # def _label_h(self, yhigh: float, ylow: float) -> float: # # compute contents label "height" in view terms @@ -1295,7 +1228,7 @@ class ChartPlotWidget(pg.PlotWidget): # TODO: this should go onto some sort of # data-view thinger..right? - ohlc = self._shm.array + ohlc = self._flows[self.name].shm.array # XXX: not sure why the time is so off here # looks like we're gonna have to do some fixing.. @@ -1341,9 +1274,6 @@ class ChartPlotWidget(pg.PlotWidget): delayed=True, ) - l, lbar, rbar, r = bars_range or self.bars_range() - profiler(f'{self.name} got bars range') - # TODO: here we should instead look up the ``Flow.shm.array`` # and read directly from shm to avoid copying to memory first # and then reading it again here. @@ -1356,6 +1286,9 @@ class ChartPlotWidget(pg.PlotWidget): res = 0, 0 else: + first, l, lbar, rbar, r, last = bars_range or flow.datums_range() + profiler(f'{self.name} got bars range') + key = round(lbar), round(rbar) res = flow.maxmin(*key) profiler(f'yrange mxmn: {key} -> {res}') @@ -1366,99 +1299,3 @@ class ChartPlotWidget(pg.PlotWidget): res = 0, 0 return res - - -# class FlowsTable(pydantic.BaseModel): -# ''' -# Data-AGGRegate: high level API onto multiple (categorized) -# ``Flow``s with high level processing routines for -# multi-graphics computations and display. - -# ''' -# flows: dict[str, np.ndarray] = {} - - -class Flow(msgspec.Struct): # , frozen=True): - ''' - (FinancialSignal-)Flow compound type which wraps a real-time - graphics (curve) and its backing data stream together for high level - access and control. - - The intention is for this type to eventually be capable of shm-passing - of incrementally updated graphics stream data between actors. - - ''' - name: str - plot: pg.PlotItem - is_ohlc: bool = False - graphics: pg.GraphicsObject - - # TODO: hackery to be able to set a shm later - # but whilst also allowing this type to hashable, - # likely will require serializable token that is used to attach - # to the underlying shm ref after startup? - _shm: Optional[ShmArray] = None # currently, may be filled in "later" - - # cache of y-range values per x-range input. - _mxmns: dict[tuple[int, int], tuple[float, float]] = {} - - @property - def shm(self) -> ShmArray: - return self._shm - - @shm.setter - def shm(self, shm: ShmArray) -> ShmArray: - self._shm = shm - - def maxmin( - self, - lbar, - rbar, - - ) -> tuple[float, float]: - ''' - Compute the cached max and min y-range values for a given - x-range determined by ``lbar`` and ``rbar``. - - ''' - rkey = (lbar, rbar) - cached_result = self._mxmns.get(rkey) - if cached_result: - return cached_result - - shm = self.shm - if shm is None: - mxmn = None - - else: # new block for profiling?.. - arr = shm.array - - # build relative indexes into shm array - # TODO: should we just add/use a method - # on the shm to do this? - ifirst = arr[0]['index'] - slice_view = arr[ - lbar - ifirst: - (rbar - ifirst) + 1 - ] - - if not slice_view.size: - mxmn = None - - else: - if self.is_ohlc: - ylow = np.min(slice_view['low']) - yhigh = np.max(slice_view['high']) - - else: - view = slice_view[self.name] - ylow = np.min(view) - yhigh = np.max(view) - - mxmn = ylow, yhigh - - if mxmn is not None: - # cache new mxmn result - self._mxmns[rkey] = mxmn - - return mxmn diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py new file mode 100644 index 00000000..a9eb6a4f --- /dev/null +++ b/piker/ui/_flows.py @@ -0,0 +1,303 @@ +# piker: trading gear for hackers +# Copyright (C) Tyler Goodlet (in stewardship for pikers) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +''' +High level streaming graphics primitives. + +This is an intermediate layer which associates real-time low latency +graphics primitives with underlying FSP related data structures for fast +incremental update. + +''' +from typing import ( + Optional, + Callable, +) + +import msgspec +import numpy as np +import pyqtgraph as pg +from PyQt5.QtGui import QPainterPath + +from ..data._sharedmem import ( + ShmArray, + # attach_shm_array +) +from ._ohlc import BarItems + + +# class FlowsTable(msgspec.Struct): +# ''' +# Data-AGGRegate: high level API onto multiple (categorized) +# ``Flow``s with high level processing routines for +# multi-graphics computations and display. + +# ''' +# flows: dict[str, np.ndarray] = {} + + +class Flow(msgspec.Struct): # , frozen=True): + ''' + (FinancialSignal-)Flow compound type which wraps a real-time + graphics (curve) and its backing data stream together for high level + access and control. + + The intention is for this type to eventually be capable of shm-passing + of incrementally updated graphics stream data between actors. + + ''' + name: str + plot: pg.PlotItem + is_ohlc: bool = False + render: bool = True # toggle for display loop + + graphics: pg.GraphicsObject + + # TODO: hackery to be able to set a shm later + # but whilst also allowing this type to hashable, + # likely will require serializable token that is used to attach + # to the underlying shm ref after startup? + _shm: Optional[ShmArray] = None # currently, may be filled in "later" + + # last read from shm (usually due to an update call) + _last_read: Optional[np.ndarray] = None + + # cache of y-range values per x-range input. + _mxmns: dict[tuple[int, int], tuple[float, float]] = {} + + @property + def shm(self) -> ShmArray: + return self._shm + + # TODO: remove this and only allow setting through + # private ``._shm`` attr? + @shm.setter + def shm(self, shm: ShmArray) -> ShmArray: + print(f'{self.name} DO NOT SET SHM THIS WAY!?') + self._shm = shm + + def maxmin( + self, + lbar, + rbar, + + ) -> tuple[float, float]: + ''' + Compute the cached max and min y-range values for a given + x-range determined by ``lbar`` and ``rbar``. + + ''' + rkey = (lbar, rbar) + cached_result = self._mxmns.get(rkey) + if cached_result: + return cached_result + + shm = self.shm + if shm is None: + mxmn = None + + else: # new block for profiling?.. + arr = shm.array + + # build relative indexes into shm array + # TODO: should we just add/use a method + # on the shm to do this? + ifirst = arr[0]['index'] + slice_view = arr[ + lbar - ifirst: + (rbar - ifirst) + 1 + ] + + if not slice_view.size: + mxmn = None + + else: + if self.is_ohlc: + ylow = np.min(slice_view['low']) + yhigh = np.max(slice_view['high']) + + else: + view = slice_view[self.name] + ylow = np.min(view) + yhigh = np.max(view) + + mxmn = ylow, yhigh + + if mxmn is not None: + # cache new mxmn result + self._mxmns[rkey] = mxmn + + return mxmn + + def view_range(self) -> tuple[int, int]: + ''' + Return the indexes in view for the associated + plot displaying this flow's data. + + ''' + vr = self.plot.viewRect() + return int(vr.left()), int(vr.right()) + + def datums_range(self) -> tuple[ + int, int, int, int, int, int + ]: + ''' + Return a range tuple for the datums present in view. + + ''' + l, r = self.view_range() + + # TODO: avoid this and have shm passed + # in earlier. + if self.shm is None: + # haven't initialized the flow yet + return (0, l, 0, 0, r, 0) + + array = self.shm.array + index = array['index'] + start = index[0] + end = index[-1] + lbar = max(l, start) + rbar = min(r, end) + return ( + start, l, lbar, rbar, r, end, + ) + + def read(self) -> tuple[ + int, int, np.ndarray, + int, int, np.ndarray, + ]: + array = self.shm.array + indexes = array['index'] + ifirst = indexes[0] + ilast = indexes[-1] + + ifirst, l, lbar, rbar, r, ilast = self.datums_range() + + # get read-relative indices adjusting + # for master shm index. + lbar_i = max(l, ifirst) - ifirst + rbar_i = min(r, ilast) - ifirst + + # TODO: we could do it this way as well no? + # to_draw = array[lbar - ifirst:(rbar - ifirst) + 1] + in_view = array[lbar_i: rbar_i + 1] + + return ( + # abs indices + full data set + ifirst, ilast, array, + + # relative indices + in view datums + lbar_i, rbar_i, in_view, + ) + + def update_graphics( + self, + use_vr: bool = True, + render: bool = True, + array_key: Optional[str] = None, + + **kwargs, + + ) -> pg.GraphicsObject: + ''' + Read latest datums from shm and render to (incrementally) + render to graphics. + + ''' + # shm read and slice to view + xfirst, xlast, array, ivl, ivr, in_view = self.read() + + if ( + not in_view.size + or not render + ): + return self.graphics + + array_key = array_key or self.name + + graphics = self.graphics + if isinstance(graphics, BarItems): + graphics.update_from_array( + array, + in_view, + view_range=(ivl, ivr) if use_vr else None, + + **kwargs, + ) + + else: + graphics.update_from_array( + x=array['index'], + y=array[array_key], + + x_iv=in_view['index'], + y_iv=in_view[array_key], + view_range=(ivl, ivr) if use_vr else None, + + **kwargs + ) + + return graphics + + # @classmethod + # def from_token( + # cls, + # shm_token: tuple[ + # str, + # str, + # tuple[str, str], + # ], + + # ) -> PathRenderer: + + # shm = attach_shm_array(token) + # return cls(shm) + + +class PathRenderer(msgspec.Struct): + + # output graphics rendering + path: Optional[QPainterPath] = None + + last_read_src_array: np.ndarray + # called on input data but before + prerender_fn: Callable[ShmArray, np.ndarray] + + def diff( + self, + ) -> dict[str, np.ndarray]: + ... + + def update(self) -> QPainterPath: + ''' + Incrementally update the internal path graphics from + updates in shm data and deliver the new (sub)-path + generated. + + ''' + ... + + + def render( + self, + + ) -> list[QPainterPath]: + ''' + Render the current graphics path(s) + + ''' + ... From 599c77ff845e446a1490192250afccfd9c038606 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 14 Apr 2022 10:04:18 -0400 Subject: [PATCH 002/113] Port ui components to use flows, drop all late assignments of shm --- piker/ui/_axes.py | 5 +++-- piker/ui/_cursor.py | 13 +++++++------ piker/ui/_display.py | 33 ++++++++++++++++----------------- piker/ui/_editors.py | 2 +- piker/ui/_fsp.py | 38 ++++++++++---------------------------- 5 files changed, 37 insertions(+), 54 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 93ac7af7..7ba52055 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -223,8 +223,9 @@ class DynamicDateAxis(Axis): ) -> list[str]: chart = self.linkedsplits.chart - bars = chart._arrays[chart.name] - shm = self.linkedsplits.chart._shm + flow = chart._flows[chart.name] + shm = flow.shm + bars = shm.array first = shm._first.value bars_len = len(bars) diff --git a/piker/ui/_cursor.py b/piker/ui/_cursor.py index a34c15c1..43207b9f 100644 --- a/piker/ui/_cursor.py +++ b/piker/ui/_cursor.py @@ -254,13 +254,13 @@ class ContentsLabels: def update_labels( self, index: int, - # array_name: str, ) -> None: - # for name, (label, update) in self._labels.items(): for chart, name, label, update in self._labels: - array = chart._arrays[name] + flow = chart._flows[name] + array = flow.shm.array + if not ( index >= 0 and index < array[-1]['index'] @@ -269,8 +269,6 @@ class ContentsLabels: print('WTF out of range?') continue - # array = chart._arrays[name] - # call provided update func with data point try: label.show() @@ -472,9 +470,12 @@ class Cursor(pg.GraphicsObject): ) -> LineDot: # if this plot contains curves add line dot "cursors" to denote # the current sample under the mouse + main_flow = plot._flows[plot.name] + # read out last index + i = main_flow.shm.array[-1]['index'] cursor = LineDot( curve, - index=plot._arrays[plot.name][-1]['index'], + index=i, plot=plot ) plot.addItem(cursor) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 927ce5df..4b695b04 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -408,9 +408,8 @@ def graphics_update_cycle( ): # TODO: make it so this doesn't have to be called # once the $vlm is up? - vlm_chart.update_graphics_from_array( + vlm_chart.update_graphics_from_flow( 'volume', - array, # UGGGh, see ``maxmin()`` impl in `._fsp` for # the overlayed plotitems... we need a better @@ -436,6 +435,11 @@ def graphics_update_cycle( vars['last_mx_vlm'] = mx_vlm_in_view for curve_name, flow in vlm_chart._flows.items(): + + if not flow.render: + print(f'skipping flow {curve_name}?') + continue + update_fsp_chart( vlm_chart, flow, @@ -500,9 +504,8 @@ def graphics_update_cycle( or i_diff > 0 or trigger_all ): - chart.update_graphics_from_array( + chart.update_graphics_from_flow( chart.name, - array, ) # iterate in FIFO order per tick-frame @@ -515,8 +518,9 @@ def graphics_update_cycle( # tick frames to determine the y-range for chart # auto-scaling. # TODO: we need a streaming minmax algo here, see def above. - mx = max(price + tick_margin, mx) - mn = min(price - tick_margin, mn) + if liv: + mx = max(price + tick_margin, mx) + mn = min(price - tick_margin, mn) if typ in clear_types: @@ -539,9 +543,8 @@ def graphics_update_cycle( if wap_in_history: # update vwap overlay line - chart.update_graphics_from_array( + chart.update_graphics_from_flow( 'bar_wap', - array, ) # L1 book label-line updates @@ -557,7 +560,7 @@ def graphics_update_cycle( if ( label is not None - # and liv + and liv ): label.update_fields( {'level': price, 'size': size} @@ -571,7 +574,7 @@ def graphics_update_cycle( typ in _tick_groups['asks'] # TODO: instead we could check if the price is in the # y-view-range? - # and liv + and liv ): l1.ask_label.update_fields({'level': price, 'size': size}) @@ -579,7 +582,7 @@ def graphics_update_cycle( typ in _tick_groups['bids'] # TODO: instead we could check if the price is in the # y-view-range? - # and liv + and liv ): l1.bid_label.update_fields({'level': price, 'size': size}) @@ -692,9 +695,10 @@ async def display_symbol_data( # create main OHLC chart chart = linked.plot_ohlc_main( symbol, - bars, + ohlcv, sidepane=pp_pane, ) + chart.default_view() chart._feeds[symbol.key] = feed chart.setFocus() @@ -714,10 +718,6 @@ async def display_symbol_data( # size view to data once at outset chart.cv._set_yrange() - # TODO: a data view api that makes this less shit - chart._shm = ohlcv - chart._flows[chart.data_key].shm = ohlcv - # NOTE: we must immediately tell Qt to show the OHLC chart # to avoid a race where the subplots get added/shown to # the linked set *before* the main price chart! @@ -780,6 +780,5 @@ async def display_symbol_data( sbar._status_groups[loading_sym_key][1]() # let the app run.. bby - chart.default_view() # linked.graphics_cycle() await trio.sleep_forever() diff --git a/piker/ui/_editors.py b/piker/ui/_editors.py index 9a99d2f7..03fd208e 100644 --- a/piker/ui/_editors.py +++ b/piker/ui/_editors.py @@ -343,7 +343,7 @@ class SelectRect(QtGui.QGraphicsRectItem): nbars = ixmx - ixmn + 1 chart = self._chart - data = chart._arrays[chart.name][ixmn:ixmx] + data = chart._flows[chart.name].shm.array[ixmn:ixmx] if len(data): std = data['close'].std() diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 52763375..9aa10fb3 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -93,9 +93,8 @@ def update_fsp_chart( # update graphics # NOTE: this does a length check internally which allows it # staying above the last row check below.. - chart.update_graphics_from_array( + chart.update_graphics_from_flow( graphics_name, - array, array_key=array_key or graphics_name, ) @@ -106,9 +105,6 @@ def update_fsp_chart( # read from last calculated value and update any label last_val_sticky = chart._ysticks.get(graphics_name) if last_val_sticky: - # array = shm.array[array_key] - # if len(array): - # value = array[-1] last = last_row[array_key] last_val_sticky.update_from_data(-1, last) @@ -246,20 +242,18 @@ async def run_fsp_ui( chart.draw_curve( name=name, - data=shm.array, + shm=shm, overlay=True, color='default_light', array_key=name, **conf.get('chart_kwargs', {}) ) - # specially store ref to shm for lookup in display loop - chart._flows[name].shm = shm else: # create a new sub-chart widget for this fsp chart = linkedsplits.add_plot( name=name, - array=shm.array, + shm=shm, array_key=name, sidepane=sidepane, @@ -271,12 +265,6 @@ async def run_fsp_ui( **conf.get('chart_kwargs', {}) ) - # XXX: ONLY for sub-chart fsps, overlays have their - # data looked up from the chart's internal array set. - # TODO: we must get a data view api going STAT!! - chart._shm = shm - chart._flows[chart.data_key].shm = shm - # should **not** be the same sub-chart widget assert chart.name != linkedsplits.chart.name @@ -626,7 +614,7 @@ async def open_vlm_displays( shm = ohlcv chart = linked.add_plot( name='volume', - array=shm.array, + shm=shm, array_key='volume', sidepane=sidepane, @@ -639,7 +627,6 @@ async def open_vlm_displays( # the curve item internals are pretty convoluted. style='step', ) - chart._flows['volume'].shm = ohlcv # force 0 to always be in view def maxmin( @@ -666,11 +653,6 @@ async def open_vlm_displays( # chart.hideAxis('right') # chart.showAxis('left') - # XXX: ONLY for sub-chart fsps, overlays have their - # data looked up from the chart's internal array set. - # TODO: we must get a data view api going STAT!! - chart._shm = shm - # send back new chart to caller task_status.started(chart) @@ -685,9 +667,9 @@ async def open_vlm_displays( last_val_sticky.update_from_data(-1, value) - vlm_curve = chart.update_graphics_from_array( + vlm_curve = chart.update_graphics_from_flow( 'volume', - shm.array, + # shm.array, ) # size view to data once at outset @@ -795,9 +777,8 @@ async def open_vlm_displays( color = 'bracket' curve, _ = chart.draw_curve( - # name='dolla_vlm', name=name, - data=shm.array, + shm=shm, array_key=name, overlay=pi, color=color, @@ -812,7 +793,6 @@ async def open_vlm_displays( # ``.draw_curve()``. flow = chart._flows[name] assert flow.plot is pi - flow.shm = shm chart_curves( fields, @@ -847,7 +827,9 @@ async def open_vlm_displays( # liquidity events (well at least on low OHLC periods - 1s). vlm_curve.hide() chart.removeItem(vlm_curve) - chart._flows.pop('volume') + vflow = chart._flows['volume'] + vflow.render = False + # avoid range sorting on volume once disabled chart.view.disable_auto_yrange() From d0af280a59cf8a0af382609ecffa1820ad8eb6d6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 14 Apr 2022 10:10:38 -0400 Subject: [PATCH 003/113] Port view downsampling handler to new update apis --- piker/ui/_interaction.py | 89 +++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 4872f595..4b3bbb45 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -492,9 +492,9 @@ class ChartView(ViewBox): log.debug("Max zoom bruh...") return - if ev.delta() < 0 and vl >= len(chart._arrays[chart.name]) + 666: - log.debug("Min zoom bruh...") - return + # if ev.delta() < 0 and vl >= len(chart._flows[chart.name].shm.array) + 666: + # log.debug("Min zoom bruh...") + # return # actual scaling factor s = 1.015 ** (ev.delta() * -1 / 20) # self.state['wheelScaleFactor']) @@ -777,9 +777,15 @@ class ChartView(ViewBox): # calculate max, min y values in viewable x-range from data. # Make sure min bars/datums on screen is adhered. - else: - br = bars_range or chart.bars_range() - profiler(f'got bars range: {br}') + # else: + # TODO: eventually we should point to the + # ``FlowsTable`` (or wtv) which should perform + # the group operations? + + # flow = chart._flows[name or chart.name] + # br = bars_range or chart.bars_range() + # br = bars_range or chart.bars_range() + # profiler(f'got bars range: {br}') # TODO: maybe should be a method on the # chart widget/item? @@ -830,6 +836,8 @@ class ChartView(ViewBox): self.setYRange(ylow, yhigh) profiler(f'set limits: {(ylow, yhigh)}') + profiler.finish() + def enable_auto_yrange( self, src_vb: Optional[ChartView] = None, @@ -890,7 +898,7 @@ class ChartView(ViewBox): graphics items which are our children. ''' - graphics = list(self._chart._graphics.values()) + graphics = [f.graphics for f in self._chart._flows.values()] if not graphics: return 0 @@ -903,44 +911,49 @@ class ChartView(ViewBox): def maybe_downsample_graphics(self): + profiler = pg.debug.Profiler( + disabled=not pg_profile_enabled(), + gt=3, + ) + uppx = self.x_uppx() - if ( + if not ( # we probably want to drop this once we are "drawing in # view" for downsampled flows.. uppx and uppx > 16 and self._ic is not None ): + + # TODO: a faster single-loop-iterator way of doing this XD + chart = self._chart + linked = self.linkedsplits + plots = linked.subplots | {chart.name: chart} + for chart_name, chart in plots.items(): + for name, flow in chart._flows.items(): + + if not flow.render: + continue + + graphics = flow.graphics + + use_vr = False + if isinstance(graphics, BarItems): + use_vr = True + + # pass in no array which will read and render from the last + # passed array (normally provided by the display loop.) + chart.update_graphics_from_flow( + name, + use_vr=use_vr, + + # gets passed down into graphics obj + profiler=profiler, + ) + + profiler(f'range change updated {chart_name}:{name}') + else: # don't bother updating since we're zoomed out bigly and # in a pan-interaction, in which case we shouldn't be # doing view-range based rendering (at least not yet). # print(f'{uppx} exiting early!') - return - - profiler = pg.debug.Profiler( - disabled=not pg_profile_enabled(), - gt=3, - delayed=True, - ) - - # TODO: a faster single-loop-iterator way of doing this XD - chart = self._chart - linked = self.linkedsplits - plots = linked.subplots | {chart.name: chart} - for chart_name, chart in plots.items(): - for name, flow in chart._flows.items(): - graphics = flow.graphics - - use_vr = False - if isinstance(graphics, BarItems): - use_vr = True - - # pass in no array which will read and render from the last - # passed array (normally provided by the display loop.) - chart.update_graphics_from_array( - name, - use_vr=use_vr, - profiler=profiler, - ) - profiler(f'range change updated {chart_name}:{name}') - - profiler.finish() + profiler(f'dowsampling skipped - not in uppx range {uppx} <= 16') From 5a9bab0b690a688e6412d75b0d328596ae36f4ea Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 16 Apr 2022 15:22:11 -0400 Subject: [PATCH 004/113] WIP incremental render apis --- piker/ui/_curve.py | 8 +- piker/ui/_display.py | 1 - piker/ui/_flows.py | 224 ++++++++++++++++++++++++++++++++++++------- piker/ui/_ohlc.py | 16 ++-- 4 files changed, 198 insertions(+), 51 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 00a4ca7a..871e55f5 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -290,9 +290,9 @@ class FastAppendCurve(pg.GraphicsObject): # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. - self.xData = x - self.yData = y - self._x, self._y = x, y + # self.xData = x + # self.yData = y + # self._x, self._y = x, y if view_range: profiler(f'view range slice {view_range}') @@ -328,7 +328,7 @@ class FastAppendCurve(pg.GraphicsObject): # x_last = x_iv[-1] # y_last = y_iv[-1] - self._last_vr = view_range + # self._last_vr = view_range # self.disable_cache() # flip_cache = True diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 4b695b04..fda3fb04 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -437,7 +437,6 @@ def graphics_update_cycle( for curve_name, flow in vlm_chart._flows.items(): if not flow.render: - print(f'skipping flow {curve_name}?') continue update_fsp_chart( diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index a9eb6a4f..d5a0d1e1 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -22,6 +22,7 @@ graphics primitives with underlying FSP related data structures for fast incremental update. ''' +from __future__ import annotations from typing import ( Optional, Callable, @@ -36,8 +37,16 @@ from ..data._sharedmem import ( ShmArray, # attach_shm_array ) -from ._ohlc import BarItems - +from ._ohlc import ( + BarItems, + gen_qpath, +) +from ._curve import ( + FastAppendCurve, +) +from ._compression import ( + ohlc_flatten, +) # class FlowsTable(msgspec.Struct): # ''' @@ -48,6 +57,20 @@ from ._ohlc import BarItems # ''' # flows: dict[str, np.ndarray] = {} +# @classmethod +# def from_token( +# cls, +# shm_token: tuple[ +# str, +# str, +# tuple[str, str], +# ], + +# ) -> Renderer: + +# shm = attach_shm_array(token) +# return cls(shm) + class Flow(msgspec.Struct): # , frozen=True): ''' @@ -61,16 +84,28 @@ class Flow(msgspec.Struct): # , frozen=True): ''' name: str plot: pg.PlotItem + graphics: pg.GraphicsObject + _shm: ShmArray + is_ohlc: bool = False render: bool = True # toggle for display loop - graphics: pg.GraphicsObject + _last_uppx: float = 0 + _in_ds: bool = False + + _graphics_tranform_fn: Optional[Callable[ShmArray, np.ndarray]] = None + + # map from uppx -> (downsampled data, incremental graphics) + _render_table: dict[ + Optional[int], + tuple[Renderer, pg.GraphicsItem], + ] = {} # TODO: hackery to be able to set a shm later # but whilst also allowing this type to hashable, # likely will require serializable token that is used to attach # to the underlying shm ref after startup? - _shm: Optional[ShmArray] = None # currently, may be filled in "later" + # _shm: Optional[ShmArray] = None # currently, may be filled in "later" # last read from shm (usually due to an update call) _last_read: Optional[np.ndarray] = None @@ -219,7 +254,7 @@ class Flow(msgspec.Struct): # , frozen=True): ''' # shm read and slice to view - xfirst, xlast, array, ivl, ivr, in_view = self.read() + read = xfirst, xlast, array, ivl, ivr, in_view = self.read() if ( not in_view.size @@ -227,10 +262,48 @@ class Flow(msgspec.Struct): # , frozen=True): ): return self.graphics - array_key = array_key or self.name - graphics = self.graphics if isinstance(graphics, BarItems): + + # ugh, not luvin dis, should we have just a designated + # instance var? + r = self._render_table.get('src') + if not r: + r = Renderer( + flow=self, + draw=gen_qpath, # TODO: rename this to something with ohlc + last_read=read, + ) + self._render_table['src'] = (r, graphics) + + ds_curve_r = Renderer( + flow=self, + draw=gen_qpath, # TODO: rename this to something with ohlc + last_read=read, + prerender_fn=ohlc_flatten, + ) + + # baseline "line" downsampled OHLC curve that should + # kick on only when we reach a certain uppx threshold. + self._render_table[0] = ( + ds_curve_r, + FastAppendCurve( + y=y, + x=x, + name='OHLC', + color=self._color, + ), + ) + + # do checks for whether or not we require downsampling: + # - if we're **not** downsampling then we simply want to + # render the bars graphics curve and update.. + # - if insteam we are in a downsamplig state then we to + # update our pre-downsample-ready data and then pass that + # new data the downsampler algo for incremental update. + else: + # do incremental update + graphics.update_from_array( array, in_view, @@ -239,7 +312,55 @@ class Flow(msgspec.Struct): # , frozen=True): **kwargs, ) + # generate and apply path to graphics obj + graphics.path, last = r.render(only_in_view=True) + graphics.draw_last(last) + else: + # should_ds = False + # should_redraw = False + + # # downsampling incremental state checking + # uppx = bars.x_uppx() + # px_width = bars.px_width() + # uppx_diff = (uppx - self._last_uppx) + + # if self.renderer is None: + # self.renderer = Renderer( + # flow=self, + + # if not self._in_ds: + # # in not currently marked as downsampling graphics + # # then only draw the full bars graphic for datums "in + # # view". + + # # check for downsampling conditions + # if ( + # # std m4 downsample conditions + # px_width + # and uppx_diff >= 4 + # or uppx_diff <= -3 + # or self._step_mode and abs(uppx_diff) >= 4 + + # ): + # log.info( + # f'{self._name} sampler change: {self._last_uppx} -> {uppx}' + # ) + # self._last_uppx = uppx + # should_ds = True + + # elif ( + # uppx <= 2 + # and self._in_ds + # ): + # # we should de-downsample back to our original + # # source data so we clear our path data in prep + # # to generate a new one from original source data. + # should_redraw = True + # should_ds = False + + array_key = array_key or self.name + graphics.update_from_array( x=array['index'], y=array[array_key], @@ -253,51 +374,80 @@ class Flow(msgspec.Struct): # , frozen=True): return graphics - # @classmethod - # def from_token( - # cls, - # shm_token: tuple[ - # str, - # str, - # tuple[str, str], - # ], - # ) -> PathRenderer: +class Renderer(msgspec.Struct): - # shm = attach_shm_array(token) - # return cls(shm) + flow: Flow + # called to render path graphics + draw: Callable[np.ndarray, QPainterPath] -class PathRenderer(msgspec.Struct): + # called on input data but before + prerender_fn: Optional[Callable[ShmArray, np.ndarray]] = None + + prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None + append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None + + # last array view read + last_read: Optional[np.ndarray] = None # output graphics rendering path: Optional[QPainterPath] = None - last_read_src_array: np.ndarray - # called on input data but before - prerender_fn: Callable[ShmArray, np.ndarray] - - def diff( - self, - ) -> dict[str, np.ndarray]: - ... - - def update(self) -> QPainterPath: - ''' - Incrementally update the internal path graphics from - updates in shm data and deliver the new (sub)-path - generated. - - ''' - ... + # def diff( + # self, + # latest_read: tuple[np.ndarray], + # ) -> tuple[np.ndarray]: + # # blah blah blah + # # do diffing for prepend, append and last entry + # return ( + # to_prepend + # to_append + # last, + # ) def render( self, + # only render datums "in view" of the ``ChartView`` + only_in_view: bool = True, + ) -> list[QPainterPath]: ''' Render the current graphics path(s) ''' - ... + # do full source data render to path + xfirst, xlast, array, ivl, ivr, in_view = self.last_read + + if only_in_view: + # get latest data from flow shm + self.last_read = ( + xfirst, xlast, array, ivl, ivr, in_view + ) = self.flow.read() + + array = in_view + + if self.path is None or in_view: + # redraw the entire source data if we have either of: + # - no prior path graphic rendered or, + # - we always intend to re-render the data only in view + + if self.prerender_fn: + array = self.prerender_fn(array) + + hist, last = array[:-1], array[-1] + + # call path render func on history + self.path = self.draw(hist) + + elif self.path: + print(f'inremental update not supported yet {self.flow.name}') + # TODO: do incremental update + # prepend, append, last = self.diff(self.flow.read()) + + # do path generation for each segment + # and then push into graphics object. + + return self.path, last diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 44dbb0c2..bf56f5f5 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -46,7 +46,7 @@ log = get_logger(__name__) def bar_from_ohlc_row( row: np.ndarray, - w: float + w: float = 0.43 ) -> tuple[QLineF]: ''' @@ -158,8 +158,9 @@ def path_arrays_from_ohlc( def gen_qpath( data: np.ndarray, - start: int, # XXX: do we need this? - w: float, + start: int = 0, # XXX: do we need this? + # 0.5 is no overlap between arms, 1.0 is full overlap + w: float = 0.43, path: Optional[QtGui.QPainterPath] = None, ) -> QtGui.QPainterPath: @@ -310,7 +311,7 @@ class BarItems(pg.GraphicsObject): self._pi.addItem(curve) self._ds_line = curve - self._ds_xrange = (index[0], index[-1]) + # self._ds_xrange = (index[0], index[-1]) # trigger render # https://doc.qt.io/qt-5/qgraphicsitem.html#update @@ -358,7 +359,7 @@ class BarItems(pg.GraphicsObject): # index = self.start_index istart, istop = self._xrange - ds_istart, ds_istop = self._ds_xrange + # ds_istart, ds_istop = self._ds_xrange index = ohlc['index'] first_index, last_index = index[0], index[-1] @@ -435,9 +436,6 @@ class BarItems(pg.GraphicsObject): # stop here since we don't need to update bars path any more # as we delegate to the downsample line with updates. - profiler.finish() - # print('terminating early') - return else: # we should be in bars mode @@ -606,7 +604,7 @@ class BarItems(pg.GraphicsObject): if flip_cache: self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - profiler.finish() + # profiler.finish() def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect From e0a72a217405f23ee3a37ca6dd9bf8d003cb24e6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 18 Apr 2022 08:30:28 -0400 Subject: [PATCH 005/113] WIP starting architecture doc str writeup.. --- piker/ui/_flows.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index d5a0d1e1..772aa026 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -302,6 +302,7 @@ class Flow(msgspec.Struct): # , frozen=True): # update our pre-downsample-ready data and then pass that # new data the downsampler algo for incremental update. else: + pass # do incremental update graphics.update_from_array( @@ -417,6 +418,14 @@ class Renderer(msgspec.Struct): ''' Render the current graphics path(s) + There are (at least) 3 stages from source data to graphics data: + - a data transform (which can be stored in additional shm) + - a graphics transform which converts discrete basis data to + a `float`-basis view-coords graphics basis. (eg. ``ohlc_flatten()``, + ``step_path_arrays_from_1d()``, etc.) + + - blah blah blah (from notes) + ''' # do full source data render to path xfirst, xlast, array, ivl, ivr, in_view = self.last_read From f4dc0fbab8918bb63c7b302379036d82da532606 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 20 Apr 2022 11:42:49 -0400 Subject: [PATCH 006/113] Add `BarItems.draw_last()` and disable `.update_from_array()` --- piker/ui/_ohlc.py | 592 +++++++++++++++++++++++++--------------------- 1 file changed, 316 insertions(+), 276 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index bf56f5f5..328d62b9 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -244,7 +244,7 @@ class BarItems(pg.GraphicsObject): self.fast_path = QtGui.QPainterPath() self._xrange: tuple[int, int] - self._yrange: tuple[float, float] + # self._yrange: tuple[float, float] self._vrange = None # TODO: don't render the full backing array each time @@ -281,10 +281,10 @@ class BarItems(pg.GraphicsObject): # self.start_index = len(ohlc) index = ohlc['index'] self._xrange = (index[0], index[-1]) - self._yrange = ( - np.nanmax(ohlc['high']), - np.nanmin(ohlc['low']), - ) + # self._yrange = ( + # np.nanmax(ohlc['high']), + # np.nanmin(ohlc['low']), + # ) # up to last to avoid double draw of last bar self._last_bar_lines = bar_from_ohlc_row(last, self.w) @@ -325,286 +325,326 @@ class BarItems(pg.GraphicsObject): else: return 0 - def update_from_array( + # def update_from_array( + # self, + + # # full array input history + # ohlc: np.ndarray, + + # # pre-sliced array data that's "in view" + # ohlc_iv: np.ndarray, + + # view_range: Optional[tuple[int, int]] = None, + # profiler: Optional[pg.debug.Profiler] = None, + + # ) -> None: + # ''' + # Update the last datum's bar graphic from input data array. + + # This routine should be interface compatible with + # ``pg.PlotCurveItem.setData()``. Normally this method in + # ``pyqtgraph`` seems to update all the data passed to the + # graphics object, and then update/rerender, but here we're + # assuming the prior graphics havent changed (OHLC history rarely + # does) so this "should" be simpler and faster. + + # This routine should be made (transitively) as fast as possible. + + # ''' + # profiler = profiler or pg.debug.Profiler( + # disabled=not pg_profile_enabled(), + # gt=ms_slower_then, + # delayed=True, + # ) + + # # index = self.start_index + # istart, istop = self._xrange + # # ds_istart, ds_istop = self._ds_xrange + + # index = ohlc['index'] + # first_index, last_index = index[0], index[-1] + + # # length = len(ohlc) + # # prepend_length = istart - first_index + # # append_length = last_index - istop + + # # ds_prepend_length = ds_istart - first_index + # # ds_append_length = last_index - ds_istop + + # flip_cache = False + + # x_gt = 16 + # if self._ds_line: + # uppx = self._ds_line.x_uppx() + # else: + # uppx = 0 + + # should_line = self._in_ds + # if ( + # self._in_ds + # and uppx < x_gt + # ): + # should_line = False + + # elif ( + # not self._in_ds + # and uppx >= x_gt + # ): + # should_line = True + + # profiler('ds logic complete') + + # if should_line: + # # update the line graphic + # # x, y = self._ds_line_xy = ohlc_flatten(ohlc_iv) + # x, y = self._ds_line_xy = ohlc_flatten(ohlc) + # x_iv, y_iv = self._ds_line_xy = ohlc_flatten(ohlc_iv) + # profiler('flattening bars to line') + + # # TODO: we should be diffing the amount of new data which + # # needs to be downsampled. Ideally we actually are just + # # doing all the ds-ing in sibling actors so that the data + # # can just be read and rendered to graphics on events of our + # # choice. + # # diff = do_diff(ohlc, new_bit) + # curve = self._ds_line + # curve.update_from_array( + # x=x, + # y=y, + # x_iv=x_iv, + # y_iv=y_iv, + # view_range=None, # hack + # profiler=profiler, + # ) + # profiler('updated ds line') + + # if not self._in_ds: + # # hide bars and show line + # self.hide() + # # XXX: is this actually any faster? + # # self._pi.removeItem(self) + + # # TODO: a `.ui()` log level? + # log.info( + # f'downsampling to line graphic {self._name}' + # ) + + # # self._pi.addItem(curve) + # curve.show() + # curve.update() + # self._in_ds = True + + # # stop here since we don't need to update bars path any more + # # as we delegate to the downsample line with updates. + + # else: + # # we should be in bars mode + + # if self._in_ds: + # # flip back to bars graphics and hide the downsample line. + # log.info(f'showing bars graphic {self._name}') + + # curve = self._ds_line + # curve.hide() + # # self._pi.removeItem(curve) + + # # XXX: is this actually any faster? + # # self._pi.addItem(self) + # self.show() + # self._in_ds = False + + # # generate in_view path + # self.path = gen_qpath( + # ohlc_iv, + # 0, + # self.w, + # # path=self.path, + # ) + + # # TODO: to make the downsampling faster + # # - allow mapping only a range of lines thus only drawing as + # # many bars as exactly specified. + # # - move ohlc "flattening" to a shmarr + # # - maybe move all this embedded logic to a higher + # # level type? + + # # if prepend_length: + # # # new history was added and we need to render a new path + # # prepend_bars = ohlc[:prepend_length] + + # # if ds_prepend_length: + # # ds_prepend_bars = ohlc[:ds_prepend_length] + # # pre_x, pre_y = ohlc_flatten(ds_prepend_bars) + # # fx = np.concatenate((pre_x, fx)) + # # fy = np.concatenate((pre_y, fy)) + # # profiler('ds line prepend diff complete') + + # # if append_length: + # # # generate new graphics to match provided array + # # # path appending logic: + # # # we need to get the previous "current bar(s)" for the time step + # # # and convert it to a sub-path to append to the historical set + # # # new_bars = ohlc[istop - 1:istop + append_length - 1] + # # append_bars = ohlc[-append_length - 1:-1] + # # # print(f'ohlc bars to append size: {append_bars.size}\n') + + # # if ds_append_length: + # # ds_append_bars = ohlc[-ds_append_length - 1:-1] + # # post_x, post_y = ohlc_flatten(ds_append_bars) + # # print( + # # f'ds curve to append sizes: {(post_x.size, post_y.size)}' + # # ) + # # fx = np.concatenate((fx, post_x)) + # # fy = np.concatenate((fy, post_y)) + + # # profiler('ds line append diff complete') + + # profiler('array diffs complete') + + # # does this work? + # last = ohlc[-1] + # # fy[-1] = last['close'] + + # # # incremental update and cache line datums + # # self._ds_line_xy = fx, fy + + # # maybe downsample to line + # # ds = self.maybe_downsample() + # # if ds: + # # # if we downsample to a line don't bother with + # # # any more path generation / updates + # # self._ds_xrange = first_index, last_index + # # profiler('downsampled to line') + # # return + + # # print(in_view.size) + + # # if self.path: + # # self.path = path + # # self.path.reserve(path.capacity()) + # # self.path.swap(path) + + # # path updates + # # if prepend_length: + # # # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path + # # # y value not matching the first value from + # # # ohlc[prepend_length + 1] ??? + # # prepend_path = gen_qpath(prepend_bars, 0, self.w) + # # old_path = self.path + # # self.path = prepend_path + # # self.path.addPath(old_path) + # # profiler('path PREPEND') + + # # if append_length: + # # append_path = gen_qpath(append_bars, 0, self.w) + + # # self.path.moveTo( + # # float(istop - self.w), + # # float(append_bars[0]['open']) + # # ) + # # self.path.addPath(append_path) + + # # profiler('path APPEND') + # # fp = self.fast_path + # # if fp is None: + # # self.fast_path = append_path + + # # else: + # # fp.moveTo( + # # float(istop - self.w), float(new_bars[0]['open']) + # # ) + # # fp.addPath(append_path) + + # # self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) + # # flip_cache = True + + # self._xrange = first_index, last_index + + # # trigger redraw despite caching + # self.prepareGeometryChange() + + # self.draw_last(last) + + # # # generate new lines objects for updatable "current bar" + # # self._last_bar_lines = bar_from_ohlc_row(last, self.w) + + # # # last bar update + # # i, o, h, l, last, v = last[ + # # ['index', 'open', 'high', 'low', 'close', 'volume'] + # # ] + # # # assert i == self.start_index - 1 + # # # assert i == last_index + # # body, larm, rarm = self._last_bar_lines + + # # # XXX: is there a faster way to modify this? + # # rarm.setLine(rarm.x1(), last, rarm.x2(), last) + + # # # writer is responsible for changing open on "first" volume of bar + # # larm.setLine(larm.x1(), o, larm.x2(), o) + + # # if l != h: # noqa + + # # if body is None: + # # body = self._last_bar_lines[0] = QLineF(i, l, i, h) + # # else: + # # # update body + # # body.setLine(i, l, i, h) + + # # # XXX: pretty sure this is causing an issue where the bar has + # # # a large upward move right before the next sample and the body + # # # is getting set to None since the next bar is flat but the shm + # # # array index update wasn't read by the time this code runs. Iow + # # # we're doing this removal of the body for a bar index that is + # # # now out of date / from some previous sample. It's weird + # # # though because i've seen it do this to bars i - 3 back? + + # profiler('last bar set') + + # self.update() + # profiler('.update()') + + # if flip_cache: + # self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + + # # profiler.finish() + + def draw_last( self, - - # full array input history - ohlc: np.ndarray, - - # pre-sliced array data that's "in view" - ohlc_iv: np.ndarray, - - view_range: Optional[tuple[int, int]] = None, - profiler: Optional[pg.debug.Profiler] = None, + last: np.ndarray, ) -> None: - ''' - Update the last datum's bar graphic from input data array. + # generate new lines objects for updatable "current bar" + self._last_bar_lines = bar_from_ohlc_row(last, self.w) - This routine should be interface compatible with - ``pg.PlotCurveItem.setData()``. Normally this method in - ``pyqtgraph`` seems to update all the data passed to the - graphics object, and then update/rerender, but here we're - assuming the prior graphics havent changed (OHLC history rarely - does) so this "should" be simpler and faster. + # last bar update + i, o, h, l, last, v = last[ + ['index', 'open', 'high', 'low', 'close', 'volume'] + ] + # assert i == self.start_index - 1 + # assert i == last_index + body, larm, rarm = self._last_bar_lines - This routine should be made (transitively) as fast as possible. + # XXX: is there a faster way to modify this? + rarm.setLine(rarm.x1(), last, rarm.x2(), last) - ''' - profiler = profiler or pg.debug.Profiler( - disabled=not pg_profile_enabled(), - gt=ms_slower_then, - delayed=True, - ) + # writer is responsible for changing open on "first" volume of bar + larm.setLine(larm.x1(), o, larm.x2(), o) - # index = self.start_index - istart, istop = self._xrange - # ds_istart, ds_istop = self._ds_xrange + if l != h: # noqa - index = ohlc['index'] - first_index, last_index = index[0], index[-1] + if body is None: + body = self._last_bar_lines[0] = QLineF(i, l, i, h) + else: + # update body + body.setLine(i, l, i, h) - # length = len(ohlc) - # prepend_length = istart - first_index - # append_length = last_index - istop - - # ds_prepend_length = ds_istart - first_index - # ds_append_length = last_index - ds_istop - - flip_cache = False - - x_gt = 16 - if self._ds_line: - uppx = self._ds_line.x_uppx() - else: - uppx = 0 - - should_line = self._in_ds - if ( - self._in_ds - and uppx < x_gt - ): - should_line = False - - elif ( - not self._in_ds - and uppx >= x_gt - ): - should_line = True - - profiler('ds logic complete') - - if should_line: - # update the line graphic - # x, y = self._ds_line_xy = ohlc_flatten(ohlc_iv) - x, y = self._ds_line_xy = ohlc_flatten(ohlc) - x_iv, y_iv = self._ds_line_xy = ohlc_flatten(ohlc_iv) - profiler('flattening bars to line') - - # TODO: we should be diffing the amount of new data which - # needs to be downsampled. Ideally we actually are just - # doing all the ds-ing in sibling actors so that the data - # can just be read and rendered to graphics on events of our - # choice. - # diff = do_diff(ohlc, new_bit) - curve = self._ds_line - curve.update_from_array( - x=x, - y=y, - x_iv=x_iv, - y_iv=y_iv, - view_range=None, # hack - profiler=profiler, - ) - profiler('updated ds line') - - if not self._in_ds: - # hide bars and show line - self.hide() - # XXX: is this actually any faster? - # self._pi.removeItem(self) - - # TODO: a `.ui()` log level? - log.info( - f'downsampling to line graphic {self._name}' - ) - - # self._pi.addItem(curve) - curve.show() - curve.update() - self._in_ds = True - - # stop here since we don't need to update bars path any more - # as we delegate to the downsample line with updates. - - else: - # we should be in bars mode - - if self._in_ds: - # flip back to bars graphics and hide the downsample line. - log.info(f'showing bars graphic {self._name}') - - curve = self._ds_line - curve.hide() - # self._pi.removeItem(curve) - - # XXX: is this actually any faster? - # self._pi.addItem(self) - self.show() - self._in_ds = False - - # generate in_view path - self.path = gen_qpath( - ohlc_iv, - 0, - self.w, - # path=self.path, - ) - - # TODO: to make the downsampling faster - # - allow mapping only a range of lines thus only drawing as - # many bars as exactly specified. - # - move ohlc "flattening" to a shmarr - # - maybe move all this embedded logic to a higher - # level type? - - # if prepend_length: - # # new history was added and we need to render a new path - # prepend_bars = ohlc[:prepend_length] - - # if ds_prepend_length: - # ds_prepend_bars = ohlc[:ds_prepend_length] - # pre_x, pre_y = ohlc_flatten(ds_prepend_bars) - # fx = np.concatenate((pre_x, fx)) - # fy = np.concatenate((pre_y, fy)) - # profiler('ds line prepend diff complete') - - # if append_length: - # # generate new graphics to match provided array - # # path appending logic: - # # we need to get the previous "current bar(s)" for the time step - # # and convert it to a sub-path to append to the historical set - # # new_bars = ohlc[istop - 1:istop + append_length - 1] - # append_bars = ohlc[-append_length - 1:-1] - # # print(f'ohlc bars to append size: {append_bars.size}\n') - - # if ds_append_length: - # ds_append_bars = ohlc[-ds_append_length - 1:-1] - # post_x, post_y = ohlc_flatten(ds_append_bars) - # print( - # f'ds curve to append sizes: {(post_x.size, post_y.size)}' - # ) - # fx = np.concatenate((fx, post_x)) - # fy = np.concatenate((fy, post_y)) - - # profiler('ds line append diff complete') - - profiler('array diffs complete') - - # does this work? - last = ohlc[-1] - # fy[-1] = last['close'] - - # # incremental update and cache line datums - # self._ds_line_xy = fx, fy - - # maybe downsample to line - # ds = self.maybe_downsample() - # if ds: - # # if we downsample to a line don't bother with - # # any more path generation / updates - # self._ds_xrange = first_index, last_index - # profiler('downsampled to line') - # return - - # print(in_view.size) - - # if self.path: - # self.path = path - # self.path.reserve(path.capacity()) - # self.path.swap(path) - - # path updates - # if prepend_length: - # # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path - # # y value not matching the first value from - # # ohlc[prepend_length + 1] ??? - # prepend_path = gen_qpath(prepend_bars, 0, self.w) - # old_path = self.path - # self.path = prepend_path - # self.path.addPath(old_path) - # profiler('path PREPEND') - - # if append_length: - # append_path = gen_qpath(append_bars, 0, self.w) - - # self.path.moveTo( - # float(istop - self.w), - # float(append_bars[0]['open']) - # ) - # self.path.addPath(append_path) - - # profiler('path APPEND') - # fp = self.fast_path - # if fp is None: - # self.fast_path = append_path - - # else: - # fp.moveTo( - # float(istop - self.w), float(new_bars[0]['open']) - # ) - # fp.addPath(append_path) - - # self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # flip_cache = True - - self._xrange = first_index, last_index - - # trigger redraw despite caching - self.prepareGeometryChange() - - # generate new lines objects for updatable "current bar" - self._last_bar_lines = bar_from_ohlc_row(last, self.w) - - # last bar update - i, o, h, l, last, v = last[ - ['index', 'open', 'high', 'low', 'close', 'volume'] - ] - # assert i == self.start_index - 1 - # assert i == last_index - body, larm, rarm = self._last_bar_lines - - # XXX: is there a faster way to modify this? - rarm.setLine(rarm.x1(), last, rarm.x2(), last) - - # writer is responsible for changing open on "first" volume of bar - larm.setLine(larm.x1(), o, larm.x2(), o) - - if l != h: # noqa - - if body is None: - body = self._last_bar_lines[0] = QLineF(i, l, i, h) - else: - # update body - body.setLine(i, l, i, h) - - # XXX: pretty sure this is causing an issue where the bar has - # a large upward move right before the next sample and the body - # is getting set to None since the next bar is flat but the shm - # array index update wasn't read by the time this code runs. Iow - # we're doing this removal of the body for a bar index that is - # now out of date / from some previous sample. It's weird - # though because i've seen it do this to bars i - 3 back? - - profiler('last bar set') - - self.update() - profiler('.update()') - - if flip_cache: - self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - # profiler.finish() + # XXX: pretty sure this is causing an issue where the bar has + # a large upward move right before the next sample and the body + # is getting set to None since the next bar is flat but the shm + # array index update wasn't read by the time this code runs. Iow + # we're doing this removal of the body for a bar index that is + # now out of date / from some previous sample. It's weird + # though because i've seen it do this to bars i - 3 back? def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect From 427a33654b4e308fb7fafc6ef8e0cb489809b04f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 20 Apr 2022 11:43:47 -0400 Subject: [PATCH 007/113] More WIP, implement `BarItems` rendering in `Flow.update_graphics()` --- piker/ui/_flows.py | 400 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 312 insertions(+), 88 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 772aa026..8d5e7e77 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -23,6 +23,8 @@ incremental update. ''' from __future__ import annotations +from functools import partial +import time from typing import ( Optional, Callable, @@ -30,6 +32,7 @@ from typing import ( import msgspec import numpy as np +from numpy.lib import recfunctions as rfn import pyqtgraph as pg from PyQt5.QtGui import QPainterPath @@ -37,6 +40,7 @@ from ..data._sharedmem import ( ShmArray, # attach_shm_array ) +from .._profile import pg_profile_enabled, ms_slower_then from ._ohlc import ( BarItems, gen_qpath, @@ -46,7 +50,12 @@ from ._curve import ( ) from ._compression import ( ohlc_flatten, + ds_m4, ) +from ..log import get_logger + + +log = get_logger(__name__) # class FlowsTable(msgspec.Struct): # ''' @@ -72,11 +81,63 @@ from ._compression import ( # return cls(shm) +def rowarr_to_path( + rows_array: np.ndarray, + x_basis: np.ndarray, + flow: Flow, + +) -> QPainterPath: + + # TODO: we could in theory use ``numba`` to flatten + # if needed? + + # to 1d + y = rows_array.flatten() + + return pg.functions.arrayToQPath( + # these get passed at render call time + x=x_basis[:y.size], + y=y, + connect='all', + finiteCheck=False, + path=flow.path, + ) + + +def ohlc_flat_view( + ohlc_shm: ShmArray, + + # XXX: we bind this in currently.. + x_basis: np.ndarray, + + # vr: Optional[slice] = None, + +) -> np.ndarray: + ''' + Return flattened-non-copy view into an OHLC shm array. + + ''' + ohlc = ohlc_shm._array[['open', 'high', 'low', 'close']] + # if vr: + # ohlc = ohlc[vr] + # x = x_basis[vr] + + unstructured = rfn.structured_to_unstructured( + ohlc, + copy=False, + ) + # breakpoint() + y = unstructured.flatten() + x = x_basis[:y.size] + return x, y + + class Flow(msgspec.Struct): # , frozen=True): ''' - (FinancialSignal-)Flow compound type which wraps a real-time - graphics (curve) and its backing data stream together for high level - access and control. + (Financial Signal-)Flow compound type which wraps a real-time + shm array stream with displayed graphics (curves, charts) + for high level access and control as well as efficient incremental + update. The intention is for this type to eventually be capable of shm-passing of incrementally updated graphics stream data between actors. @@ -89,6 +150,8 @@ class Flow(msgspec.Struct): # , frozen=True): is_ohlc: bool = False render: bool = True # toggle for display loop + flat: Optional[ShmArray] = None + x_basis: Optional[np.ndarray] = None _last_uppx: float = 0 _in_ds: bool = False @@ -96,6 +159,7 @@ class Flow(msgspec.Struct): # , frozen=True): _graphics_tranform_fn: Optional[Callable[ShmArray, np.ndarray]] = None # map from uppx -> (downsampled data, incremental graphics) + _src_r: Optional[Renderer] = None _render_table: dict[ Optional[int], tuple[Renderer, pg.GraphicsItem], @@ -215,7 +279,9 @@ class Flow(msgspec.Struct): # , frozen=True): int, int, np.ndarray, int, int, np.ndarray, ]: + # read call array = self.shm.array + indexes = array['index'] ifirst = indexes[0] ilast = indexes[-1] @@ -245,6 +311,8 @@ class Flow(msgspec.Struct): # , frozen=True): render: bool = True, array_key: Optional[str] = None, + profiler=None, + **kwargs, ) -> pg.GraphicsObject: @@ -253,8 +321,19 @@ class Flow(msgspec.Struct): # , frozen=True): render to graphics. ''' + + profiler = profiler or pg.debug.Profiler( + msg=f'Flow.update_graphics() for {self.name}', + disabled=not pg_profile_enabled(), + gt=ms_slower_then, + delayed=True, + ) # shm read and slice to view - read = xfirst, xlast, array, ivl, ivr, in_view = self.read() + read = ( + xfirst, xlast, array, + ivl, ivr, in_view, + ) = self.read() + profiler('read src shm data') if ( not in_view.size @@ -265,100 +344,182 @@ class Flow(msgspec.Struct): # , frozen=True): graphics = self.graphics if isinstance(graphics, BarItems): - # ugh, not luvin dis, should we have just a designated - # instance var? - r = self._render_table.get('src') + # if no source data renderer exists create one. + r = self._src_r if not r: - r = Renderer( + # OHLC bars path renderer + r = self._src_r = Renderer( flow=self, - draw=gen_qpath, # TODO: rename this to something with ohlc + # TODO: rename this to something with ohlc + draw_path=gen_qpath, last_read=read, ) - self._render_table['src'] = (r, graphics) + + # create a flattened view onto the OHLC array + # which can be read as a line-style format + # shm = self.shm + # self.flat = shm.unstruct_view(['open', 'high', 'low', 'close']) + # import pdbpp + # pdbpp.set_trace() + # x = self.x_basis = ( + # np.broadcast_to( + # shm._array['index'][:, None], + # # self.flat._array.shape, + # self.flat.shape, + # ) + np.array([-0.5, 0, 0, 0.5]) + # ) ds_curve_r = Renderer( flow=self, - draw=gen_qpath, # TODO: rename this to something with ohlc + + # just swap in the flat view + data_t=lambda array: self.flat.array, + # data_t=partial( + # ohlc_flat_view, + # self.shm, + # ), last_read=read, - prerender_fn=ohlc_flatten, + draw_path=partial( + rowarr_to_path, + x_basis=None, + ), + ) + curve = FastAppendCurve( + # y=y, + # x=x, + name='OHLC', + color=graphics._color, + ) + curve.hide() + self.plot.addItem(curve) # baseline "line" downsampled OHLC curve that should # kick on only when we reach a certain uppx threshold. self._render_table[0] = ( ds_curve_r, - FastAppendCurve( - y=y, - x=x, - name='OHLC', - color=self._color, - ), + curve, ) + dsc_r, curve = self._render_table[0] + # do checks for whether or not we require downsampling: # - if we're **not** downsampling then we simply want to # render the bars graphics curve and update.. # - if insteam we are in a downsamplig state then we to + x_gt = 8 + uppx = curve.x_uppx() + in_line = should_line = curve.isVisible() + if ( + should_line + and uppx < x_gt + ): + should_line = False + + elif ( + not should_line + and uppx >= x_gt + ): + should_line = True + + profiler(f'ds logic complete line={should_line}') + + # do graphics updates + if should_line: + # start = time.time() + # y = self.shm.unstruct_view( + # ['open', 'high', 'low', 'close'], + # ) + # print(f'unstruct diff: {time.time() - start}') + # profiler('read unstr view bars to line') + # # start = self.flat._first.value + + # x = self.x_basis[:y.size].flatten() + # y = y.flatten() + # profiler('flattening bars to line') + # path, last = dsc_r.render(read) + # x, flat = ohlc_flat_view( + # ohlc_shm=self.shm, + # x_basis=x_basis, + # ) + # y = y.flatten() + # y_iv = y[ivl:ivr].flatten() + # x_iv = x[ivl:ivr].flatten() + # assert y.size == x.size + + x, y = self.flat = ohlc_flatten(array) + x_iv, y_iv = ohlc_flatten(in_view) + profiler('flattened OHLC data') + + curve.update_from_array( + x, + y, + x_iv=x_iv, + y_iv=y_iv, + view_range=None, # hack + profiler=profiler, + ) + profiler('updated ds curve') + + else: + # render incremental or in-view update + # and apply ouput (path) to graphics. + path, last = r.render( + read, + only_in_view=True, + ) + + graphics.path = path + graphics.draw_last(last) + + # NOTE: on appends we used to have to flip the coords + # cache thought it doesn't seem to be required any more? + # graphics.setCacheMode(QtWidgets.QGraphicsItem.NoCache) + # graphics.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + + graphics.prepareGeometryChange() + graphics.update() + + if ( + not in_line + and should_line + ): + # change to line graphic + + log.info( + f'downsampling to line graphic {self.name}' + ) + graphics.hide() + # graphics.update() + curve.show() + curve.update() + + elif in_line and not should_line: + log.info(f'showing bars graphic {self.name}') + curve.hide() + graphics.show() + graphics.update() + # update our pre-downsample-ready data and then pass that # new data the downsampler algo for incremental update. - else: - pass - # do incremental update - graphics.update_from_array( - array, - in_view, - view_range=(ivl, ivr) if use_vr else None, + # graphics.update_from_array( + # array, + # in_view, + # view_range=(ivl, ivr) if use_vr else None, - **kwargs, - ) + # **kwargs, + # ) - # generate and apply path to graphics obj - graphics.path, last = r.render(only_in_view=True) - graphics.draw_last(last) + # generate and apply path to graphics obj + # graphics.path, last = r.render( + # read, + # only_in_view=True, + # ) + # graphics.draw_last(last) else: - # should_ds = False - # should_redraw = False - - # # downsampling incremental state checking - # uppx = bars.x_uppx() - # px_width = bars.px_width() - # uppx_diff = (uppx - self._last_uppx) - - # if self.renderer is None: - # self.renderer = Renderer( - # flow=self, - - # if not self._in_ds: - # # in not currently marked as downsampling graphics - # # then only draw the full bars graphic for datums "in - # # view". - - # # check for downsampling conditions - # if ( - # # std m4 downsample conditions - # px_width - # and uppx_diff >= 4 - # or uppx_diff <= -3 - # or self._step_mode and abs(uppx_diff) >= 4 - - # ): - # log.info( - # f'{self._name} sampler change: {self._last_uppx} -> {uppx}' - # ) - # self._last_uppx = uppx - # should_ds = True - - # elif ( - # uppx <= 2 - # and self._in_ds - # ): - # # we should de-downsample back to our original - # # source data so we clear our path data in prep - # # to generate a new one from original source data. - # should_redraw = True - # should_ds = False + # ``FastAppendCurve`` case: array_key = array_key or self.name @@ -376,23 +537,64 @@ class Flow(msgspec.Struct): # , frozen=True): return graphics +def xy_downsample( + x, + y, + px_width, + uppx, + + x_spacer: float = 0.5, + +) -> tuple[np.ndarray, np.ndarray]: + + # downsample whenever more then 1 pixels per datum can be shown. + # always refresh data bounds until we get diffing + # working properly, see above.. + bins, x, y = ds_m4( + x, + y, + px_width=px_width, + uppx=uppx, + log_scale=bool(uppx) + ) + + # flatten output to 1d arrays suitable for path-graphics generation. + x = np.broadcast_to(x[:, None], y.shape) + x = (x + np.array( + [-x_spacer, 0, 0, x_spacer] + )).flatten() + y = y.flatten() + + return x, y + + class Renderer(msgspec.Struct): flow: Flow # called to render path graphics - draw: Callable[np.ndarray, QPainterPath] + draw_path: Callable[np.ndarray, QPainterPath] - # called on input data but before - prerender_fn: Optional[Callable[ShmArray, np.ndarray]] = None + # called on input data but before any graphics format + # conversions or processing. + data_t: Optional[Callable[ShmArray, np.ndarray]] = None + data_t_shm: Optional[ShmArray] = None + # called on the final data (transform) output to convert + # to "graphical data form" a format that can be passed to + # the ``.draw()`` implementation. + graphics_t: Optional[Callable[ShmArray, np.ndarray]] = None + graphics_t_shm: Optional[ShmArray] = None + + # path graphics update implementation methods prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None # last array view read last_read: Optional[np.ndarray] = None - # output graphics rendering + # output graphics rendering, the main object + # processed in ``QGraphicsObject.paint()`` path: Optional[QPainterPath] = None # def diff( @@ -411,8 +613,10 @@ class Renderer(msgspec.Struct): def render( self, + new_read, + # only render datums "in view" of the ``ChartView`` - only_in_view: bool = True, + only_in_view: bool = False, ) -> list[QPainterPath]: ''' @@ -428,28 +632,42 @@ class Renderer(msgspec.Struct): ''' # do full source data render to path - xfirst, xlast, array, ivl, ivr, in_view = self.last_read + last_read = ( + xfirst, xlast, array, + ivl, ivr, in_view, + ) = self.last_read if only_in_view: - # get latest data from flow shm - self.last_read = ( - xfirst, xlast, array, ivl, ivr, in_view - ) = self.flow.read() - array = in_view + # # get latest data from flow shm + # self.last_read = ( + # xfirst, xlast, array, ivl, ivr, in_view + # ) = new_read - if self.path is None or in_view: + if self.path is None or only_in_view: # redraw the entire source data if we have either of: # - no prior path graphic rendered or, # - we always intend to re-render the data only in view - if self.prerender_fn: - array = self.prerender_fn(array) + # data transform: convert source data to a format + # expected to be incrementally updates and later rendered + # to a more graphics native format. + if self.data_t: + array = self.data_t(array) - hist, last = array[:-1], array[-1] + # maybe allocate shm for data transform output + # if self.data_t_shm is None: + # fshm = self.flow.shm - # call path render func on history - self.path = self.draw(hist) + # shm, opened = maybe_open_shm_array( + # f'{self.flow.name}_data_t', + # # TODO: create entry for each time frame + # dtype=array.dtype, + # readonly=False, + # ) + # assert opened + # shm.push(array) + # self.data_t_shm = shm elif self.path: print(f'inremental update not supported yet {self.flow.name}') @@ -459,4 +677,10 @@ class Renderer(msgspec.Struct): # do path generation for each segment # and then push into graphics object. + hist, last = array[:-1], array[-1] + + # call path render func on history + self.path = self.draw_path(hist) + + self.last_read = new_read return self.path, last From 239c9d701a5e6d3441ea02624001b3579b1eaac2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 20 Apr 2022 11:44:27 -0400 Subject: [PATCH 008/113] Don't require data input to constructor --- piker/ui/_curve.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 871e55f5..c1c525de 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -44,6 +44,7 @@ from ..log import get_logger log = get_logger(__name__) +# TODO: numba this instead.. def step_path_arrays_from_1d( x: np.ndarray, y: np.ndarray, @@ -119,8 +120,8 @@ class FastAppendCurve(pg.GraphicsObject): def __init__( self, - x: np.ndarray, - y: np.ndarray, + x: np.ndarray = None, + y: np.ndarray = None, *args, step_mode: bool = False, @@ -461,6 +462,7 @@ class FastAppendCurve(pg.GraphicsObject): ): new_x = x[-append_length - 2:-1] new_y = y[-append_length - 2:-1] + profiler('sliced append path') if self._step_mode: new_x, new_y = step_path_arrays_from_1d( @@ -474,7 +476,12 @@ class FastAppendCurve(pg.GraphicsObject): new_x = new_x[1:] new_y = new_y[1:] - profiler('diffed append arrays') + profiler('generated step data') + + else: + profiler( + f'diffed array input, append_length={append_length}' + ) if should_ds: new_x, new_y = self.downsample( @@ -491,6 +498,7 @@ class FastAppendCurve(pg.GraphicsObject): finiteCheck=False, path=self.fast_path, ) + profiler(f'generated append qpath') if self.use_fpath: # an attempt at trying to make append-updates faster.. From 3dbce6f891c81cc868f7f77de0e350fa966696a5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 20 Apr 2022 12:13:18 -0400 Subject: [PATCH 009/113] Add `FastAppendCurve.draw_last()` --- piker/ui/_curve.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index c1c525de..47132d7c 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -337,9 +337,6 @@ class FastAppendCurve(pg.GraphicsObject): else: self._xrange = x[0], x[-1] - x_last = x[-1] - y_last = y[-1] - # check for downsampling conditions if ( # std m4 downsample conditions @@ -498,7 +495,7 @@ class FastAppendCurve(pg.GraphicsObject): finiteCheck=False, path=self.fast_path, ) - profiler(f'generated append qpath') + profiler('generated append qpath') if self.use_fpath: # an attempt at trying to make append-updates faster.. @@ -537,6 +534,28 @@ class FastAppendCurve(pg.GraphicsObject): # self.disable_cache() # flip_cache = True + self.draw_last(x, y) + profiler('draw last segment') + + # trigger redraw of path + # do update before reverting to cache mode + # self.prepareGeometryChange() + self.update() + profiler('.update()') + + # if flip_cache: + # # XXX: seems to be needed to avoid artifacts (see above). + # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + + def draw_last( + self, + x: np.ndarray, + y: np.ndarray, + + ) -> None: + x_last = x[-1] + y_last = y[-1] + # draw the "current" step graphic segment so it lines up with # the "middle" of the current (OHLC) sample. if self._step_mode: @@ -556,21 +575,9 @@ class FastAppendCurve(pg.GraphicsObject): else: self._last_line = QLineF( x[-2], y[-2], - x[-1], y_last + x_last, y_last ) - profiler('draw last segment') - - # trigger redraw of path - # do update before reverting to cache mode - # self.prepareGeometryChange() - self.update() - profiler('.update()') - - # if flip_cache: - # # XXX: seems to be needed to avoid artifacts (see above). - # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. def getData(self): From 7e1ec7b5a7de679396e2747f72f271aca5c5d3c8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 10 May 2022 17:57:14 -0400 Subject: [PATCH 010/113] Incrementally update flattend OHLC data After much effort (and exhaustion) but failure to get a view into our `numpy` OHLC struct-array, this instead allocates an in-thread-memory array which is updated with flattened data every flow update cycle. I need to report what I think is a bug to `numpy` core about the whole view thing not working but, more or less this gets the same behaviour and minimizes work to flatten the sampled data for line-graphics drawing thus improving refresh latency when drawing large downsampled curves. Update the OHLC ds curve with view aware data sliced out from the pre-allocated and incrementally updated data (we had to add a last index var `._iflat` to track appends - this should be moved into a renderer eventually?). --- piker/ui/_flows.py | 101 ++++++++++++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 37 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 8d5e7e77..20ffc510 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -24,7 +24,6 @@ incremental update. ''' from __future__ import annotations from functools import partial -import time from typing import ( Optional, Callable, @@ -38,7 +37,7 @@ from PyQt5.QtGui import QPainterPath from ..data._sharedmem import ( ShmArray, - # attach_shm_array + open_shm_array, ) from .._profile import pg_profile_enabled, ms_slower_then from ._ohlc import ( @@ -152,6 +151,7 @@ class Flow(msgspec.Struct): # , frozen=True): render: bool = True # toggle for display loop flat: Optional[ShmArray] = None x_basis: Optional[np.ndarray] = None + _iflat: int = 0 _last_uppx: float = 0 _in_ds: bool = False @@ -344,6 +344,7 @@ class Flow(msgspec.Struct): # , frozen=True): graphics = self.graphics if isinstance(graphics, BarItems): + fields = ['open', 'high', 'low', 'close'] # if no source data renderer exists create one. r = self._src_r if not r: @@ -357,17 +358,37 @@ class Flow(msgspec.Struct): # , frozen=True): # create a flattened view onto the OHLC array # which can be read as a line-style format - # shm = self.shm - # self.flat = shm.unstruct_view(['open', 'high', 'low', 'close']) + shm = self.shm + + # flat = self.flat = self.shm.unstruct_view(fields) + self.flat = self.shm.ustruct(fields) + self._iflat = self.shm._last.value + # import pdbpp # pdbpp.set_trace() - # x = self.x_basis = ( - # np.broadcast_to( - # shm._array['index'][:, None], - # # self.flat._array.shape, - # self.flat.shape, - # ) + np.array([-0.5, 0, 0, 0.5]) + # assert len(flat._array) == len(self.shm._array[fields]) + + x = self.x_basis = ( + np.broadcast_to( + shm._array['index'][:, None], + ( + shm._array.size, + # 4, # only ohlc + self.flat.shape[1], + ), + ) + np.array([-0.5, 0, 0, 0.5]) + ) + + # fshm = self.flat = open_shm_array( + # f'{self.name}_flat', + # dtype=flattened.dtype, + # size=flattened.size, # ) + # fshm.push(flattened) + + # print(f'unstruct diff: {time.time() - start}') + # profiler('read unstr view bars to line') + # start = self.flat._first.value ds_curve_r = Renderer( flow=self, @@ -407,7 +428,7 @@ class Flow(msgspec.Struct): # , frozen=True): # - if we're **not** downsampling then we simply want to # render the bars graphics curve and update.. # - if insteam we are in a downsamplig state then we to - x_gt = 8 + x_gt = 6 uppx = curve.x_uppx() in_line = should_line = curve.isVisible() if ( @@ -426,37 +447,43 @@ class Flow(msgspec.Struct): # , frozen=True): # do graphics updates if should_line: - # start = time.time() - # y = self.shm.unstruct_view( - # ['open', 'high', 'low', 'close'], - # ) - # print(f'unstruct diff: {time.time() - start}') - # profiler('read unstr view bars to line') - # # start = self.flat._first.value - # x = self.x_basis[:y.size].flatten() - # y = y.flatten() - # profiler('flattening bars to line') - # path, last = dsc_r.render(read) - # x, flat = ohlc_flat_view( - # ohlc_shm=self.shm, - # x_basis=x_basis, - # ) - # y = y.flatten() - # y_iv = y[ivl:ivr].flatten() - # x_iv = x[ivl:ivr].flatten() - # assert y.size == x.size + # update flatted ohlc copy + iflat, ishm = self._iflat, self.shm._last.value + to_update = rfn.structured_to_unstructured( + self.shm._array[iflat:ishm][fields] + ) - x, y = self.flat = ohlc_flatten(array) - x_iv, y_iv = ohlc_flatten(in_view) - profiler('flattened OHLC data') + # print(to_update) + self.flat[iflat:ishm][:] = to_update + profiler('updated ustruct OHLC data') + + y_flat = self.flat[:ishm] + x_flat = self.x_basis[:ishm] + + self._iflat = ishm + + y = y_flat.reshape(-1) + x = x_flat.reshape(-1) + profiler('flattened ustruct OHLC data') + + y_iv_flat = y_flat[ivl:ivr] + x_iv_flat = x_flat[ivl:ivr] + + y_iv = y_iv_flat.reshape(-1) + x_iv = x_iv_flat.reshape(-1) + profiler('flattened ustruct in-view OHLC data') + + # x, y = ohlc_flatten(array) + # x_iv, y_iv = ohlc_flatten(in_view) + # profiler('flattened OHLC data') curve.update_from_array( x, y, x_iv=x_iv, - y_iv=y_iv, - view_range=None, # hack + y_iv=y_iv, + view_range=(ivl, ivr), # hack profiler=profiler, ) profiler('updated ds curve') @@ -477,7 +504,7 @@ class Flow(msgspec.Struct): # , frozen=True): # graphics.setCacheMode(QtWidgets.QGraphicsItem.NoCache) # graphics.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - graphics.prepareGeometryChange() + # graphics.prepareGeometryChange() graphics.update() if ( @@ -632,7 +659,7 @@ class Renderer(msgspec.Struct): ''' # do full source data render to path - last_read = ( + ( xfirst, xlast, array, ivl, ivr, in_view, ) = self.last_read From df78e9ba964b319dd0f367c19cf560528074c142 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 21 Apr 2022 15:15:00 -0400 Subject: [PATCH 011/113] Delegate graphics cycle max/min to chart/flows --- piker/ui/_display.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index fda3fb04..0e03395e 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -96,28 +96,17 @@ def chart_maxmin( Compute max and min datums "in view" for range limits. ''' - array = ohlcv_shm.array - ifirst = array[0]['index'] - last_bars_range = chart.bars_range() - l, lbar, rbar, r = last_bars_range - in_view = array[lbar - ifirst:rbar - ifirst + 1] + out = chart.maxmin() - if not in_view.size: - log.warning('Resetting chart to data') - chart.default_view() + if out is None: return (last_bars_range, 0, 0, 0) - mx, mn = ( - np.nanmax(in_view['high']), - np.nanmin(in_view['low'],) - ) + mn, mx = out mx_vlm_in_view = 0 if vlm_chart: - mx_vlm_in_view = np.max( - in_view['volume'] - ) + _, mx_vlm_in_view = vlm_chart.maxmin() return ( last_bars_range, From 2af4050e5e3615204cf42724faf46458df1bab95 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 21 Apr 2022 15:47:24 -0400 Subject: [PATCH 012/113] Remove `._set_yrange()` handler from x-range-change signal --- piker/ui/_interaction.py | 56 ++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 4b3bbb45..ca49a35d 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -492,7 +492,10 @@ class ChartView(ViewBox): log.debug("Max zoom bruh...") return - # if ev.delta() < 0 and vl >= len(chart._flows[chart.name].shm.array) + 666: + # if ( + # ev.delta() < 0 + # and vl >= len(chart._flows[chart.name].shm.array) + 666 + # ): # log.debug("Min zoom bruh...") # return @@ -748,6 +751,7 @@ class ChartView(ViewBox): ''' profiler = pg.debug.Profiler( + msg=f'`ChartView._set_yrange()`: `{self.name}`', disabled=not pg_profile_enabled(), gt=ms_slower_then, delayed=True, @@ -815,6 +819,7 @@ class ChartView(ViewBox): if yrange is None: log.warning(f'No yrange provided for {self.name}!?') + print(f"WTF NO YRANGE {self.name}") return ylow, yhigh = yrange @@ -851,11 +856,6 @@ class ChartView(ViewBox): if src_vb is None: src_vb = self - # such that when a linked chart changes its range - # this local view is also automatically changed and - # resized to data. - src_vb.sigXRangeChanged.connect(self._set_yrange) - # splitter(s) resizing src_vb.sigResized.connect(self._set_yrange) @@ -876,11 +876,6 @@ class ChartView(ViewBox): self, ) -> None: - # self._chart._static_yrange = 'axis' - - self.sigXRangeChanged.disconnect( - self._set_yrange, - ) self.sigResized.disconnect( self._set_yrange, ) @@ -911,18 +906,20 @@ class ChartView(ViewBox): def maybe_downsample_graphics(self): - profiler = pg.debug.Profiler( - disabled=not pg_profile_enabled(), - gt=3, - ) - uppx = self.x_uppx() if not ( # we probably want to drop this once we are "drawing in # view" for downsampled flows.. - uppx and uppx > 16 + uppx and uppx > 6 and self._ic is not None ): + profiler = pg.debug.Profiler( + msg=f'ChartView.maybe_downsample_graphics() for {self.name}', + disabled=not pg_profile_enabled(), + # delayed=True, + gt=3, + # gt=ms_slower_then, + ) # TODO: a faster single-loop-iterator way of doing this XD chart = self._chart @@ -931,7 +928,12 @@ class ChartView(ViewBox): for chart_name, chart in plots.items(): for name, flow in chart._flows.items(): - if not flow.render: + if ( + not flow.render + + # XXX: super important to be aware of this. + # or not flow.graphics.isVisible() + ): continue graphics = flow.graphics @@ -947,13 +949,17 @@ class ChartView(ViewBox): use_vr=use_vr, # gets passed down into graphics obj - profiler=profiler, + # profiler=profiler, ) profiler(f'range change updated {chart_name}:{name}') - else: - # don't bother updating since we're zoomed out bigly and - # in a pan-interaction, in which case we shouldn't be - # doing view-range based rendering (at least not yet). - # print(f'{uppx} exiting early!') - profiler(f'dowsampling skipped - not in uppx range {uppx} <= 16') + + profiler.finish() + # else: + # # don't bother updating since we're zoomed out bigly and + # # in a pan-interaction, in which case we shouldn't be + # # doing view-range based rendering (at least not yet). + # # print(f'{uppx} exiting early!') + # profiler(f'dowsampling skipped - not in uppx range {uppx} <= 16') + + # profiler.finish() From c94c53286b6e6261742d0c89b2fa31ede6a71314 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 22 Apr 2022 13:59:20 -0400 Subject: [PATCH 013/113] `FastAppendCurve`: Only render in-view data if possible More or less this improves update latency like mad. Only draw data in view and avoid full path regen as much as possible within a given (down)sampling setting. We now support append path updates with in-view data and the *SPECIAL CAVEAT* is that we avoid redrawing the whole curve **only when** we calc an `append_length <= 1` **even if the view range changed**. XXX: this should change in the future probably such that the caller graphics update code can pass a flag which says whether or not to do a full redraw based on it knowing where it's an interaction based view-range change or a flow update change which doesn't require a full path re-render. --- piker/ui/_curve.py | 114 +++++++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 45 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 47132d7c..8c3c329f 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -138,6 +138,7 @@ class FastAppendCurve(pg.GraphicsObject): # brutaaalll, see comments within.. self._y = self.yData = y self._x = self.xData = x + self._vr: Optional[tuple] = None self._name = name self.path: Optional[QtGui.QPainterPath] = None @@ -287,6 +288,17 @@ class FastAppendCurve(pg.GraphicsObject): istart, istop = self._xrange else: self._xrange = istart, istop = x[0], x[-1] + + # compute the length diffs between the first/last index entry in + # the input data and the last indexes we have on record from the + # last time we updated the curve index. + prepend_length = int(istart - x[0]) + append_length = int(x[-1] - istop) + + # this is the diff-mode, "data"-rendered index + # tracking var.. + self._xrange = x[0], x[-1] + # print(f"xrange: {self._xrange}") # XXX: lol brutal, the internals of `CurvePoint` (inherited by @@ -295,37 +307,36 @@ class FastAppendCurve(pg.GraphicsObject): # self.yData = y # self._x, self._y = x, y - if view_range: - profiler(f'view range slice {view_range}') - # downsampling incremental state checking uppx = self.x_uppx() px_width = self.px_width() uppx_diff = (uppx - self._last_uppx) + new_sample_rate = False should_ds = False + showing_src_data = self._in_ds should_redraw = False # if a view range is passed, plan to draw the # source ouput that's "in view" of the chart. - if view_range and not self._in_ds: + if ( + view_range + # and not self._in_ds + # and not prepend_length > 0 + ): # print(f'{self._name} vr: {view_range}') # by default we only pull data up to the last (current) index x_out, y_out = x_iv[:-1], y_iv[:-1] + profiler(f'view range slice {view_range}') - # step mode: draw flat top discrete "step" - # over the index space for each datum. - if self._step_mode: - # TODO: numba this bish - x_out, y_out = step_path_arrays_from_1d( - x_out, - y_out - ) - profiler('generated step arrays') + if ( + view_range != self._vr + and append_length > 1 + ): + should_redraw = True - should_redraw = True - profiler('sliced in-view array history') + self._vr = view_range # x_last = x_iv[-1] # y_last = y_iv[-1] @@ -335,7 +346,15 @@ class FastAppendCurve(pg.GraphicsObject): # flip_cache = True else: - self._xrange = x[0], x[-1] + # if ( + # not view_range + # or self._in_ds + # ): + # by default we only pull data up to the last (current) index + x_out, y_out = x[:-1], y[:-1] + + if prepend_length > 0: + should_redraw = True # check for downsampling conditions if ( @@ -350,6 +369,8 @@ class FastAppendCurve(pg.GraphicsObject): f'{self._name} sampler change: {self._last_uppx} -> {uppx}' ) self._last_uppx = uppx + new_sample_rate = True + showing_src_data = False should_ds = True elif ( @@ -360,49 +381,47 @@ class FastAppendCurve(pg.GraphicsObject): # source data so we clear our path data in prep # to generate a new one from original source data. should_redraw = True + new_sample_rate = True should_ds = False - - # compute the length diffs between the first/last index entry in - # the input data and the last indexes we have on record from the - # last time we updated the curve index. - prepend_length = int(istart - x[0]) - append_length = int(x[-1] - istop) + showing_src_data = True # no_path_yet = self.path is None if ( self.path is None or should_redraw - or should_ds + or new_sample_rate or prepend_length > 0 ): - if ( - not view_range - or self._in_ds - ): - # by default we only pull data up to the last (current) index - x_out, y_out = x[:-1], y[:-1] + # if ( + # not view_range + # or self._in_ds + # ): + # # by default we only pull data up to the last (current) index + # x_out, y_out = x[:-1], y[:-1] - # step mode: draw flat top discrete "step" - # over the index space for each datum. - if self._step_mode: - x_out, y_out = step_path_arrays_from_1d( - x_out, - y_out, - ) - # TODO: numba this bish - profiler('generated step arrays') + # step mode: draw flat top discrete "step" + # over the index space for each datum. + if self._step_mode: + x_out, y_out = step_path_arrays_from_1d( + x_out, + y_out, + ) + # TODO: numba this bish + profiler('generated step arrays') if should_redraw: - profiler('path reversion to non-ds') if self.path: + # print(f'CLEARING PATH {self._name}') self.path.clear() if self.fast_path: self.fast_path.clear() - if should_redraw and not should_ds: - if self._in_ds: - log.info(f'DEDOWN -> {self._name}') + profiler('cleared paths due to `should_redraw` set') + + if new_sample_rate and showing_src_data: + # if self._in_ds: + log.info(f'DEDOWN -> {self._name}') self._in_ds = False @@ -423,7 +442,12 @@ class FastAppendCurve(pg.GraphicsObject): finiteCheck=False, path=self.path, ) - profiler('generated fresh path') + profiler( + 'generated fresh path\n' + f'should_redraw: {should_redraw}\n' + f'should_ds: {should_ds}\n' + f'new_sample_rate: {new_sample_rate}\n' + ) # profiler(f'DRAW PATH IN VIEW -> {self._name}') # reserve mem allocs see: @@ -455,7 +479,7 @@ class FastAppendCurve(pg.GraphicsObject): elif ( append_length > 0 - and not view_range + # and not view_range ): new_x = x[-append_length - 2:-1] new_y = y[-append_length - 2:-1] @@ -696,7 +720,7 @@ class FastAppendCurve(pg.GraphicsObject): if path: p.drawPath(path) - profiler('.drawPath(path)') + profiler(f'.drawPath(path): {path.capacity()}') fp = self.fast_path if fp: From af6aad4e9c612c144b5304fbf68b666ce32ace05 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 22 Apr 2022 14:06:48 -0400 Subject: [PATCH 014/113] If a sample stream is already ded, just warn --- piker/data/_sampling.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/piker/data/_sampling.py b/piker/data/_sampling.py index 8bc677cf..466ef0e7 100644 --- a/piker/data/_sampling.py +++ b/piker/data/_sampling.py @@ -168,7 +168,12 @@ async def broadcast( log.error( f'{stream._ctx.chan.uid} dropped connection' ) - subs.remove(stream) + try: + subs.remove(stream) + except ValueError: + log.warning( + f'{stream._ctx.chan.uid} sub already removed!?' + ) @tractor.context From 64206543cda6ecd370b2c94e4b880c198dcb3abc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 22 Apr 2022 19:01:37 -0400 Subject: [PATCH 015/113] Put mxmn profile mapping at end of method --- piker/ui/_chart.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 8aa10091..cecbbff5 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1283,7 +1283,7 @@ class ChartPlotWidget(pg.PlotWidget): flow is None ): log.error(f"flow {flow_key} doesn't exist in chart {self.name} !?") - res = 0, 0 + key = res = 0, 0 else: first, l, lbar, rbar, r, last = bars_range or flow.datums_range() @@ -1291,11 +1291,11 @@ class ChartPlotWidget(pg.PlotWidget): key = round(lbar), round(rbar) res = flow.maxmin(*key) - profiler(f'yrange mxmn: {key} -> {res}') if res == (None, None): log.error( f"{flow_key} no mxmn for bars_range => {key} !?" ) res = 0, 0 + profiler(f'yrange mxmn: {key} -> {res}') return res From db727910bebe8e43449fefc5fa8a3b9b8650f385 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 22 Apr 2022 19:02:22 -0400 Subject: [PATCH 016/113] Always use coord cache, add naive view range diffing logic --- piker/ui/_curve.py | 56 ++++++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 8c3c329f..4b87d117 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -183,16 +183,16 @@ class FastAppendCurve(pg.GraphicsObject): # interactions slower (such as zooming) and if so maybe if/when # we implement a "history" mode for the view we disable this in # that mode? - if step_mode: - # don't enable caching by default for the case where the - # only thing drawn is the "last" line segment which can - # have a weird artifact where it won't be fully drawn to its - # endpoint (something we saw on trade rate curves) - self.setCacheMode( - QGraphicsItem.DeviceCoordinateCache - ) + # if step_mode: + # don't enable caching by default for the case where the + # only thing drawn is the "last" line segment which can + # have a weird artifact where it won't be fully drawn to its + # endpoint (something we saw on trade rate curves) + self.setCacheMode( + QGraphicsItem.DeviceCoordinateCache + ) - self.update() + # self.update() # TODO: probably stick this in a new parent # type which will contain our own version of @@ -313,7 +313,7 @@ class FastAppendCurve(pg.GraphicsObject): uppx_diff = (uppx - self._last_uppx) new_sample_rate = False - should_ds = False + should_ds = self._in_ds showing_src_data = self._in_ds should_redraw = False @@ -330,11 +330,27 @@ class FastAppendCurve(pg.GraphicsObject): x_out, y_out = x_iv[:-1], y_iv[:-1] profiler(f'view range slice {view_range}') + ivl, ivr = view_range + + probably_zoom_change = False + last_vr = self._vr + if last_vr: + livl, livr = last_vr + if ( + ivl < livl + or (ivr - livr) > 2 + ): + probably_zoom_change = True + if ( - view_range != self._vr - and append_length > 1 + view_range != last_vr + and ( + append_length > 1 + or probably_zoom_change + ) ): should_redraw = True + # print("REDRAWING BRUH") self._vr = view_range @@ -371,6 +387,7 @@ class FastAppendCurve(pg.GraphicsObject): self._last_uppx = uppx new_sample_rate = True showing_src_data = False + should_redraw = True should_ds = True elif ( @@ -504,13 +521,14 @@ class FastAppendCurve(pg.GraphicsObject): f'diffed array input, append_length={append_length}' ) - if should_ds: - new_x, new_y = self.downsample( - new_x, - new_y, - **should_ds, - ) - profiler(f'fast path downsample redraw={should_ds}') + # if should_ds: + # new_x, new_y = self.downsample( + # new_x, + # new_y, + # px_width, + # uppx, + # ) + # profiler(f'fast path downsample redraw={should_ds}') append_path = pg.functions.arrayToQPath( new_x, From aee44fed46351791de0bb50ef6cc3ef0b3b144c5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 22 Apr 2022 23:02:02 -0400 Subject: [PATCH 017/113] Right, handle the case where the shm prepend history isn't full XD --- piker/ui/_flows.py | 145 +++++++++++++++++++++++++++------------------ 1 file changed, 88 insertions(+), 57 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 20ffc510..a8f7d0a5 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -37,7 +37,7 @@ from PyQt5.QtGui import QPainterPath from ..data._sharedmem import ( ShmArray, - open_shm_array, + # open_shm_array, ) from .._profile import pg_profile_enabled, ms_slower_then from ._ohlc import ( @@ -48,7 +48,7 @@ from ._curve import ( FastAppendCurve, ) from ._compression import ( - ohlc_flatten, + # ohlc_flatten, ds_m4, ) from ..log import get_logger @@ -103,15 +103,15 @@ def rowarr_to_path( ) -def ohlc_flat_view( +def mk_ohlc_flat_copy( ohlc_shm: ShmArray, # XXX: we bind this in currently.. - x_basis: np.ndarray, + # x_basis: np.ndarray, # vr: Optional[slice] = None, -) -> np.ndarray: +) -> tuple[np.ndarray, np.ndarray]: ''' Return flattened-non-copy view into an OHLC shm array. @@ -127,8 +127,8 @@ def ohlc_flat_view( ) # breakpoint() y = unstructured.flatten() - x = x_basis[:y.size] - return x, y + # x = x_basis[:y.size] + return y class Flow(msgspec.Struct): # , frozen=True): @@ -151,7 +151,8 @@ class Flow(msgspec.Struct): # , frozen=True): render: bool = True # toggle for display loop flat: Optional[ShmArray] = None x_basis: Optional[np.ndarray] = None - _iflat: int = 0 + _iflat_last: int = 0 + _iflat_first: int = 0 _last_uppx: float = 0 _in_ds: bool = False @@ -344,7 +345,6 @@ class Flow(msgspec.Struct): # , frozen=True): graphics = self.graphics if isinstance(graphics, BarItems): - fields = ['open', 'high', 'low', 'close'] # if no source data renderer exists create one. r = self._src_r if not r: @@ -356,49 +356,11 @@ class Flow(msgspec.Struct): # , frozen=True): last_read=read, ) - # create a flattened view onto the OHLC array - # which can be read as a line-style format - shm = self.shm - - # flat = self.flat = self.shm.unstruct_view(fields) - self.flat = self.shm.ustruct(fields) - self._iflat = self.shm._last.value - - # import pdbpp - # pdbpp.set_trace() - # assert len(flat._array) == len(self.shm._array[fields]) - - x = self.x_basis = ( - np.broadcast_to( - shm._array['index'][:, None], - ( - shm._array.size, - # 4, # only ohlc - self.flat.shape[1], - ), - ) + np.array([-0.5, 0, 0, 0.5]) - ) - - # fshm = self.flat = open_shm_array( - # f'{self.name}_flat', - # dtype=flattened.dtype, - # size=flattened.size, - # ) - # fshm.push(flattened) - - # print(f'unstruct diff: {time.time() - start}') - # profiler('read unstr view bars to line') - # start = self.flat._first.value - ds_curve_r = Renderer( flow=self, # just swap in the flat view - data_t=lambda array: self.flat.array, - # data_t=partial( - # ohlc_flat_view, - # self.shm, - # ), + # data_t=lambda array: self.flat.array, last_read=read, draw_path=partial( rowarr_to_path, @@ -435,12 +397,14 @@ class Flow(msgspec.Struct): # , frozen=True): should_line and uppx < x_gt ): + print('FLIPPING TO BARS') should_line = False elif ( not should_line and uppx >= x_gt ): + print('FLIPPING TO LINE') should_line = True profiler(f'ds logic complete line={should_line}') @@ -448,32 +412,98 @@ class Flow(msgspec.Struct): # , frozen=True): # do graphics updates if should_line: + fields = ['open', 'high', 'low', 'close'] + if self.flat is None: + # create a flattened view onto the OHLC array + # which can be read as a line-style format + shm = self.shm + + # flat = self.flat = self.shm.unstruct_view(fields) + self.flat = self.shm.ustruct(fields) + first = self._iflat_first = self.shm._first.value + last = self._iflat_last = self.shm._last.value + + # write pushed data to flattened copy + self.flat[first:last] = rfn.structured_to_unstructured( + self.shm.array[fields] + ) + + # generate an flat-interpolated x-domain + self.x_basis = ( + np.broadcast_to( + shm._array['index'][:, None], + ( + shm._array.size, + # 4, # only ohlc + self.flat.shape[1], + ), + ) + np.array([-0.5, 0, 0, 0.5]) + ) + assert self.flat.any() + + # print(f'unstruct diff: {time.time() - start}') + # profiler('read unstr view bars to line') + # start = self.flat._first.value # update flatted ohlc copy - iflat, ishm = self._iflat, self.shm._last.value - to_update = rfn.structured_to_unstructured( - self.shm._array[iflat:ishm][fields] + ( + iflat_first, + iflat, + ishm_last, + ishm_first, + ) = ( + self._iflat_first, + self._iflat_last, + self.shm._last.value, + self.shm._first.value ) - # print(to_update) - self.flat[iflat:ishm][:] = to_update + # check for shm prepend updates since last read. + if iflat_first != ishm_first: + + # write newly prepended data to flattened copy + self.flat[ + ishm_first:iflat_first + ] = rfn.structured_to_unstructured( + self.shm.array[fields][:iflat_first] + ) + self._iflat_first = ishm_first + + # # flat = self.flat = self.shm.unstruct_view(fields) + # self.flat = self.shm.ustruct(fields) + # # self._iflat_last = self.shm._last.value + + # # self._iflat_first = self.shm._first.value + # # do an update for the most recent prepend + # # index + # iflat = ishm_first + + to_update = rfn.structured_to_unstructured( + self.shm._array[iflat:ishm_last][fields] + ) + + self.flat[iflat:ishm_last][:] = to_update profiler('updated ustruct OHLC data') - y_flat = self.flat[:ishm] - x_flat = self.x_basis[:ishm] + # slice out up-to-last step contents + y_flat = self.flat[ishm_first:ishm_last] + x_flat = self.x_basis[ishm_first:ishm_last] - self._iflat = ishm + # update local last-index tracking + self._iflat_last = ishm_last + # reshape to 1d for graphics rendering y = y_flat.reshape(-1) x = x_flat.reshape(-1) profiler('flattened ustruct OHLC data') + # do all the same for only in-view data y_iv_flat = y_flat[ivl:ivr] x_iv_flat = x_flat[ivl:ivr] - y_iv = y_iv_flat.reshape(-1) x_iv = x_iv_flat.reshape(-1) profiler('flattened ustruct in-view OHLC data') + # legacy full-recompute-everytime method # x, y = ohlc_flatten(array) # x_iv, y_iv = ohlc_flatten(in_view) # profiler('flattened OHLC data') @@ -486,6 +516,7 @@ class Flow(msgspec.Struct): # , frozen=True): view_range=(ivl, ivr), # hack profiler=profiler, ) + curve.show() profiler('updated ds curve') else: From 69282a99246f50c36ebb22f89c03261c067fa46e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 23 Apr 2022 15:33:40 -0400 Subject: [PATCH 018/113] Handle null output case for vlm chart mxmn --- piker/ui/_display.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 0e03395e..b82d1253 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -106,7 +106,9 @@ def chart_maxmin( mx_vlm_in_view = 0 if vlm_chart: - _, mx_vlm_in_view = vlm_chart.maxmin() + out = vlm_chart.maxmin() + if out: + _, mx_vlm_in_view = out return ( last_bars_range, From 64c6287cd1f046e9c4c67526bf4ec0169558515f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 23 Apr 2022 17:22:02 -0400 Subject: [PATCH 019/113] Always set coords cache on curves --- piker/ui/_curve.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 4b87d117..0befe454 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -188,11 +188,7 @@ class FastAppendCurve(pg.GraphicsObject): # only thing drawn is the "last" line segment which can # have a weird artifact where it won't be fully drawn to its # endpoint (something we saw on trade rate curves) - self.setCacheMode( - QGraphicsItem.DeviceCoordinateCache - ) - - # self.update() + self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) # TODO: probably stick this in a new parent # type which will contain our own version of @@ -423,6 +419,9 @@ class FastAppendCurve(pg.GraphicsObject): x_out, y_out, ) + # self.disable_cache() + # flip_cache = True + # TODO: numba this bish profiler('generated step arrays') @@ -514,6 +513,9 @@ class FastAppendCurve(pg.GraphicsObject): new_x = new_x[1:] new_y = new_y[1:] + # self.disable_cache() + # flip_cache = True + profiler('generated step data') else: From b97ec38baff4f8ca5db1bdb13cdf5885ceda4316 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 23 Apr 2022 17:22:28 -0400 Subject: [PATCH 020/113] Always maybe render graphics Since we have in-view style rendering working for all curve types (finally) we can avoid the guard for low uppx levels and without losing interaction speed. Further don't delay the profiler so that the nested method calls correctly report upward - which wasn't working likely due to some kinda GC collection related issue. --- piker/ui/_interaction.py | 84 +++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index ca49a35d..943f3370 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -20,7 +20,6 @@ Chart view box primitives """ from __future__ import annotations from contextlib import asynccontextmanager -# import itertools import time from typing import Optional, Callable @@ -907,54 +906,59 @@ class ChartView(ViewBox): def maybe_downsample_graphics(self): uppx = self.x_uppx() - if not ( - # we probably want to drop this once we are "drawing in - # view" for downsampled flows.. - uppx and uppx > 6 - and self._ic is not None - ): - profiler = pg.debug.Profiler( - msg=f'ChartView.maybe_downsample_graphics() for {self.name}', - disabled=not pg_profile_enabled(), - # delayed=True, - gt=3, - # gt=ms_slower_then, - ) + # if not ( + # # we probably want to drop this once we are "drawing in + # # view" for downsampled flows.. + # uppx and uppx > 6 + # and self._ic is not None + # ): + profiler = pg.debug.Profiler( + msg=f'ChartView.maybe_downsample_graphics() for {self.name}', + disabled=not pg_profile_enabled(), - # TODO: a faster single-loop-iterator way of doing this XD - chart = self._chart - linked = self.linkedsplits - plots = linked.subplots | {chart.name: chart} - for chart_name, chart in plots.items(): - for name, flow in chart._flows.items(): + # XXX: important to avoid not seeing underlying + # ``.update_graphics_from_flow()`` nested profiling likely + # due to the way delaying works and garbage collection of + # the profiler in the delegated method calls. + delayed=False, + # gt=3, + # gt=ms_slower_then, + ) - if ( - not flow.render + # TODO: a faster single-loop-iterator way of doing this XD + chart = self._chart + linked = self.linkedsplits + plots = linked.subplots | {chart.name: chart} + for chart_name, chart in plots.items(): + for name, flow in chart._flows.items(): - # XXX: super important to be aware of this. - # or not flow.graphics.isVisible() - ): - continue + if ( + not flow.render - graphics = flow.graphics + # XXX: super important to be aware of this. + # or not flow.graphics.isVisible() + ): + continue - use_vr = False - if isinstance(graphics, BarItems): - use_vr = True + graphics = flow.graphics - # pass in no array which will read and render from the last - # passed array (normally provided by the display loop.) - chart.update_graphics_from_flow( - name, - use_vr=use_vr, + # use_vr = False + # if isinstance(graphics, BarItems): + # use_vr = True - # gets passed down into graphics obj - # profiler=profiler, - ) + # pass in no array which will read and render from the last + # passed array (normally provided by the display loop.) + chart.update_graphics_from_flow( + name, + use_vr=True, - profiler(f'range change updated {chart_name}:{name}') + # gets passed down into graphics obj + # profiler=profiler, + ) - profiler.finish() + profiler(f'range change updated {chart_name}:{name}') + + profiler.finish() # else: # # don't bother updating since we're zoomed out bigly and # # in a pan-interaction, in which case we shouldn't be From b2b31b8f841799d843eb991d35dc4eff02703a76 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 24 Apr 2022 12:33:25 -0400 Subject: [PATCH 021/113] WIP incrementally update step array format --- piker/ui/_curve.py | 114 ++++++++++++++++++------------ piker/ui/_flows.py | 170 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 215 insertions(+), 69 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 0befe454..e2803549 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -58,36 +58,55 @@ def step_path_arrays_from_1d( ''' y_out = y.copy() x_out = x.copy() - x2 = np.empty( - # the data + 2 endpoints on either end for - # "termination of the path". - (len(x) + 1, 2), - # we want to align with OHLC or other sampling style - # bars likely so we need fractinal values - dtype=float, - ) - x2[0] = x[0] - 0.5 - x2[1] = x[0] + 0.5 - x2[1:] = x[:, np.newaxis] + 0.5 + + # x2 = np.empty( + # # the data + 2 endpoints on either end for + # # "termination of the path". + # (len(x) + 1, 2), + # # we want to align with OHLC or other sampling style + # # bars likely so we need fractinal values + # dtype=float, + # ) + + x2 = np.broadcast_to( + x[:, None], + ( + x_out.size, + # 4, # only ohlc + 2, + ), + ) + np.array([-0.5, 0.5]) + + # x2[0] = x[0] - 0.5 + # x2[1] = x[0] + 0.5 + # x2[0, 0] = x[0] - 0.5 + # x2[0, 1] = x[0] + 0.5 + # x2[1:] = x[:, np.newaxis] + 0.5 + # import pdbpp + # pdbpp.set_trace() # flatten to 1-d - x_out = x2.reshape(x2.size) + # x_out = x2.reshape(x2.size) + x_out = x2 # we create a 1d with 2 extra indexes to # hold the start and (current) end value for the steps # on either end y2 = np.empty((len(y), 2), dtype=y.dtype) y2[:] = y[:, np.newaxis] + y2[-1] = 0 - y_out = np.empty( - 2*len(y) + 2, - dtype=y.dtype - ) + y_out = y2 + +# y_out = np.empty( +# 2*len(y) + 2, +# dtype=y.dtype +# ) # flatten and set 0 endpoints - y_out[1:-1] = y2.reshape(y2.size) - y_out[0] = 0 - y_out[-1] = 0 + # y_out[1:-1] = y2.reshape(y2.size) + # y_out[0] = 0 + # y_out[-1] = 0 if not include_endpoints: return x_out[:-1], y_out[:-1] @@ -414,16 +433,16 @@ class FastAppendCurve(pg.GraphicsObject): # step mode: draw flat top discrete "step" # over the index space for each datum. - if self._step_mode: - x_out, y_out = step_path_arrays_from_1d( - x_out, - y_out, - ) - # self.disable_cache() - # flip_cache = True + # if self._step_mode: + # x_out, y_out = step_path_arrays_from_1d( + # x_out, + # y_out, + # ) + # # self.disable_cache() + # # flip_cache = True - # TODO: numba this bish - profiler('generated step arrays') + # # TODO: numba this bish + # profiler('generated step arrays') if should_redraw: if self.path: @@ -501,27 +520,26 @@ class FastAppendCurve(pg.GraphicsObject): new_y = y[-append_length - 2:-1] profiler('sliced append path') - if self._step_mode: - new_x, new_y = step_path_arrays_from_1d( - new_x, - new_y, - ) - # [1:] since we don't need the vertical line normally at - # the beginning of the step curve taking the first (x, - # y) poing down to the x-axis **because** this is an - # appended path graphic. - new_x = new_x[1:] - new_y = new_y[1:] + # if self._step_mode: + # new_x, new_y = step_path_arrays_from_1d( + # new_x, + # new_y, + # ) + # # [1:] since we don't need the vertical line normally at + # # the beginning of the step curve taking the first (x, + # # y) poing down to the x-axis **because** this is an + # # appended path graphic. + # new_x = new_x[1:] + # new_y = new_y[1:] - # self.disable_cache() - # flip_cache = True + # # self.disable_cache() + # # flip_cache = True - profiler('generated step data') + # profiler('generated step data') - else: - profiler( - f'diffed array input, append_length={append_length}' - ) + profiler( + f'diffed array input, append_length={append_length}' + ) # if should_ds: # new_x, new_y = self.downsample( @@ -655,6 +673,10 @@ class FastAppendCurve(pg.GraphicsObject): # self.disable_cache() # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + def reset_cache(self) -> None: + self.disable_cache() + self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + def disable_cache(self) -> None: ''' Disable the use of the pixel coordinate cache and trigger a geo event. diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index a8f7d0a5..b150a2d1 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -46,6 +46,7 @@ from ._ohlc import ( ) from ._curve import ( FastAppendCurve, + step_path_arrays_from_1d, ) from ._compression import ( # ohlc_flatten, @@ -149,8 +150,8 @@ class Flow(msgspec.Struct): # , frozen=True): is_ohlc: bool = False render: bool = True # toggle for display loop - flat: Optional[ShmArray] = None - x_basis: Optional[np.ndarray] = None + gy: Optional[ShmArray] = None + gx: Optional[np.ndarray] = None _iflat_last: int = 0 _iflat_first: int = 0 @@ -360,7 +361,7 @@ class Flow(msgspec.Struct): # , frozen=True): flow=self, # just swap in the flat view - # data_t=lambda array: self.flat.array, + # data_t=lambda array: self.gy.array, last_read=read, draw_path=partial( rowarr_to_path, @@ -413,37 +414,37 @@ class Flow(msgspec.Struct): # , frozen=True): if should_line: fields = ['open', 'high', 'low', 'close'] - if self.flat is None: + if self.gy is None: # create a flattened view onto the OHLC array # which can be read as a line-style format shm = self.shm - # flat = self.flat = self.shm.unstruct_view(fields) - self.flat = self.shm.ustruct(fields) + # flat = self.gy = self.shm.unstruct_view(fields) + self.gy = self.shm.ustruct(fields) first = self._iflat_first = self.shm._first.value last = self._iflat_last = self.shm._last.value # write pushed data to flattened copy - self.flat[first:last] = rfn.structured_to_unstructured( + self.gy[first:last] = rfn.structured_to_unstructured( self.shm.array[fields] ) # generate an flat-interpolated x-domain - self.x_basis = ( + self.gx = ( np.broadcast_to( shm._array['index'][:, None], ( shm._array.size, # 4, # only ohlc - self.flat.shape[1], + self.gy.shape[1], ), ) + np.array([-0.5, 0, 0, 0.5]) ) - assert self.flat.any() + assert self.gy.any() # print(f'unstruct diff: {time.time() - start}') # profiler('read unstr view bars to line') - # start = self.flat._first.value + # start = self.gy._first.value # update flatted ohlc copy ( iflat_first, @@ -461,15 +462,15 @@ class Flow(msgspec.Struct): # , frozen=True): if iflat_first != ishm_first: # write newly prepended data to flattened copy - self.flat[ + self.gy[ ishm_first:iflat_first ] = rfn.structured_to_unstructured( self.shm.array[fields][:iflat_first] ) self._iflat_first = ishm_first - # # flat = self.flat = self.shm.unstruct_view(fields) - # self.flat = self.shm.ustruct(fields) + # # flat = self.gy = self.shm.unstruct_view(fields) + # self.gy = self.shm.ustruct(fields) # # self._iflat_last = self.shm._last.value # # self._iflat_first = self.shm._first.value @@ -481,12 +482,12 @@ class Flow(msgspec.Struct): # , frozen=True): self.shm._array[iflat:ishm_last][fields] ) - self.flat[iflat:ishm_last][:] = to_update + self.gy[iflat:ishm_last][:] = to_update profiler('updated ustruct OHLC data') # slice out up-to-last step contents - y_flat = self.flat[ishm_first:ishm_last] - x_flat = self.x_basis[ishm_first:ishm_last] + y_flat = self.gy[ishm_first:ishm_last] + x_flat = self.gx[ishm_first:ishm_last] # update local last-index tracking self._iflat_last = ishm_last @@ -577,16 +578,139 @@ class Flow(msgspec.Struct): # , frozen=True): # graphics.draw_last(last) else: - # ``FastAppendCurve`` case: array_key = array_key or self.name - graphics.update_from_array( - x=array['index'], - y=array[array_key], + # ``FastAppendCurve`` case: + if graphics._step_mode and self.gy is None: + + # create a flattened view onto the OHLC array + # which can be read as a line-style format + shm = self.shm + + # fields = ['index', array_key] + i = shm._array['index'] + out = shm._array[array_key] + + self.gx, self.gy = step_path_arrays_from_1d(i, out) + + # flat = self.gy = self.shm.unstruct_view(fields) + # self.gy = self.shm.ustruct(fields) + # first = self._iflat_first = self.shm._first.value + # last = self._iflat_last = self.shm._last.value + + # # write pushed data to flattened copy + # self.gy[first:last] = rfn.structured_to_unstructured( + # self.shm.array[fields] + # ) + + # # generate an flat-interpolated x-domain + # self.gx = ( + # np.broadcast_to( + # shm._array['index'][:, None], + # ( + # shm._array.size, + # # 4, # only ohlc + # self.gy.shape[1], + # ), + # ) + np.array([-0.5, 0, 0, 0.5]) + # ) + # assert self.gy.any() + + # print(f'unstruct diff: {time.time() - start}') + # profiler('read unstr view bars to line') + # start = self.gy._first.value + # update flatted ohlc copy + + if graphics._step_mode: + ( + iflat_first, + iflat, + ishm_last, + ishm_first, + ) = ( + self._iflat_first, + self._iflat_last, + self.shm._last.value, + self.shm._first.value + ) + + # check for shm prepend updates since last read. + if iflat_first != ishm_first: + + # write newly prepended data to flattened copy + _gx, self.gy[ + ishm_first:iflat_first + ] = step_path_arrays_from_1d( + self.shm.array['index'][:iflat_first], + self.shm.array[array_key][:iflat_first], + ) + self._iflat_first = ishm_first + # # flat = self.gy = self.shm.unstruct_view(fields) + # self.gy = self.shm.ustruct(fields) + # # self._iflat_last = self.shm._last.value + + # # self._iflat_first = self.shm._first.value + # # do an update for the most recent prepend + # # index + # iflat = ishm_first + if iflat != ishm_last: + _x, to_update = step_path_arrays_from_1d( + self.shm._array[iflat:ishm_last]['index'], + self.shm._array[iflat:ishm_last][array_key], + ) + + # to_update = rfn.structured_to_unstructured( + # self.shm._array[iflat:ishm_last][fields] + # ) + + # import pdbpp + # pdbpp.set_trace() + self.gy[iflat:ishm_last-1] = to_update + self.gy[-1] = 0 + print(f'updating step curve {to_update}') + profiler('updated step curve data') + + # slice out up-to-last step contents + x_step = self.gx[ishm_first:ishm_last] + x = x_step.reshape(-1) + y_step = self.gy[ishm_first:ishm_last] + y = y_step.reshape(-1) + profiler('sliced step data') + + # update local last-index tracking + self._iflat_last = ishm_last + + # reshape to 1d for graphics rendering + # y = y_flat.reshape(-1) + # x = x_flat.reshape(-1) + + # do all the same for only in-view data + y_iv = y_step[ivl:ivr].reshape(-1) + x_iv = x_step[ivl:ivr].reshape(-1) + # y_iv = y_iv_flat.reshape(-1) + # x_iv = x_iv_flat.reshape(-1) + profiler('flattened ustruct in-view OHLC data') + + # legacy full-recompute-everytime method + # x, y = ohlc_flatten(array) + # x_iv, y_iv = ohlc_flatten(in_view) + # profiler('flattened OHLC data') + graphics.reset_cache() + + else: + x = array['index'] + y = array[array_key] + x_iv = in_view['index'] + y_iv = in_view[array_key] + + graphics.update_from_array( + x=x, + y=y, + + x_iv=x_iv, + y_iv=y_iv, - x_iv=in_view['index'], - y_iv=in_view[array_key], view_range=(ivl, ivr) if use_vr else None, **kwargs From 82b2d2ee3a4be377246330c2dadb6e0c0ccbf755 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 5 Apr 2022 14:54:13 -0400 Subject: [PATCH 022/113] Hipshot, use uppx to drive theoretical px w --- piker/ui/_compression.py | 44 ++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index adb42251..a6102eab 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -181,6 +181,7 @@ def ds_m4( # in display-device-local pixel units. px_width: int, uppx: Optional[float] = None, + xrange: Optional[float] = None, log_scale: bool = True, ) -> tuple[int, np.ndarray, np.ndarray]: @@ -212,6 +213,7 @@ def ds_m4( # as the units-per-px (uppx) get's large. if log_scale: assert uppx, 'You must provide a `uppx` value to use log scaling!' + # uppx = uppx * math.log(uppx, 2) # scaler = 2**7 / (1 + math.log(uppx, 2)) scaler = round( @@ -223,37 +225,63 @@ def ds_m4( 1 ) ) - px_width *= scaler + # px_width *= scaler + + # else: + # px_width *= 16 assert px_width > 1 # width of screen in pxs? + assert uppx > 0 # NOTE: if we didn't pre-slice the data to downsample # you could in theory pass these as the slicing params, # do we care though since we can always just pre-slice the # input? x_start = x[0] # x value start/lowest in domain - x_end = x[-1] # x end value/highest in domain + + if xrange is None: + x_end = x[-1] # x end value/highest in domain + xrange = (x_end - x_start) # XXX: always round up on the input pixels - px_width = math.ceil(px_width) + # lnx = len(x) + # uppx *= max(4 / (1 + math.log(uppx, 2)), 1) - x_range = x_end - x_start + pxw = math.ceil(xrange / uppx) + px_width = math.ceil(px_width) # ratio of indexed x-value to width of raster in pixels. # this is more or less, uppx: units-per-pixel. - w = x_range / float(px_width) + # w = xrange / float(px_width) + # uppx = uppx * math.log(uppx, 2) + w2 = px_width / uppx + + # scale up the width as the uppx get's large + w = uppx# * math.log(uppx, 666) # ensure we make more then enough # frames (windows) for the output pixel - frames = px_width + frames = pxw # if we have more and then exact integer's # (uniform quotient output) worth of datum-domain-points # per windows-frame, add one more window to ensure # we have room for all output down-samples. - pts_per_pixel, r = divmod(len(x), frames) + pts_per_pixel, r = divmod(xrange, frames) if r: - frames += 1 + while r: + frames += 1 + pts_per_pixel, r = divmod(xrange, frames) + + print( + f'uppx: {uppx}\n' + f'xrange: {xrange}\n' + f'px_width: {px_width}\n' + f'pxw: {pxw}\n' + f'WTF w:{w}, w2:{w2}\n' + f'frames: {frames}\n' + ) + assert frames >= (xrange / uppx) # call into ``numba`` nb, i_win, y_out = _m4( From ba0ba346ec951324392fccd00422b46db8bb2f5f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 24 Apr 2022 17:08:16 -0400 Subject: [PATCH 023/113] Drop log scaling support since uppx driven scaling seems way faster/better --- piker/ui/_compression.py | 64 ++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index a6102eab..5e8b759a 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -162,7 +162,7 @@ def ohlc_to_m4_line( flat, px_width=px_width, uppx=uppx, - log_scale=bool(uppx) + # log_scale=bool(uppx) ) x = np.broadcast_to(x[:, None], y.shape) x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() @@ -182,7 +182,7 @@ def ds_m4( px_width: int, uppx: Optional[float] = None, xrange: Optional[float] = None, - log_scale: bool = True, + # log_scale: bool = True, ) -> tuple[int, np.ndarray, np.ndarray]: ''' @@ -211,27 +211,27 @@ def ds_m4( # optionally log-scale down the "supposed pxs on screen" # as the units-per-px (uppx) get's large. - if log_scale: - assert uppx, 'You must provide a `uppx` value to use log scaling!' - # uppx = uppx * math.log(uppx, 2) + # if log_scale: + # assert uppx, 'You must provide a `uppx` value to use log scaling!' + # # uppx = uppx * math.log(uppx, 2) - # scaler = 2**7 / (1 + math.log(uppx, 2)) - scaler = round( - max( - # NOTE: found that a 16x px width brought greater - # detail, likely due to dpi scaling? - # px_width=px_width * 16, - 2**7 / (1 + math.log(uppx, 2)), - 1 - ) - ) - # px_width *= scaler + # # scaler = 2**7 / (1 + math.log(uppx, 2)) + # scaler = round( + # max( + # # NOTE: found that a 16x px width brought greater + # # detail, likely due to dpi scaling? + # # px_width=px_width * 16, + # 2**7 / (1 + math.log(uppx, 2)), + # 1 + # ) + # ) + # px_width *= scaler # else: # px_width *= 16 - assert px_width > 1 # width of screen in pxs? - assert uppx > 0 + # should never get called unless actually needed + assert px_width > 1 and uppx > 0 # NOTE: if we didn't pre-slice the data to downsample # you could in theory pass these as the slicing params, @@ -248,16 +248,16 @@ def ds_m4( # uppx *= max(4 / (1 + math.log(uppx, 2)), 1) pxw = math.ceil(xrange / uppx) - px_width = math.ceil(px_width) + # px_width = math.ceil(px_width) # ratio of indexed x-value to width of raster in pixels. # this is more or less, uppx: units-per-pixel. # w = xrange / float(px_width) # uppx = uppx * math.log(uppx, 2) - w2 = px_width / uppx + # w2 = px_width / uppx # scale up the width as the uppx get's large - w = uppx# * math.log(uppx, 666) + w = uppx # * math.log(uppx, 666) # ensure we make more then enough # frames (windows) for the output pixel @@ -269,18 +269,18 @@ def ds_m4( # we have room for all output down-samples. pts_per_pixel, r = divmod(xrange, frames) if r: - while r: - frames += 1 - pts_per_pixel, r = divmod(xrange, frames) + # while r: + frames += 1 + pts_per_pixel, r = divmod(xrange, frames) - print( - f'uppx: {uppx}\n' - f'xrange: {xrange}\n' - f'px_width: {px_width}\n' - f'pxw: {pxw}\n' - f'WTF w:{w}, w2:{w2}\n' - f'frames: {frames}\n' - ) + # print( + # f'uppx: {uppx}\n' + # f'xrange: {xrange}\n' + # f'px_width: {px_width}\n' + # f'pxw: {pxw}\n' + # f'WTF w:{w}, w2:{w2}\n' + # f'frames: {frames}\n' + # ) assert frames >= (xrange / uppx) # call into ``numba`` From 629ea8ba9d9c7dcbb0b6d955a7c6d75023484e39 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 24 Apr 2022 17:09:30 -0400 Subject: [PATCH 024/113] Downsample on every uppx inrement since it's way faster --- piker/ui/_curve.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index e2803549..cf987203 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -259,7 +259,7 @@ class FastAppendCurve(pg.GraphicsObject): y, px_width=px_width, uppx=uppx, - log_scale=bool(uppx) + # log_scale=bool(uppx) ) x = np.broadcast_to(x[:, None], y.shape) # x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() @@ -391,9 +391,9 @@ class FastAppendCurve(pg.GraphicsObject): if ( # std m4 downsample conditions px_width - and uppx_diff >= 4 - or uppx_diff <= -3 - or self._step_mode and abs(uppx_diff) >= 4 + and uppx_diff >= 1 + or uppx_diff <= -1 + or self._step_mode and abs(uppx_diff) >= 2 ): log.info( @@ -460,7 +460,7 @@ class FastAppendCurve(pg.GraphicsObject): self._in_ds = False - elif should_ds and px_width: + elif should_ds and px_width and uppx: x_out, y_out = self.downsample( x_out, y_out, From c5beecf8a1a1b3ffe5c24e68c57dc5d2ac4c1274 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 24 Apr 2022 17:09:58 -0400 Subject: [PATCH 025/113] Drop cursor debounce delay, decrease rate limit --- piker/ui/_cursor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_cursor.py b/piker/ui/_cursor.py index 43207b9f..8f18fe45 100644 --- a/piker/ui/_cursor.py +++ b/piker/ui/_cursor.py @@ -43,8 +43,8 @@ log = get_logger(__name__) # latency (in terms of perceived lag in cross hair) so really be sure # there's an improvement if you want to change it! -_mouse_rate_limit = 120 # TODO; should we calc current screen refresh rate? -_debounce_delay = 1 / 40 +_mouse_rate_limit = 60 # TODO; should we calc current screen refresh rate? +_debounce_delay = 0 _ch_label_opac = 1 From 12d60e6d9c0df35390fad59167717fb060099fd8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 08:34:53 -0400 Subject: [PATCH 026/113] WIP get incremental step curve updates working This took longer then i care to admit XD but it definitely adds a huge speedup and with only a few outstanding correctness bugs: - panning from left to right causes strange trailing artifacts in the flows fsp (vlm) sub-plot but only when some data is off-screen on the left but doesn't appear to be an issue if we keep the `._set_yrange()` handler hooked up to the `.sigXRangeChanged` signal (but we aren't going to because this makes panning way slower). i've got a feeling this is a bug todo with the device coordinate cache stuff and we may need to report to Qt core? - factoring out the step curve logic from `FastAppendCurve.update_from_array()` (un)fortunately required some logic branch uncoupling but also meant we needed special input controls to avoid things like redraws and curve appends for special cases, this will hopefully all be better rectified in code when the core of this method is moved into a renderer type/implementation. - the `tina_vwap` fsp curve now somehow causes hangs when doing erratic scrolling on downsampled graphics data. i have no idea why or how but disabling it makes the issue go away (ui will literally just freeze and gobble CPU on a `.paint()` call until you ctrl-c the hell out of it). my guess is that something in the logic for standard line curves and appends on large data sets is the issue? Code related changes/hacks: - drop use of `step_path_arrays_from_1d()`, it was always a bit hacky (being based on `pyqtgraph` internals) and was generally hard to understand since it returns 1d data instead of the more expected (N,2) array of "step levels"; instead this is now implemented (uglily) in the `Flow.update_graphics()` block for step curves (which will obviously get cleaned up and factored elsewhere). - add a bunch of new flags to the update method on the fast append curve: `draw_last: bool`, `slice_to_head: int`, `do_append: bool`, `should_redraw: bool` which are all controls to aid with previously mentioned issues specific to getting step curve updates working correctly. - add a ton of commented tinkering related code (that we may end up using) to both the flow and append curve methods that was written as part of the effort to get this all working. - implement all step curve updating inline in `Flow.update_graphics()` including prepend and append logic for pre-graphics incremental step data maintenance and in-view slicing as well as "last step" graphics updating. Obviously clean up commits coming stat B) --- piker/ui/_curve.py | 274 +++++++++++++++++++++++++++++---------------- piker/ui/_flows.py | 233 +++++++++++++++++++++++++++++++------- 2 files changed, 369 insertions(+), 138 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index cf987203..60353f08 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -45,74 +45,77 @@ log = get_logger(__name__) # TODO: numba this instead.. -def step_path_arrays_from_1d( - x: np.ndarray, - y: np.ndarray, - include_endpoints: bool = False, +# def step_path_arrays_from_1d( +# x: np.ndarray, +# y: np.ndarray, +# include_endpoints: bool = True, -) -> (np.ndarray, np.ndarray): - ''' - Generate a "step mode" curve aligned with OHLC style bars - such that each segment spans each bar (aka "centered" style). +# ) -> (np.ndarray, np.ndarray): +# ''' +# Generate a "step mode" curve aligned with OHLC style bars +# such that each segment spans each bar (aka "centered" style). - ''' - y_out = y.copy() - x_out = x.copy() +# ''' +# # y_out = y.copy() +# # x_out = x.copy() - # x2 = np.empty( - # # the data + 2 endpoints on either end for - # # "termination of the path". - # (len(x) + 1, 2), - # # we want to align with OHLC or other sampling style - # # bars likely so we need fractinal values - # dtype=float, - # ) +# # x2 = np.empty( +# # # the data + 2 endpoints on either end for +# # # "termination of the path". +# # (len(x) + 1, 2), +# # # we want to align with OHLC or other sampling style +# # # bars likely so we need fractinal values +# # dtype=float, +# # ) - x2 = np.broadcast_to( - x[:, None], - ( - x_out.size, - # 4, # only ohlc - 2, - ), - ) + np.array([-0.5, 0.5]) +# x2 = np.broadcast_to( +# x[:, None], +# ( +# x.size + 1, +# # 4, # only ohlc +# 2, +# ), +# ) + np.array([-0.5, 0.5]) - # x2[0] = x[0] - 0.5 - # x2[1] = x[0] + 0.5 - # x2[0, 0] = x[0] - 0.5 - # x2[0, 1] = x[0] + 0.5 - # x2[1:] = x[:, np.newaxis] + 0.5 - # import pdbpp - # pdbpp.set_trace() +# # x2[0] = x[0] - 0.5 +# # x2[1] = x[0] + 0.5 +# # x2[0, 0] = x[0] - 0.5 +# # x2[0, 1] = x[0] + 0.5 +# # x2[1:] = x[:, np.newaxis] + 0.5 +# # import pdbpp +# # pdbpp.set_trace() - # flatten to 1-d - # x_out = x2.reshape(x2.size) - x_out = x2 +# # flatten to 1-d +# # x_out = x2.reshape(x2.size) +# # x_out = x2 - # we create a 1d with 2 extra indexes to - # hold the start and (current) end value for the steps - # on either end - y2 = np.empty((len(y), 2), dtype=y.dtype) - y2[:] = y[:, np.newaxis] - y2[-1] = 0 - - y_out = y2 - -# y_out = np.empty( -# 2*len(y) + 2, -# dtype=y.dtype +# # we create a 1d with 2 extra indexes to +# # hold the start and (current) end value for the steps +# # on either end +# y2 = np.empty( +# (len(y) + 1, 2), +# dtype=y.dtype, # ) +# y2[:] = y[:, np.newaxis] +# # y2[-1] = 0 - # flatten and set 0 endpoints - # y_out[1:-1] = y2.reshape(y2.size) - # y_out[0] = 0 - # y_out[-1] = 0 +# # y_out = y2 - if not include_endpoints: - return x_out[:-1], y_out[:-1] +# # y_out = np.empty( +# # 2*len(y) + 2, +# # dtype=y.dtype +# # ) - else: - return x_out, y_out +# # flatten and set 0 endpoints +# # y_out[1:-1] = y2.reshape(y2.size) +# # y_out[0] = 0 +# # y_out[-1] = 0 + +# if not include_endpoints: +# return x2[:-1], y2[:-1] + +# else: +# return x2, y2 _line_styles: dict[str, int] = { @@ -158,6 +161,8 @@ class FastAppendCurve(pg.GraphicsObject): self._y = self.yData = y self._x = self.xData = x self._vr: Optional[tuple] = None + self._avr: Optional[tuple] = None + self._br = None self._name = name self.path: Optional[QtGui.QPainterPath] = None @@ -171,6 +176,7 @@ class FastAppendCurve(pg.GraphicsObject): # self._xrange: tuple[int, int] = self.dataBounds(ax=0) self._xrange: Optional[tuple[int, int]] = None + # self._x_iv_range = None # self._last_draw = time.time() self._in_ds: bool = False @@ -283,6 +289,10 @@ class FastAppendCurve(pg.GraphicsObject): view_range: Optional[tuple[int, int]] = None, profiler: Optional[pg.debug.Profiler] = None, + draw_last: bool = True, + slice_to_head: int = -1, + do_append: bool = True, + should_redraw: bool = False, ) -> QtGui.QPainterPath: ''' @@ -297,7 +307,7 @@ class FastAppendCurve(pg.GraphicsObject): disabled=not pg_profile_enabled(), gt=ms_slower_then, ) - # flip_cache = False + flip_cache = False if self._xrange: istart, istop = self._xrange @@ -330,7 +340,7 @@ class FastAppendCurve(pg.GraphicsObject): new_sample_rate = False should_ds = self._in_ds showing_src_data = self._in_ds - should_redraw = False + # should_redraw = False # if a view range is passed, plan to draw the # source ouput that's "in view" of the chart. @@ -342,32 +352,60 @@ class FastAppendCurve(pg.GraphicsObject): # print(f'{self._name} vr: {view_range}') # by default we only pull data up to the last (current) index - x_out, y_out = x_iv[:-1], y_iv[:-1] + x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head] profiler(f'view range slice {view_range}') - ivl, ivr = view_range + vl, vr = view_range - probably_zoom_change = False + # last_ivr = self._x_iv_range + # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) + + zoom_or_append = False last_vr = self._vr + last_ivr = self._avr + if last_vr: - livl, livr = last_vr + # relative slice indices + lvl, lvr = last_vr + # abs slice indices + al, ar = last_ivr + + # append_length = int(x[-1] - istop) + # append_length = int(x_iv[-1] - ar) + + # left_change = abs(x_iv[0] - al) >= 1 + # right_change = abs(x_iv[-1] - ar) >= 1 + if ( - ivl < livl - or (ivr - livr) > 2 + # likely a zoom view change + (vr - lvr) > 2 or vl < lvl + # append / prepend update + # we had an append update where the view range + # didn't change but the data-viewed (shifted) + # underneath, so we need to redraw. + # or left_change and right_change and last_vr == view_range + + # not (left_change and right_change) and ivr + # ( + # or abs(x_iv[ivr] - livr) > 1 ): - probably_zoom_change = True + zoom_or_append = True + + # if last_ivr: + # liivl, liivr = last_ivr if ( view_range != last_vr and ( append_length > 1 - or probably_zoom_change + or zoom_or_append ) ): should_redraw = True # print("REDRAWING BRUH") self._vr = view_range + self._avr = x_iv[0], x_iv[slice_to_head] # x_last = x_iv[-1] # y_last = y_iv[-1] @@ -382,7 +420,7 @@ class FastAppendCurve(pg.GraphicsObject): # or self._in_ds # ): # by default we only pull data up to the last (current) index - x_out, y_out = x[:-1], y[:-1] + x_out, y_out = x[:slice_to_head], y[:slice_to_head] if prepend_length > 0: should_redraw = True @@ -434,12 +472,12 @@ class FastAppendCurve(pg.GraphicsObject): # step mode: draw flat top discrete "step" # over the index space for each datum. # if self._step_mode: + # self.disable_cache() + # flip_cache = True # x_out, y_out = step_path_arrays_from_1d( # x_out, # y_out, # ) - # # self.disable_cache() - # # flip_cache = True # # TODO: numba this bish # profiler('generated step arrays') @@ -460,7 +498,7 @@ class FastAppendCurve(pg.GraphicsObject): self._in_ds = False - elif should_ds and px_width and uppx: + elif should_ds and uppx and px_width > 1: x_out, y_out = self.downsample( x_out, y_out, @@ -477,11 +515,9 @@ class FastAppendCurve(pg.GraphicsObject): finiteCheck=False, path=self.path, ) + self.prepareGeometryChange() profiler( - 'generated fresh path\n' - f'should_redraw: {should_redraw}\n' - f'should_ds: {should_ds}\n' - f'new_sample_rate: {new_sample_rate}\n' + f'generated fresh path. (should_redraw: {should_redraw} should_ds: {should_ds} new_sample_rate: {new_sample_rate})' ) # profiler(f'DRAW PATH IN VIEW -> {self._name}') @@ -514,26 +550,29 @@ class FastAppendCurve(pg.GraphicsObject): elif ( append_length > 0 + and do_append + and not should_redraw # and not view_range ): - new_x = x[-append_length - 2:-1] - new_y = y[-append_length - 2:-1] + print(f'{self._name} append len: {append_length}') + new_x = x[-append_length - 2:slice_to_head] + new_y = y[-append_length - 2:slice_to_head] profiler('sliced append path') # if self._step_mode: - # new_x, new_y = step_path_arrays_from_1d( - # new_x, - # new_y, - # ) - # # [1:] since we don't need the vertical line normally at - # # the beginning of the step curve taking the first (x, - # # y) poing down to the x-axis **because** this is an - # # appended path graphic. - # new_x = new_x[1:] - # new_y = new_y[1:] + # # new_x, new_y = step_path_arrays_from_1d( + # # new_x, + # # new_y, + # # ) + # # # [1:] since we don't need the vertical line normally at + # # # the beginning of the step curve taking the first (x, + # # # y) poing down to the x-axis **because** this is an + # # # appended path graphic. + # # new_x = new_x[1:] + # # new_y = new_y[1:] - # # self.disable_cache() - # # flip_cache = True + # self.disable_cache() + # flip_cache = True # profiler('generated step data') @@ -563,7 +602,7 @@ class FastAppendCurve(pg.GraphicsObject): # an attempt at trying to make append-updates faster.. if self.fast_path is None: self.fast_path = append_path - self.fast_path.reserve(int(6e3)) + # self.fast_path.reserve(int(6e3)) else: self.fast_path.connectPath(append_path) size = self.fast_path.capacity() @@ -596,19 +635,20 @@ class FastAppendCurve(pg.GraphicsObject): # self.disable_cache() # flip_cache = True - self.draw_last(x, y) - profiler('draw last segment') + if draw_last: + self.draw_last(x, y) + profiler('draw last segment') + + + # if flip_cache: + # # # XXX: seems to be needed to avoid artifacts (see above). + # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) # trigger redraw of path # do update before reverting to cache mode - # self.prepareGeometryChange() self.update() profiler('.update()') - # if flip_cache: - # # XXX: seems to be needed to avoid artifacts (see above). - # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - def draw_last( self, x: np.ndarray, @@ -624,10 +664,14 @@ class FastAppendCurve(pg.GraphicsObject): self._last_line = QLineF( x_last - 0.5, 0, x_last + 0.5, 0, + # x_last, 0, + # x_last, 0, ) self._last_step_rect = QRectF( x_last - 0.5, 0, x_last + 0.5, y_last + # x_last, 0, + # x_last, y_last ) # print( # f"path br: {self.path.boundingRect()}", @@ -640,6 +684,8 @@ class FastAppendCurve(pg.GraphicsObject): x_last, y_last ) + self.update() + # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. def getData(self): @@ -685,7 +731,7 @@ class FastAppendCurve(pg.GraphicsObject): # XXX: pretty annoying but, without this there's little # artefacts on the append updates to the curve... self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - self.prepareGeometryChange() + # self.prepareGeometryChange() def boundingRect(self): ''' @@ -705,6 +751,7 @@ class FastAppendCurve(pg.GraphicsObject): ''' hb = self.path.controlPointRect() + # hb = self.path.boundingRect() hb_size = hb.size() fp = self.fast_path @@ -713,17 +760,47 @@ class FastAppendCurve(pg.GraphicsObject): hb_size = fhb.size() + hb_size # print(f'hb_size: {hb_size}') + # if self._last_step_rect: + # hb_size += self._last_step_rect.size() + + # if self._line: + # br = self._last_step_rect.bottomRight() + + # tl = QPointF( + # # self._vr[0], + # # hb.topLeft().y(), + # # 0, + # # hb_size.height() + 1 + # ) + + # if self._last_step_rect: + # br = self._last_step_rect.bottomRight() + + # else: + # hb_size += QSizeF(1, 1) w = hb_size.width() + 1 h = hb_size.height() + 1 + # br = QPointF( + # self._vr[-1], + # # tl.x() + w, + # tl.y() + h, + # ) + br = QRectF( # top left + # hb.topLeft() + # tl, QPointF(hb.topLeft()), + # br, # total size + # QSizeF(hb_size) + # hb_size, QSizeF(w, h) ) + self._br = br # print(f'bounding rect: {br}') return br @@ -740,6 +817,7 @@ class FastAppendCurve(pg.GraphicsObject): disabled=not pg_profile_enabled(), gt=ms_slower_then, ) + self.prepareGeometryChange() if ( self._step_mode diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index b150a2d1..03d95a35 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -34,6 +34,13 @@ import numpy as np from numpy.lib import recfunctions as rfn import pyqtgraph as pg from PyQt5.QtGui import QPainterPath +from PyQt5.QtCore import ( + # Qt, + QLineF, + # QSizeF, + QRectF, + # QPointF, +) from ..data._sharedmem import ( ShmArray, @@ -465,7 +472,7 @@ class Flow(msgspec.Struct): # , frozen=True): self.gy[ ishm_first:iflat_first ] = rfn.structured_to_unstructured( - self.shm.array[fields][:iflat_first] + self.shm._array[fields][ishm_first:iflat_first] ) self._iflat_first = ishm_first @@ -516,6 +523,8 @@ class Flow(msgspec.Struct): # , frozen=True): y_iv=y_iv, view_range=(ivl, ivr), # hack profiler=profiler, + # should_redraw=False, + # do_append=False, ) curve.show() profiler('updated ds curve') @@ -578,21 +587,36 @@ class Flow(msgspec.Struct): # , frozen=True): # graphics.draw_last(last) else: - + # ``FastAppendCurve`` case: array_key = array_key or self.name - # ``FastAppendCurve`` case: if graphics._step_mode and self.gy is None: + self._iflat_first = self.shm._first.value # create a flattened view onto the OHLC array # which can be read as a line-style format shm = self.shm # fields = ['index', array_key] - i = shm._array['index'] - out = shm._array[array_key] + i = shm._array['index'].copy() + out = shm._array[array_key].copy() - self.gx, self.gy = step_path_arrays_from_1d(i, out) + self.gx = np.broadcast_to( + i[:, None], + (i.size, 2), + ) + np.array([-0.5, 0.5]) + + + # self.gy = np.broadcast_to( + # out[:, None], (out.size, 2), + # ) + self.gy = np.empty((len(out), 2), dtype=out.dtype) + self.gy[:] = out[:, np.newaxis] + + # start y at origin level + self.gy[0, 0] = 0 + + # self.gx, self.gy = step_path_arrays_from_1d(i, out) # flat = self.gy = self.shm.unstruct_view(fields) # self.gy = self.shm.ustruct(fields) @@ -635,17 +659,29 @@ class Flow(msgspec.Struct): # , frozen=True): self.shm._first.value ) + il = max(iflat - 1, 0) + # check for shm prepend updates since last read. if iflat_first != ishm_first: - # write newly prepended data to flattened copy - _gx, self.gy[ - ishm_first:iflat_first - ] = step_path_arrays_from_1d( - self.shm.array['index'][:iflat_first], - self.shm.array[array_key][:iflat_first], + print(f'prepend {array_key}') + + i_prepend = self.shm._array['index'][ishm_first:iflat_first] + y_prepend = self.shm._array[array_key][ishm_first:iflat_first] + + y2_prepend = np.broadcast_to( + y_prepend[:, None], (y_prepend.size, 2), ) + + # write newly prepended data to flattened copy + self.gy[ishm_first:iflat_first] = y2_prepend + # ] = step_path_arrays_from_1d( + # ] = step_path_arrays_from_1d( + # i_prepend, + # y_prepend, + # ) self._iflat_first = ishm_first + # # flat = self.gy = self.shm.unstruct_view(fields) # self.gy = self.shm.ustruct(fields) # # self._iflat_last = self.shm._last.value @@ -654,40 +690,112 @@ class Flow(msgspec.Struct): # , frozen=True): # # do an update for the most recent prepend # # index # iflat = ishm_first - if iflat != ishm_last: - _x, to_update = step_path_arrays_from_1d( - self.shm._array[iflat:ishm_last]['index'], - self.shm._array[iflat:ishm_last][array_key], + append_diff = ishm_last - iflat + # if iflat != ishm_last: + if append_diff: + + # slice up to the last datum since last index/append update + new_x = self.shm._array[il:ishm_last]['index']#.copy() + new_y = self.shm._array[il:ishm_last][array_key]#.copy() + + # _x, to_update = step_path_arrays_from_1d(new_x, new_y) + + # new_x2 = = np.broadcast_to( + # new_x2[:, None], + # (new_x2.size, 2), + # ) + np.array([-0.5, 0.5]) + + new_y2 = np.broadcast_to( + new_y[:, None], (new_y.size, 2), ) + # new_y2 = np.empty((len(new_y), 2), dtype=new_y.dtype) + # new_y2[:] = new_y[:, np.newaxis] + + # import pdbpp + # pdbpp.set_trace() + + # print( + # f'updating step curve {to_update}\n' + # f'last array val: {new_x}, {new_y}' + # ) # to_update = rfn.structured_to_unstructured( # self.shm._array[iflat:ishm_last][fields] # ) - # import pdbpp - # pdbpp.set_trace() - self.gy[iflat:ishm_last-1] = to_update - self.gy[-1] = 0 - print(f'updating step curve {to_update}') + # if not to_update.any(): + # if new_y.any() and not to_update.any(): + # import pdbpp + # pdbpp.set_trace() + + # print(f'{array_key} new values new_x:{new_x}, new_y:{new_y}') + # head, last = to_update[:-1], to_update[-1] + self.gy[il:ishm_last] = new_y2 + + gy = self.gy[il:ishm_last] + + # self.gy[-1] = to_update[-1] profiler('updated step curve data') - # slice out up-to-last step contents - x_step = self.gx[ishm_first:ishm_last] - x = x_step.reshape(-1) - y_step = self.gy[ishm_first:ishm_last] - y = y_step.reshape(-1) - profiler('sliced step data') + # print( + # f'append size: {append_diff}\n' + # f'new_x: {new_x}\n' + # f'new_y: {new_y}\n' + # f'new_y2: {new_y2}\n' + # f'new gy: {gy}\n' + # ) - # update local last-index tracking - self._iflat_last = ishm_last + # update local last-index tracking + self._iflat_last = ishm_last + + # ( + # iflat_first, + # iflat, + # ishm_last, + # ishm_first, + # ) = ( + # self._iflat_first, + # self._iflat_last, + # self.shm._last.value, + # self.shm._first.value + # ) + # graphics.draw_last(last['index'], last[array_key]) + + # slice out up-to-last step contents + x_step = self.gx[ishm_first:ishm_last+2] + # x_step[-1] = last['index'] + # x_step[-1] = last['index'] + # to 1d + x = x_step.reshape(-1) + + y_step = self.gy[ishm_first:ishm_last+2] + lasts = self.shm.array[['index', array_key]] + last = lasts[array_key][-1] + y_step[-1] = last + # to 1d + y = y_step.reshape(-1) + # y[-1] = 0 + + # s = 6 + # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') + + profiler('sliced step data') # reshape to 1d for graphics rendering # y = y_flat.reshape(-1) # x = x_flat.reshape(-1) # do all the same for only in-view data - y_iv = y_step[ivl:ivr].reshape(-1) - x_iv = x_step[ivl:ivr].reshape(-1) + ys_iv = y_step[ivl:ivr+1] + xs_iv = x_step[ivl:ivr+1] + y_iv = ys_iv.reshape(ys_iv.size) + x_iv = xs_iv.reshape(xs_iv.size) + # print( + # f'ys_iv : {ys_iv[-s:]}\n' + # f'y_iv: {y_iv[-s:]}\n' + # f'xs_iv: {xs_iv[-s:]}\n' + # f'x_iv: {x_iv[-s:]}\n' + # ) # y_iv = y_iv_flat.reshape(-1) # x_iv = x_iv_flat.reshape(-1) profiler('flattened ustruct in-view OHLC data') @@ -696,7 +804,49 @@ class Flow(msgspec.Struct): # , frozen=True): # x, y = ohlc_flatten(array) # x_iv, y_iv = ohlc_flatten(in_view) # profiler('flattened OHLC data') - graphics.reset_cache() + + x_last = array['index'][-1] + y_last = array[array_key][-1] + graphics._last_line = QLineF( + x_last - 0.5, 0, + x_last + 0.5, 0, + # x_last, 0, + # x_last, 0, + ) + graphics._last_step_rect = QRectF( + x_last - 0.5, 0, + x_last + 0.5, y_last, + # x_last, 0, + # x_last, y_last + ) + # graphics.update() + + graphics.update_from_array( + x=x, + y=y, + + x_iv=x_iv, + y_iv=y_iv, + + view_range=(ivl, ivr) if use_vr else None, + + draw_last=False, + slice_to_head=-2, + + should_redraw=bool(append_diff), + # do_append=False, + + **kwargs + ) + # graphics.reset_cache() + # print( + # f"path br: {graphics.path.boundingRect()}\n", + # # f"fast path br: {graphics.fast_path.boundingRect()}", + # f"last rect br: {graphics._last_step_rect}\n", + # f"full br: {graphics._br}\n", + # ) + + # graphics.boundingRect() else: x = array['index'] @@ -704,17 +854,20 @@ class Flow(msgspec.Struct): # , frozen=True): x_iv = in_view['index'] y_iv = in_view[array_key] - graphics.update_from_array( - x=x, - y=y, + # graphics.draw_last(x, y) + profiler('draw last segment {array_key}') - x_iv=x_iv, - y_iv=y_iv, + graphics.update_from_array( + x=x, + y=y, - view_range=(ivl, ivr) if use_vr else None, + x_iv=x_iv, + y_iv=y_iv, - **kwargs - ) + view_range=(ivl, ivr) if use_vr else None, + + **kwargs + ) return graphics From 186658ab090b646e05ba91ebb051f5e5850a75da Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 08:52:34 -0400 Subject: [PATCH 027/113] Drop uppx guard around downsamples on interaction Since downsampling with the more correct version of m4 (uppx driven windows sizing) is super fast now we don't need to avoid downsampling on low uppx values. Further all graphics objects now support in-view slicing so make sure to use it on interaction updates. Pass in the view profiler to update method calls for more detailed measuring. Even moar, - Add a manual call to `.maybe_downsample_graphics()` inside the mouse wheel event handler since it seems that sometimes trailing events get lost from the `.sigRangeChangedManually` signal which can result in "non-downsampled-enough" graphics on chart given the scroll amount; this manual call seems to entirely fix this? - drop "max zoom" guard since internals now support (near) infinite scroll out to graphics becoming a single pixel column line XD - add back in commented xrange signal connect code for easy testing to verify against range updates not happening without it --- piker/ui/_interaction.py | 71 +++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 943f3370..a2c99e38 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -34,10 +34,9 @@ import trio from ..log import get_logger from .._profile import pg_profile_enabled, ms_slower_then -from ._style import _min_points_to_show +# from ._style import _min_points_to_show from ._editors import SelectRect from . import _event -from ._ohlc import BarItems log = get_logger(__name__) @@ -485,11 +484,11 @@ class ChartView(ViewBox): # don't zoom more then the min points setting l, lbar, rbar, r = chart.bars_range() - vl = r - l + # vl = r - l - if ev.delta() > 0 and vl <= _min_points_to_show: - log.debug("Max zoom bruh...") - return + # if ev.delta() > 0 and vl <= _min_points_to_show: + # log.debug("Max zoom bruh...") + # return # if ( # ev.delta() < 0 @@ -570,6 +569,17 @@ class ChartView(ViewBox): self._resetTarget() self.scaleBy(s, focal) + + # XXX: without this is seems as though sometimes + # when zooming in from far out (and maybe vice versa?) + # the signal isn't being fired enough since if you pan + # just after you'll see further downsampling code run + # (pretty noticeable on the OHLC ds curve) but with this + # that never seems to happen? Only question is how much this + # "double work" is causing latency when these missing event + # fires don't happen? + self.maybe_downsample_graphics() + self.sigRangeChangedManually.emit(mask) # self._ic.set() @@ -736,7 +746,7 @@ class ChartView(ViewBox): # flag to prevent triggering sibling charts from the same linked # set from recursion errors. - autoscale_linked_plots: bool = True, + autoscale_linked_plots: bool = False, name: Optional[str] = None, # autoscale_overlays: bool = False, @@ -804,7 +814,7 @@ class ChartView(ViewBox): # for chart in plots: # if chart and not chart._static_yrange: # chart.cv._set_yrange( - # bars_range=br, + # # bars_range=br, # autoscale_linked_plots=False, # ) # profiler('autoscaled linked plots') @@ -858,9 +868,6 @@ class ChartView(ViewBox): # splitter(s) resizing src_vb.sigResized.connect(self._set_yrange) - # mouse wheel doesn't emit XRangeChanged - src_vb.sigRangeChangedManually.connect(self._set_yrange) - # TODO: a smarter way to avoid calling this needlessly? # 2 things i can think of: # - register downsample-able graphics specially and only @@ -871,9 +878,15 @@ class ChartView(ViewBox): self.maybe_downsample_graphics ) - def disable_auto_yrange( - self, - ) -> None: + # mouse wheel doesn't emit XRangeChanged + src_vb.sigRangeChangedManually.connect(self._set_yrange) + + # src_vb.sigXRangeChanged.connect(self._set_yrange) + # src_vb.sigXRangeChanged.connect( + # self.maybe_downsample_graphics + # ) + + def disable_auto_yrange(self) -> None: self.sigResized.disconnect( self._set_yrange, @@ -885,6 +898,11 @@ class ChartView(ViewBox): self._set_yrange, ) + # self.sigXRangeChanged.disconnect(self._set_yrange) + # self.sigXRangeChanged.disconnect( + # self.maybe_downsample_graphics + # ) + def x_uppx(self) -> float: ''' Return the "number of x units" within a single @@ -905,13 +923,6 @@ class ChartView(ViewBox): def maybe_downsample_graphics(self): - uppx = self.x_uppx() - # if not ( - # # we probably want to drop this once we are "drawing in - # # view" for downsampled flows.. - # uppx and uppx > 6 - # and self._ic is not None - # ): profiler = pg.debug.Profiler( msg=f'ChartView.maybe_downsample_graphics() for {self.name}', disabled=not pg_profile_enabled(), @@ -922,7 +933,7 @@ class ChartView(ViewBox): # the profiler in the delegated method calls. delayed=False, # gt=3, - # gt=ms_slower_then, + gt=ms_slower_then, ) # TODO: a faster single-loop-iterator way of doing this XD @@ -940,12 +951,6 @@ class ChartView(ViewBox): ): continue - graphics = flow.graphics - - # use_vr = False - # if isinstance(graphics, BarItems): - # use_vr = True - # pass in no array which will read and render from the last # passed array (normally provided by the display loop.) chart.update_graphics_from_flow( @@ -953,17 +958,9 @@ class ChartView(ViewBox): use_vr=True, # gets passed down into graphics obj - # profiler=profiler, + profiler=profiler, ) profiler(f'range change updated {chart_name}:{name}') profiler.finish() - # else: - # # don't bother updating since we're zoomed out bigly and - # # in a pan-interaction, in which case we shouldn't be - # # doing view-range based rendering (at least not yet). - # # print(f'{uppx} exiting early!') - # profiler(f'dowsampling skipped - not in uppx range {uppx} <= 16') - - # profiler.finish() From b12921678b286e0d30ef261e721d11be6d04b134 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 09:27:04 -0400 Subject: [PATCH 028/113] Drop step routine import --- piker/ui/_flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 03d95a35..d2b1fa90 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -53,7 +53,7 @@ from ._ohlc import ( ) from ._curve import ( FastAppendCurve, - step_path_arrays_from_1d, + # step_path_arrays_from_1d, ) from ._compression import ( # ohlc_flatten, From 859eaffa2950873559ff39e3bc06ec5b87231280 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 09:27:24 -0400 Subject: [PATCH 029/113] Drop vwap fsp for now; causes hangs.. --- piker/ui/_fsp.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 9aa10fb3..5ed85d9b 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -75,6 +75,7 @@ def update_fsp_chart( flow, graphics_name: str, array_key: Optional[str], + **kwargs, ) -> None: @@ -96,6 +97,7 @@ def update_fsp_chart( chart.update_graphics_from_flow( graphics_name, array_key=array_key or graphics_name, + **kwargs, ) # XXX: re: ``array_key``: fsp func names must be unique meaning we @@ -884,10 +886,10 @@ async def open_vlm_displays( # built-in vlm fsps for target, conf in { - tina_vwap: { - 'overlay': 'ohlc', # overlays with OHLCV (main) chart - 'anchor': 'session', - }, + # tina_vwap: { + # 'overlay': 'ohlc', # overlays with OHLCV (main) chart + # 'anchor': 'session', + # }, }.items(): started = await admin.open_fsp_chart( target, From 2b6041465ccf05c36c408a4ecbc8db7c49678431 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 09:27:38 -0400 Subject: [PATCH 030/113] Startup up with 3k bars --- piker/ui/_chart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index cecbbff5..8a95327f 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -856,7 +856,7 @@ class ChartPlotWidget(pg.PlotWidget): def default_view( self, - bars_from_y: int = 5000, + bars_from_y: int = 3000, ) -> None: ''' From 0770a39125eab777b206619636716074f6fb2aa2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 09:28:09 -0400 Subject: [PATCH 031/113] Only do curve appends on low uppx levels --- piker/ui/_display.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index b82d1253..6c8bdddf 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -331,7 +331,7 @@ def graphics_update_cycle( vars = ds.vars tick_margin = vars['tick_margin'] - update_uppx = 6 + update_uppx = 16 for sym, quote in ds.quotes.items(): @@ -392,7 +392,8 @@ def graphics_update_cycle( if ( ( - xpx < update_uppx or i_diff > 0 + xpx < update_uppx + or i_diff > 0 and liv ) or trigger_all @@ -401,7 +402,6 @@ def graphics_update_cycle( # once the $vlm is up? vlm_chart.update_graphics_from_flow( 'volume', - # UGGGh, see ``maxmin()`` impl in `._fsp` for # the overlayed plotitems... we need a better # bay to invoke a maxmin per overlay.. @@ -435,6 +435,7 @@ def graphics_update_cycle( flow, curve_name, array_key=curve_name, + do_append=xpx < update_uppx, ) # is this even doing anything? # (pretty sure it's the real-time @@ -496,6 +497,7 @@ def graphics_update_cycle( ): chart.update_graphics_from_flow( chart.name, + do_append=xpx < update_uppx, ) # iterate in FIFO order per tick-frame From 0744dd041517054d72ce9ad35da1d6fedc982091 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 27 Apr 2022 17:18:11 -0400 Subject: [PATCH 032/113] Up the display throttle rate to 22Hz --- piker/ui/_display.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 6c8bdddf..bbc708fd 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -63,7 +63,7 @@ from ..log import get_logger log = get_logger(__name__) # TODO: load this from a config.toml! -_quote_throttle_rate: int = 12 # Hz +_quote_throttle_rate: int = 22 # Hz # a working tick-type-classes template From 7a3437348ddb319322fef20913c1090d1567ac2d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 27 Apr 2022 17:19:08 -0400 Subject: [PATCH 033/113] An absolute uppx diff of >= 1 seems more then fine --- piker/ui/_curve.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 60353f08..66862086 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -429,10 +429,7 @@ class FastAppendCurve(pg.GraphicsObject): if ( # std m4 downsample conditions px_width - and uppx_diff >= 1 - or uppx_diff <= -1 - or self._step_mode and abs(uppx_diff) >= 2 - + and abs(uppx_diff) >= 1 ): log.info( f'{self._name} sampler change: {self._last_uppx} -> {uppx}' From 36a10155bc7e3cc0e5a9416fb0064db0e2d23448 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 29 Apr 2022 11:24:21 -0400 Subject: [PATCH 034/113] Add profiler passthrough type annot, comments about appends vs. uppx --- piker/ui/_flows.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index d2b1fa90..4901a6e7 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -320,7 +320,7 @@ class Flow(msgspec.Struct): # , frozen=True): render: bool = True, array_key: Optional[str] = None, - profiler=None, + profiler: Optional[pg.debug.Profiler] = None, **kwargs, @@ -524,7 +524,10 @@ class Flow(msgspec.Struct): # , frozen=True): view_range=(ivl, ivr), # hack profiler=profiler, # should_redraw=False, - # do_append=False, + + # NOTE: already passed through by display loop? + # do_append=uppx < 16, + **kwargs, ) curve.show() profiler('updated ds curve') @@ -589,6 +592,7 @@ class Flow(msgspec.Struct): # , frozen=True): else: # ``FastAppendCurve`` case: array_key = array_key or self.name + uppx = graphics.x_uppx() if graphics._step_mode and self.gy is None: self._iflat_first = self.shm._first.value @@ -834,7 +838,9 @@ class Flow(msgspec.Struct): # , frozen=True): slice_to_head=-2, should_redraw=bool(append_diff), - # do_append=False, + + # NOTE: already passed through by display loop? + # do_append=uppx < 16, **kwargs ) @@ -866,6 +872,8 @@ class Flow(msgspec.Struct): # , frozen=True): view_range=(ivl, ivr) if use_vr else None, + # NOTE: already passed through by display loop? + # do_append=uppx < 16, **kwargs ) From e163a7e3369b5dbcc905a459bbcc40bb7c9fe450 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 29 Apr 2022 11:27:18 -0400 Subject: [PATCH 035/113] Drop `bar_wap` curve for now, seems to also be causing hangs?! --- piker/ui/_display.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index bbc708fd..29b4c6d4 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -697,15 +697,17 @@ async def display_symbol_data( # plot historical vwap if available wap_in_history = False - if brokermod._show_wap_in_history: + # XXX: FOR SOME REASON THIS IS CAUSING HANGZ!?! + # if brokermod._show_wap_in_history: - if 'bar_wap' in bars.dtype.fields: - wap_in_history = True - chart.draw_curve( - name='bar_wap', - data=bars, - add_label=False, - ) + # if 'bar_wap' in bars.dtype.fields: + # wap_in_history = True + # chart.draw_curve( + # name='bar_wap', + # shm=ohlcv, + # color='default_light', + # add_label=False, + # ) # size view to data once at outset chart.cv._set_yrange() From fb38265199e609f42925263de24624fe64f87073 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 1 May 2022 19:13:21 -0400 Subject: [PATCH 036/113] Clean out legacy code from `Flow.update_graphics()` --- piker/ui/_flows.py | 125 ++++----------------------------------------- 1 file changed, 10 insertions(+), 115 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 4901a6e7..707be683 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -377,8 +377,6 @@ class Flow(msgspec.Struct): # , frozen=True): ) curve = FastAppendCurve( - # y=y, - # x=x, name='OHLC', color=graphics._color, ) @@ -610,7 +608,6 @@ class Flow(msgspec.Struct): # , frozen=True): (i.size, 2), ) + np.array([-0.5, 0.5]) - # self.gy = np.broadcast_to( # out[:, None], (out.size, 2), # ) @@ -620,36 +617,6 @@ class Flow(msgspec.Struct): # , frozen=True): # start y at origin level self.gy[0, 0] = 0 - # self.gx, self.gy = step_path_arrays_from_1d(i, out) - - # flat = self.gy = self.shm.unstruct_view(fields) - # self.gy = self.shm.ustruct(fields) - # first = self._iflat_first = self.shm._first.value - # last = self._iflat_last = self.shm._last.value - - # # write pushed data to flattened copy - # self.gy[first:last] = rfn.structured_to_unstructured( - # self.shm.array[fields] - # ) - - # # generate an flat-interpolated x-domain - # self.gx = ( - # np.broadcast_to( - # shm._array['index'][:, None], - # ( - # shm._array.size, - # # 4, # only ohlc - # self.gy.shape[1], - # ), - # ) + np.array([-0.5, 0, 0, 0.5]) - # ) - # assert self.gy.any() - - # print(f'unstruct diff: {time.time() - start}') - # profiler('read unstr view bars to line') - # start = self.gy._first.value - # update flatted ohlc copy - if graphics._step_mode: ( iflat_first, @@ -670,8 +637,11 @@ class Flow(msgspec.Struct): # , frozen=True): print(f'prepend {array_key}') - i_prepend = self.shm._array['index'][ishm_first:iflat_first] - y_prepend = self.shm._array[array_key][ishm_first:iflat_first] + # i_prepend = self.shm._array['index'][ + # ishm_first:iflat_first] + y_prepend = self.shm._array[array_key][ + ishm_first:iflat_first + ] y2_prepend = np.broadcast_to( y_prepend[:, None], (y_prepend.size, 2), @@ -679,66 +649,19 @@ class Flow(msgspec.Struct): # , frozen=True): # write newly prepended data to flattened copy self.gy[ishm_first:iflat_first] = y2_prepend - # ] = step_path_arrays_from_1d( - # ] = step_path_arrays_from_1d( - # i_prepend, - # y_prepend, - # ) self._iflat_first = ishm_first - # # flat = self.gy = self.shm.unstruct_view(fields) - # self.gy = self.shm.ustruct(fields) - # # self._iflat_last = self.shm._last.value - - # # self._iflat_first = self.shm._first.value - # # do an update for the most recent prepend - # # index - # iflat = ishm_first append_diff = ishm_last - iflat - # if iflat != ishm_last: if append_diff: # slice up to the last datum since last index/append update - new_x = self.shm._array[il:ishm_last]['index']#.copy() - new_y = self.shm._array[il:ishm_last][array_key]#.copy() - - # _x, to_update = step_path_arrays_from_1d(new_x, new_y) - - # new_x2 = = np.broadcast_to( - # new_x2[:, None], - # (new_x2.size, 2), - # ) + np.array([-0.5, 0.5]) + # new_x = self.shm._array[il:ishm_last]['index'] + new_y = self.shm._array[il:ishm_last][array_key] new_y2 = np.broadcast_to( new_y[:, None], (new_y.size, 2), ) - # new_y2 = np.empty((len(new_y), 2), dtype=new_y.dtype) - # new_y2[:] = new_y[:, np.newaxis] - - # import pdbpp - # pdbpp.set_trace() - - # print( - # f'updating step curve {to_update}\n' - # f'last array val: {new_x}, {new_y}' - # ) - - # to_update = rfn.structured_to_unstructured( - # self.shm._array[iflat:ishm_last][fields] - # ) - - # if not to_update.any(): - # if new_y.any() and not to_update.any(): - # import pdbpp - # pdbpp.set_trace() - - # print(f'{array_key} new values new_x:{new_x}, new_y:{new_y}') - # head, last = to_update[:-1], to_update[-1] self.gy[il:ishm_last] = new_y2 - - gy = self.gy[il:ishm_last] - - # self.gy[-1] = to_update[-1] profiler('updated step curve data') # print( @@ -752,43 +675,23 @@ class Flow(msgspec.Struct): # , frozen=True): # update local last-index tracking self._iflat_last = ishm_last - # ( - # iflat_first, - # iflat, - # ishm_last, - # ishm_first, - # ) = ( - # self._iflat_first, - # self._iflat_last, - # self.shm._last.value, - # self.shm._first.value - # ) - # graphics.draw_last(last['index'], last[array_key]) - # slice out up-to-last step contents x_step = self.gx[ishm_first:ishm_last+2] - # x_step[-1] = last['index'] - # x_step[-1] = last['index'] - # to 1d + # shape to 1d x = x_step.reshape(-1) y_step = self.gy[ishm_first:ishm_last+2] lasts = self.shm.array[['index', array_key]] last = lasts[array_key][-1] y_step[-1] = last - # to 1d + # shape to 1d y = y_step.reshape(-1) - # y[-1] = 0 # s = 6 # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') profiler('sliced step data') - # reshape to 1d for graphics rendering - # y = y_flat.reshape(-1) - # x = x_flat.reshape(-1) - # do all the same for only in-view data ys_iv = y_step[ivl:ivr+1] xs_iv = x_step[ivl:ivr+1] @@ -800,8 +703,6 @@ class Flow(msgspec.Struct): # , frozen=True): # f'xs_iv: {xs_iv[-s:]}\n' # f'x_iv: {x_iv[-s:]}\n' # ) - # y_iv = y_iv_flat.reshape(-1) - # x_iv = x_iv_flat.reshape(-1) profiler('flattened ustruct in-view OHLC data') # legacy full-recompute-everytime method @@ -814,14 +715,10 @@ class Flow(msgspec.Struct): # , frozen=True): graphics._last_line = QLineF( x_last - 0.5, 0, x_last + 0.5, 0, - # x_last, 0, - # x_last, 0, ) graphics._last_step_rect = QRectF( x_last - 0.5, 0, x_last + 0.5, y_last, - # x_last, 0, - # x_last, y_last ) # graphics.update() @@ -852,8 +749,6 @@ class Flow(msgspec.Struct): # , frozen=True): # f"full br: {graphics._br}\n", # ) - # graphics.boundingRect() - else: x = array['index'] y = array[array_key] @@ -861,7 +756,7 @@ class Flow(msgspec.Struct): # , frozen=True): y_iv = in_view[array_key] # graphics.draw_last(x, y) - profiler('draw last segment {array_key}') + profiler(f'draw last segment {array_key}') graphics.update_from_array( x=x, From 1fcb9233b452ecde3e58be5928c384075425e778 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 2 May 2022 11:40:53 -0400 Subject: [PATCH 037/113] Add back mx/mn updates for L1-in-view, lost during rebase --- piker/ui/_display.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 29b4c6d4..aa7761db 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -366,7 +366,7 @@ def graphics_update_cycle( mx = mx_in_view + tick_margin mn = mn_in_view - tick_margin profiler('maxmin call') - liv = r > i_step # the last datum is in view + liv = r >= i_step # the last datum is in view # don't real-time "shift" the curve to the # left unless we get one of the following: @@ -374,7 +374,6 @@ def graphics_update_cycle( ( i_diff > 0 # no new sample step and xpx < 4 # chart is zoomed out very far - and r >= i_step # the last datum isn't in view and liv ) or trigger_all @@ -589,6 +588,7 @@ def graphics_update_cycle( main_vb._ic is None or not main_vb._ic.is_set() ): + # print(f'updating range due to mxmn') main_vb._set_yrange( # TODO: we should probably scale # the view margin based on the size @@ -599,7 +599,8 @@ def graphics_update_cycle( yrange=(mn, mx), ) - vars['last_mx'], vars['last_mn'] = mx, mn + # XXX: update this every draw cycle to make L1-always-in-view work. + vars['last_mx'], vars['last_mn'] = mx, mn # run synchronous update on all linked flows for curve_name, flow in chart._flows.items(): From 4f36743f6415dd4b232764f0ae393ca799b8802b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 9 May 2022 10:26:44 -0400 Subject: [PATCH 038/113] Only udpate prepended graphics when actually in view --- piker/data/_sampling.py | 19 +++++++++++++++---- piker/data/feed.py | 9 +++++++++ piker/fsp/_engine.py | 7 ++++++- piker/ui/_display.py | 14 +++++++++++++- piker/ui/_fsp.py | 8 ++++++-- 5 files changed, 49 insertions(+), 8 deletions(-) diff --git a/piker/data/_sampling.py b/piker/data/_sampling.py index 466ef0e7..10dc43f6 100644 --- a/piker/data/_sampling.py +++ b/piker/data/_sampling.py @@ -142,11 +142,17 @@ async def broadcast( shm: Optional[ShmArray] = None, ) -> None: - # broadcast the buffer index step to any subscribers for - # a given sample period. + ''' + Broadcast the given ``shm: ShmArray``'s buffer index step to any + subscribers for a given sample period. + + The sent msg will include the first and last index which slice into + the buffer's non-empty data. + + ''' subs = sampler.subscribers.get(delay_s, ()) - last = -1 + first = last = -1 if shm is None: periods = sampler.ohlcv_shms.keys() @@ -156,11 +162,16 @@ async def broadcast( if periods: lowest = min(periods) shm = sampler.ohlcv_shms[lowest][0] + first = shm._first.value last = shm._last.value for stream in subs: try: - await stream.send({'index': last}) + await stream.send({ + 'first': first, + 'last': last, + 'index': last, + }) except ( trio.BrokenResourceError, trio.ClosedResourceError diff --git a/piker/data/feed.py b/piker/data/feed.py index 605349e9..848fcc10 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -795,6 +795,15 @@ async def manage_history( # manually trigger step update to update charts/fsps # which need an incremental update. + # NOTE: the way this works is super duper + # un-intuitive right now: + # - the broadcaster fires a msg to the fsp subsystem. + # - fsp subsys then checks for a sample step diff and + # possibly recomputes prepended history. + # - the fsp then sends back to the parent actor + # (usually a chart showing graphics for said fsp) + # which tells the chart to conduct a manual full + # graphics loop cycle. for delay_s in sampler.subscribers: await broadcast(delay_s) diff --git a/piker/fsp/_engine.py b/piker/fsp/_engine.py index 0776c7a2..cf45c40e 100644 --- a/piker/fsp/_engine.py +++ b/piker/fsp/_engine.py @@ -369,7 +369,12 @@ async def cascade( # always trigger UI refresh after history update, # see ``piker.ui._fsp.FspAdmin.open_chain()`` and # ``piker.ui._display.trigger_update()``. - await client_stream.send('update') + await client_stream.send({ + 'fsp_update': { + 'key': dst_shm_token, + 'first': dst._first.value, + 'last': dst._last.value, + }}) return tracker, index def is_synced( diff --git a/piker/ui/_display.py b/piker/ui/_display.py index aa7761db..d0654b10 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -309,6 +309,7 @@ def graphics_update_cycle( ds: DisplayState, wap_in_history: bool = False, trigger_all: bool = False, # flag used by prepend history updates + prepend_update_index: Optional[int] = None, ) -> None: # TODO: eventually optimize this whole graphics stack with ``numba`` @@ -368,6 +369,17 @@ def graphics_update_cycle( profiler('maxmin call') liv = r >= i_step # the last datum is in view + if ( + prepend_update_index is not None + and lbar > prepend_update_index + ): + # on a history update (usually from the FSP subsys) + # if the segment of history that is being prepended + # isn't in view there is no reason to do a graphics + # update. + log.debug('Skipping prepend graphics cycle: frame not in view') + return + # don't real-time "shift" the curve to the # left unless we get one of the following: if ( @@ -639,7 +651,7 @@ async def display_symbol_data( ) # historical data fetch - brokermod = brokers.get_brokermod(provider) + # brokermod = brokers.get_brokermod(provider) # ohlc_status_done = sbar.open_status( # 'retreiving OHLC history.. ', diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 5ed85d9b..3d90f014 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -435,12 +435,16 @@ class FspAdmin: # wait for graceful shutdown signal async with stream.subscribe() as stream: async for msg in stream: - if msg == 'update': + info = msg.get('fsp_update') + if info: # if the chart isn't hidden try to update # the data on screen. if not self.linked.isHidden(): log.info(f'Re-syncing graphics for fsp: {ns_path}') - self.linked.graphics_cycle(trigger_all=True) + self.linked.graphics_cycle( + trigger_all=True, + prepend_update_index=info['first'], + ) else: log.info(f'recved unexpected fsp engine msg: {msg}') From 47cf4aa4f752a677114542eb1669a8268937826c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 10 May 2022 09:22:46 -0400 Subject: [PATCH 039/113] Error log brokerd msgs that have `.reqid == None` Relates to the bug discovered in #310, this should avoid out-of-order msgs which do not have a `.reqid` set to be error logged to console. Further, add `pformat()` to kraken logging of ems msging. --- piker/brokers/kraken.py | 10 +++++++++- piker/clearing/_ems.py | 26 ++++++++++++++++++++------ piker/ui/order_mode.py | 4 +++- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index 30e57b9e..670eed6f 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -21,6 +21,7 @@ Kraken backend. from contextlib import asynccontextmanager as acm from dataclasses import asdict, field from datetime import datetime +from pprint import pformat from typing import Any, Optional, AsyncIterator, Callable, Union import time @@ -569,7 +570,10 @@ async def handle_order_requests( order: BrokerdOrder async for request_msg in ems_order_stream: - log.info(f'Received order request {request_msg}') + log.info( + 'Received order request:\n' + f'{pformat(request_msg)}' + ) action = request_msg['action'] @@ -628,6 +632,7 @@ async def handle_order_requests( # update the internal pairing of oid to krakens # txid with the new txid that is returned on edit reqid = resp['result']['txid'] + # deliver ack that order has been submitted to broker routing await ems_order_stream.send( BrokerdOrderAck( @@ -788,7 +793,10 @@ async def trades_dialogue( # Get websocket token for authenticated data stream # Assert that a token was actually received. resp = await client.endpoint('GetWebSocketsToken', {}) + + # lol wtf is this.. assert resp['error'] == [] + token = resp['result']['token'] async with ( diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index a5d04f0c..e00676f2 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -561,7 +561,10 @@ async def translate_and_relay_brokerd_events( name = brokerd_msg['name'] - log.info(f'Received broker trade event:\n{pformat(brokerd_msg)}') + log.info( + f'Received broker trade event:\n' + f'{pformat(brokerd_msg)}' + ) if name == 'position': @@ -613,19 +616,28 @@ async def translate_and_relay_brokerd_events( # packed at submission since we already know it ahead of # time paper = brokerd_msg['broker_details'].get('paper_info') + ext = brokerd_msg['broker_details'].get('external') if paper: # paperboi keeps the ems id up front oid = paper['oid'] - else: + elif ext: # may be an order msg specified as "external" to the # piker ems flow (i.e. generated by some other # external broker backend client (like tws for ib) - ext = brokerd_msg['broker_details'].get('external') - if ext: - log.error(f"External trade event {ext}") + log.error(f"External trade event {ext}") continue + + else: + # something is out of order, we don't have an oid for + # this broker-side message. + log.error( + 'Unknown oid:{oid} for msg:\n' + f'{pformat(brokerd_msg)}' + 'Unable to relay message to client side!?' + ) + else: # check for existing live flow entry entry = book._ems_entries.get(oid) @@ -823,7 +835,9 @@ async def process_client_order_cmds( if reqid: # send cancel to brokerd immediately! - log.info("Submitting cancel for live order {reqid}") + log.info( + f'Submitting cancel for live order {reqid}' + ) await brokerd_order_stream.send(msg.dict()) diff --git a/piker/ui/order_mode.py b/piker/ui/order_mode.py index 3e230b71..a86fe816 100644 --- a/piker/ui/order_mode.py +++ b/piker/ui/order_mode.py @@ -873,7 +873,9 @@ async def process_trades_and_update_ui( mode.lines.remove_line(uuid=oid) # each clearing tick is responded individually - elif resp in ('broker_filled',): + elif resp in ( + 'broker_filled', + ): known_order = book._sent_orders.get(oid) if not known_order: From c455df7fa85a599ba0e6d604fc0dfbc5135f8831 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 13 May 2022 16:04:31 -0400 Subject: [PATCH 040/113] Drop legacy step path gen, always slice full data Mostly just dropping old commented code for "step mode" format generation. Always slice the tail part of the input data and move to the new `ms_threshold` in the `pg` profiler' --- piker/ui/_curve.py | 63 ++++++++-------------------------------------- 1 file changed, 10 insertions(+), 53 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 66862086..88917a0c 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -272,8 +272,6 @@ class FastAppendCurve(pg.GraphicsObject): x = (x + np.array([-0.5, 0, 0, 0.5])).flatten() y = y.flatten() - # presumably? - self._in_ds = True return x, y def update_from_array( @@ -305,7 +303,7 @@ class FastAppendCurve(pg.GraphicsObject): profiler = profiler or pg.debug.Profiler( msg=f'FastAppendCurve.update_from_array(): `{self._name}`', disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, ) flip_cache = False @@ -342,6 +340,10 @@ class FastAppendCurve(pg.GraphicsObject): showing_src_data = self._in_ds # should_redraw = False + # by default we only pull data up to the last (current) index + x_out_full = x_out = x[:slice_to_head] + y_out_full = y_out = y[:slice_to_head] + # if a view range is passed, plan to draw the # source ouput that's "in view" of the chart. if ( @@ -358,7 +360,7 @@ class FastAppendCurve(pg.GraphicsObject): vl, vr = view_range # last_ivr = self._x_iv_range - # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) + # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) zoom_or_append = False last_vr = self._vr @@ -414,16 +416,8 @@ class FastAppendCurve(pg.GraphicsObject): # self.disable_cache() # flip_cache = True - else: - # if ( - # not view_range - # or self._in_ds - # ): - # by default we only pull data up to the last (current) index - x_out, y_out = x[:slice_to_head], y[:slice_to_head] - - if prepend_length > 0: - should_redraw = True + if prepend_length > 0: + should_redraw = True # check for downsampling conditions if ( @@ -459,30 +453,10 @@ class FastAppendCurve(pg.GraphicsObject): or new_sample_rate or prepend_length > 0 ): - # if ( - # not view_range - # or self._in_ds - # ): - # # by default we only pull data up to the last (current) index - # x_out, y_out = x[:-1], y[:-1] - - # step mode: draw flat top discrete "step" - # over the index space for each datum. - # if self._step_mode: - # self.disable_cache() - # flip_cache = True - # x_out, y_out = step_path_arrays_from_1d( - # x_out, - # y_out, - # ) - - # # TODO: numba this bish - # profiler('generated step arrays') - if should_redraw: if self.path: - # print(f'CLEARING PATH {self._name}') self.path.clear() + profiler('cleared paths due to `should_redraw=True`') if self.fast_path: self.fast_path.clear() @@ -556,23 +530,6 @@ class FastAppendCurve(pg.GraphicsObject): new_y = y[-append_length - 2:slice_to_head] profiler('sliced append path') - # if self._step_mode: - # # new_x, new_y = step_path_arrays_from_1d( - # # new_x, - # # new_y, - # # ) - # # # [1:] since we don't need the vertical line normally at - # # # the beginning of the step curve taking the first (x, - # # # y) poing down to the x-axis **because** this is an - # # # appended path graphic. - # # new_x = new_x[1:] - # # new_y = new_y[1:] - - # self.disable_cache() - # flip_cache = True - - # profiler('generated step data') - profiler( f'diffed array input, append_length={append_length}' ) @@ -812,7 +769,7 @@ class FastAppendCurve(pg.GraphicsObject): profiler = pg.debug.Profiler( msg=f'FastAppendCurve.paint(): `{self._name}`', disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, ) self.prepareGeometryChange() From cfc4198837fb3ed96cb7002f509bb29316de4b5a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 13 May 2022 16:16:17 -0400 Subject: [PATCH 041/113] Use new profiler arg name, add more marks throughout flow update --- piker/ui/_flows.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 707be683..bf6fc3f3 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -46,7 +46,10 @@ from ..data._sharedmem import ( ShmArray, # open_shm_array, ) -from .._profile import pg_profile_enabled, ms_slower_then +from .._profile import ( + pg_profile_enabled, + ms_slower_then, +) from ._ohlc import ( BarItems, gen_qpath, @@ -331,11 +334,13 @@ class Flow(msgspec.Struct): # , frozen=True): ''' - profiler = profiler or pg.debug.Profiler( + # profiler = profiler or pg.debug.Profiler( + profiler = pg.debug.Profiler( msg=f'Flow.update_graphics() for {self.name}', disabled=not pg_profile_enabled(), - gt=ms_slower_then, - delayed=True, + # disabled=False, + ms_threshold=4, + # ms_threshold=ms_slower_then, ) # shm read and slice to view read = ( @@ -591,6 +596,7 @@ class Flow(msgspec.Struct): # , frozen=True): # ``FastAppendCurve`` case: array_key = array_key or self.name uppx = graphics.x_uppx() + profiler('read uppx') if graphics._step_mode and self.gy is None: self._iflat_first = self.shm._first.value @@ -616,6 +622,7 @@ class Flow(msgspec.Struct): # , frozen=True): # start y at origin level self.gy[0, 0] = 0 + profiler('generated step mode data') if graphics._step_mode: ( @@ -631,6 +638,7 @@ class Flow(msgspec.Struct): # , frozen=True): ) il = max(iflat - 1, 0) + profiler('read step mode incr update indices') # check for shm prepend updates since last read. if iflat_first != ishm_first: @@ -650,6 +658,7 @@ class Flow(msgspec.Struct): # , frozen=True): # write newly prepended data to flattened copy self.gy[ishm_first:iflat_first] = y2_prepend self._iflat_first = ishm_first + profiler('prepended step mode history') append_diff = ishm_last - iflat if append_diff: @@ -679,6 +688,7 @@ class Flow(msgspec.Struct): # , frozen=True): x_step = self.gx[ishm_first:ishm_last+2] # shape to 1d x = x_step.reshape(-1) + profiler('sliced step x') y_step = self.gy[ishm_first:ishm_last+2] lasts = self.shm.array[['index', array_key]] @@ -690,7 +700,7 @@ class Flow(msgspec.Struct): # , frozen=True): # s = 6 # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') - profiler('sliced step data') + profiler('sliced step y') # do all the same for only in-view data ys_iv = y_step[ivl:ivr+1] @@ -703,7 +713,7 @@ class Flow(msgspec.Struct): # , frozen=True): # f'xs_iv: {xs_iv[-s:]}\n' # f'x_iv: {x_iv[-s:]}\n' # ) - profiler('flattened ustruct in-view OHLC data') + profiler('sliced in view step data') # legacy full-recompute-everytime method # x, y = ohlc_flatten(array) @@ -738,9 +748,11 @@ class Flow(msgspec.Struct): # , frozen=True): # NOTE: already passed through by display loop? # do_append=uppx < 16, + profiler=profiler, **kwargs ) + profiler('updated step mode curve') # graphics.reset_cache() # print( # f"path br: {graphics.path.boundingRect()}\n", @@ -754,9 +766,9 @@ class Flow(msgspec.Struct): # , frozen=True): y = array[array_key] x_iv = in_view['index'] y_iv = in_view[array_key] + profiler('sliced input arrays') # graphics.draw_last(x, y) - profiler(f'draw last segment {array_key}') graphics.update_from_array( x=x, @@ -769,8 +781,10 @@ class Flow(msgspec.Struct): # , frozen=True): # NOTE: already passed through by display loop? # do_append=uppx < 16, + profiler=profiler, **kwargs ) + profiler(f'`graphics.update_from_array()` complete') return graphics From 5e602214bed506b90641e94dcb23c58bb9445c99 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 13 May 2022 16:22:49 -0400 Subject: [PATCH 042/113] Use new flag, add more marks through display loop --- piker/ui/_display.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index d0654b10..c551fc98 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -54,10 +54,10 @@ from ._forms import ( mk_order_pane_layout, ) from .order_mode import open_order_mode -# from .._profile import ( -# pg_profile_enabled, -# ms_slower_then, -# ) +from .._profile import ( + pg_profile_enabled, + ms_slower_then, +) from ..log import get_logger log = get_logger(__name__) @@ -319,9 +319,12 @@ def graphics_update_cycle( profiler = pg.debug.Profiler( msg=f'Graphics loop cycle for: `{chart.name}`', - disabled=True, # not pg_profile_enabled(), - gt=1/12 * 1e3, - # gt=ms_slower_then, + delayed=True, + # disabled=not pg_profile_enabled(), + disabled=True, + ms_threshold=ms_slower_then, + + # ms_threshold=1/12 * 1e3, ) # unpack multi-referenced components @@ -366,7 +369,9 @@ def graphics_update_cycle( l, lbar, rbar, r = brange mx = mx_in_view + tick_margin mn = mn_in_view - tick_margin - profiler('maxmin call') + + profiler('`ds.maxmin()` call') + liv = r >= i_step # the last datum is in view if ( @@ -394,6 +399,7 @@ def graphics_update_cycle( # pixel in a curve should show new data based on uppx # and then iff update curves and shift? chart.increment_view(steps=i_diff) + profiler('view incremented') if vlm_chart: # always update y-label @@ -425,6 +431,7 @@ def graphics_update_cycle( # connected to update accompanying overlay # graphics.. ) + profiler('`vlm_chart.update_graphics_from_flow()`') if ( mx_vlm_in_view != vars['last_mx_vlm'] @@ -433,6 +440,7 @@ def graphics_update_cycle( vlm_chart.view._set_yrange( yrange=yrange, ) + profiler('`vlm_chart.view._set_yrange()`') # print(f'mx vlm: {last_mx_vlm} -> {mx_vlm_in_view}') vars['last_mx_vlm'] = mx_vlm_in_view From 09e988ec3e1dce0201dbbca903c446b35d5b95af Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 13 May 2022 16:23:31 -0400 Subject: [PATCH 043/113] Use `ms_threshold` throughout remaining profilers --- piker/ui/_chart.py | 4 ++-- piker/ui/_interaction.py | 14 ++++++-------- piker/ui/_ohlc.py | 6 +++--- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 8a95327f..f6fc44ec 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1268,9 +1268,9 @@ class ChartPlotWidget(pg.PlotWidget): ''' profiler = pg.debug.Profiler( - msg=f'`{str(self)}.maxmin()` loop cycle for: `{self.name}`', + msg=f'`{str(self)}.maxmin(name={name})`: `{self.name}`', disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, delayed=True, ) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index a2c99e38..b2b46050 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -762,7 +762,7 @@ class ChartView(ViewBox): profiler = pg.debug.Profiler( msg=f'`ChartView._set_yrange()`: `{self.name}`', disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, delayed=True, ) set_range = True @@ -833,7 +833,7 @@ class ChartView(ViewBox): ylow, yhigh = yrange - profiler(f'maxmin(): {yrange}') + profiler(f'callback ._maxmin(): {yrange}') # view margins: stay within a % of the "true range" diff = yhigh - ylow @@ -932,8 +932,8 @@ class ChartView(ViewBox): # due to the way delaying works and garbage collection of # the profiler in the delegated method calls. delayed=False, - # gt=3, - gt=ms_slower_then, + ms_threshold=6, + # ms_threshold=ms_slower_then, ) # TODO: a faster single-loop-iterator way of doing this XD @@ -958,9 +958,7 @@ class ChartView(ViewBox): use_vr=True, # gets passed down into graphics obj - profiler=profiler, + # profiler=profiler, ) - profiler(f'range change updated {chart_name}:{name}') - - profiler.finish() + profiler(f'<{chart_name}>.update_graphics_from_flow({name})') diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 328d62b9..6199b9ea 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -170,7 +170,7 @@ def gen_qpath( profiler = pg.debug.Profiler( msg='gen_qpath ohlc', disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, ) x, y, c = path_arrays_from_ohlc( @@ -353,7 +353,7 @@ class BarItems(pg.GraphicsObject): # ''' # profiler = profiler or pg.debug.Profiler( # disabled=not pg_profile_enabled(), - # gt=ms_slower_then, + # ms_threshold=ms_slower_then, # delayed=True, # ) @@ -718,7 +718,7 @@ class BarItems(pg.GraphicsObject): profiler = pg.debug.Profiler( disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, ) # p.setCompositionMode(0) From 5d2660969354af953f99cdb52c8c319e774eaa6b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 13:36:08 -0400 Subject: [PATCH 044/113] Add "no-tsdb-found" history load length defaults --- piker/data/feed.py | 47 +++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index 848fcc10..d5e5d3b3 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -228,7 +228,7 @@ def diff_history( # the + 1 is because ``last_tsdb_dt`` is pulled from # the last row entry for the ``'time'`` field retreived # from the tsdb. - to_push = array[abs(s_diff)+1:] + to_push = array[abs(s_diff) + 1:] else: # pass back only the portion of the array that is @@ -251,6 +251,7 @@ async def start_backfill( last_tsdb_dt: Optional[datetime] = None, storage: Optional[Storage] = None, write_tsdb: bool = True, + tsdb_is_up: bool = False, task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED, @@ -266,8 +267,8 @@ async def start_backfill( # sample period step size in seconds step_size_s = ( - pendulum.from_timestamp(times[-1]) - - pendulum.from_timestamp(times[-2]) + pendulum.from_timestamp(times[-1]) + - pendulum.from_timestamp(times[-2]) ).seconds # "frame"'s worth of sample period steps in seconds @@ -292,25 +293,33 @@ async def start_backfill( # let caller unblock and deliver latest history frame task_status.started((shm, start_dt, end_dt, bf_done)) + # based on the sample step size, maybe load a certain amount history if last_tsdb_dt is None: - # maybe a better default (they don't seem to define epoch?!) - - # based on the sample step size load a certain amount - # history - if step_size_s == 1: - last_tsdb_dt = pendulum.now().subtract(days=2) - - elif step_size_s == 60: - last_tsdb_dt = pendulum.now().subtract(years=2) - - else: + if step_size_s not in (1, 60): raise ValueError( '`piker` only needs to support 1m and 1s sampling ' 'but ur api is trying to deliver a longer ' f'timeframe of {step_size_s} ' 'seconds.. so ye, dun ' - 'do dat bruh.' + 'do dat brudder.' ) + # when no tsdb "last datum" is provided, we just load + # some near-term history. + periods = { + 1: {'days': 1}, + 60: {'days': 14}, + } + + if tsdb_is_up: + # do a decently sized backfill and load it into storage. + periods = { + 1: {'days': 6}, + 60: {'years': 2}, + } + + kwargs = periods[step_size_s] + last_tsdb_dt = start_dt.subtract(**kwargs) + # configure async query throttling erlangs = config.get('erlangs', 1) rate = config.get('rate', 1) @@ -568,8 +577,8 @@ async def start_backfill( start_dt, end_dt, ) = await get_ohlc_frame( - input_end_dt=last_shm_prepend_dt, - iter_dts_gen=idts, + input_end_dt=last_shm_prepend_dt, + iter_dts_gen=idts, ) last_epoch = to_push['time'][-1] diff = start - last_epoch @@ -1003,7 +1012,7 @@ async def open_feed_bus( brokername: str, symbol: str, # normally expected to the broker-specific fqsn loglevel: str, - tick_throttle: Optional[float] = None, + tick_throttle: Optional[float] = None, start_stream: bool = True, ) -> None: @@ -1264,7 +1273,7 @@ async def install_brokerd_search( # a backend module? pause_period=getattr( brokermod, '_search_conf', {} - ).get('pause_period', 0.0616), + ).get('pause_period', 0.0616), ): yield From b609f46d26b80d48e6a8365e40c014e08293d707 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 13:44:56 -0400 Subject: [PATCH 045/113] Always delay interaction update profiling --- piker/ui/_interaction.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index b2b46050..90242c99 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -931,7 +931,6 @@ class ChartView(ViewBox): # ``.update_graphics_from_flow()`` nested profiling likely # due to the way delaying works and garbage collection of # the profiler in the delegated method calls. - delayed=False, ms_threshold=6, # ms_threshold=ms_slower_then, ) From f6909ae3952e91bc0e60e96dba67b0f7925e7694 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 14:12:09 -0400 Subject: [PATCH 046/113] Drop legacy step mode data formatter --- piker/ui/_curve.py | 74 ---------------------------------------------- 1 file changed, 74 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 88917a0c..d038f085 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -44,80 +44,6 @@ from ..log import get_logger log = get_logger(__name__) -# TODO: numba this instead.. -# def step_path_arrays_from_1d( -# x: np.ndarray, -# y: np.ndarray, -# include_endpoints: bool = True, - -# ) -> (np.ndarray, np.ndarray): -# ''' -# Generate a "step mode" curve aligned with OHLC style bars -# such that each segment spans each bar (aka "centered" style). - -# ''' -# # y_out = y.copy() -# # x_out = x.copy() - -# # x2 = np.empty( -# # # the data + 2 endpoints on either end for -# # # "termination of the path". -# # (len(x) + 1, 2), -# # # we want to align with OHLC or other sampling style -# # # bars likely so we need fractinal values -# # dtype=float, -# # ) - -# x2 = np.broadcast_to( -# x[:, None], -# ( -# x.size + 1, -# # 4, # only ohlc -# 2, -# ), -# ) + np.array([-0.5, 0.5]) - -# # x2[0] = x[0] - 0.5 -# # x2[1] = x[0] + 0.5 -# # x2[0, 0] = x[0] - 0.5 -# # x2[0, 1] = x[0] + 0.5 -# # x2[1:] = x[:, np.newaxis] + 0.5 -# # import pdbpp -# # pdbpp.set_trace() - -# # flatten to 1-d -# # x_out = x2.reshape(x2.size) -# # x_out = x2 - -# # we create a 1d with 2 extra indexes to -# # hold the start and (current) end value for the steps -# # on either end -# y2 = np.empty( -# (len(y) + 1, 2), -# dtype=y.dtype, -# ) -# y2[:] = y[:, np.newaxis] -# # y2[-1] = 0 - -# # y_out = y2 - -# # y_out = np.empty( -# # 2*len(y) + 2, -# # dtype=y.dtype -# # ) - -# # flatten and set 0 endpoints -# # y_out[1:-1] = y2.reshape(y2.size) -# # y_out[0] = 0 -# # y_out[-1] = 0 - -# if not include_endpoints: -# return x2[:-1], y2[:-1] - -# else: -# return x2, y2 - - _line_styles: dict[str, int] = { 'solid': Qt.PenStyle.SolidLine, 'dash': Qt.PenStyle.DashLine, From e8e26232ea87359c528cdef016a55933e5d54de1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 14:29:03 -0400 Subject: [PATCH 047/113] Drop `BarItems.update_from_array()`; moved into `Flow` --- piker/ui/_ohlc.py | 283 ---------------------------------------------- 1 file changed, 283 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 6199b9ea..c8da5ba5 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -325,289 +325,6 @@ class BarItems(pg.GraphicsObject): else: return 0 - # def update_from_array( - # self, - - # # full array input history - # ohlc: np.ndarray, - - # # pre-sliced array data that's "in view" - # ohlc_iv: np.ndarray, - - # view_range: Optional[tuple[int, int]] = None, - # profiler: Optional[pg.debug.Profiler] = None, - - # ) -> None: - # ''' - # Update the last datum's bar graphic from input data array. - - # This routine should be interface compatible with - # ``pg.PlotCurveItem.setData()``. Normally this method in - # ``pyqtgraph`` seems to update all the data passed to the - # graphics object, and then update/rerender, but here we're - # assuming the prior graphics havent changed (OHLC history rarely - # does) so this "should" be simpler and faster. - - # This routine should be made (transitively) as fast as possible. - - # ''' - # profiler = profiler or pg.debug.Profiler( - # disabled=not pg_profile_enabled(), - # ms_threshold=ms_slower_then, - # delayed=True, - # ) - - # # index = self.start_index - # istart, istop = self._xrange - # # ds_istart, ds_istop = self._ds_xrange - - # index = ohlc['index'] - # first_index, last_index = index[0], index[-1] - - # # length = len(ohlc) - # # prepend_length = istart - first_index - # # append_length = last_index - istop - - # # ds_prepend_length = ds_istart - first_index - # # ds_append_length = last_index - ds_istop - - # flip_cache = False - - # x_gt = 16 - # if self._ds_line: - # uppx = self._ds_line.x_uppx() - # else: - # uppx = 0 - - # should_line = self._in_ds - # if ( - # self._in_ds - # and uppx < x_gt - # ): - # should_line = False - - # elif ( - # not self._in_ds - # and uppx >= x_gt - # ): - # should_line = True - - # profiler('ds logic complete') - - # if should_line: - # # update the line graphic - # # x, y = self._ds_line_xy = ohlc_flatten(ohlc_iv) - # x, y = self._ds_line_xy = ohlc_flatten(ohlc) - # x_iv, y_iv = self._ds_line_xy = ohlc_flatten(ohlc_iv) - # profiler('flattening bars to line') - - # # TODO: we should be diffing the amount of new data which - # # needs to be downsampled. Ideally we actually are just - # # doing all the ds-ing in sibling actors so that the data - # # can just be read and rendered to graphics on events of our - # # choice. - # # diff = do_diff(ohlc, new_bit) - # curve = self._ds_line - # curve.update_from_array( - # x=x, - # y=y, - # x_iv=x_iv, - # y_iv=y_iv, - # view_range=None, # hack - # profiler=profiler, - # ) - # profiler('updated ds line') - - # if not self._in_ds: - # # hide bars and show line - # self.hide() - # # XXX: is this actually any faster? - # # self._pi.removeItem(self) - - # # TODO: a `.ui()` log level? - # log.info( - # f'downsampling to line graphic {self._name}' - # ) - - # # self._pi.addItem(curve) - # curve.show() - # curve.update() - # self._in_ds = True - - # # stop here since we don't need to update bars path any more - # # as we delegate to the downsample line with updates. - - # else: - # # we should be in bars mode - - # if self._in_ds: - # # flip back to bars graphics and hide the downsample line. - # log.info(f'showing bars graphic {self._name}') - - # curve = self._ds_line - # curve.hide() - # # self._pi.removeItem(curve) - - # # XXX: is this actually any faster? - # # self._pi.addItem(self) - # self.show() - # self._in_ds = False - - # # generate in_view path - # self.path = gen_qpath( - # ohlc_iv, - # 0, - # self.w, - # # path=self.path, - # ) - - # # TODO: to make the downsampling faster - # # - allow mapping only a range of lines thus only drawing as - # # many bars as exactly specified. - # # - move ohlc "flattening" to a shmarr - # # - maybe move all this embedded logic to a higher - # # level type? - - # # if prepend_length: - # # # new history was added and we need to render a new path - # # prepend_bars = ohlc[:prepend_length] - - # # if ds_prepend_length: - # # ds_prepend_bars = ohlc[:ds_prepend_length] - # # pre_x, pre_y = ohlc_flatten(ds_prepend_bars) - # # fx = np.concatenate((pre_x, fx)) - # # fy = np.concatenate((pre_y, fy)) - # # profiler('ds line prepend diff complete') - - # # if append_length: - # # # generate new graphics to match provided array - # # # path appending logic: - # # # we need to get the previous "current bar(s)" for the time step - # # # and convert it to a sub-path to append to the historical set - # # # new_bars = ohlc[istop - 1:istop + append_length - 1] - # # append_bars = ohlc[-append_length - 1:-1] - # # # print(f'ohlc bars to append size: {append_bars.size}\n') - - # # if ds_append_length: - # # ds_append_bars = ohlc[-ds_append_length - 1:-1] - # # post_x, post_y = ohlc_flatten(ds_append_bars) - # # print( - # # f'ds curve to append sizes: {(post_x.size, post_y.size)}' - # # ) - # # fx = np.concatenate((fx, post_x)) - # # fy = np.concatenate((fy, post_y)) - - # # profiler('ds line append diff complete') - - # profiler('array diffs complete') - - # # does this work? - # last = ohlc[-1] - # # fy[-1] = last['close'] - - # # # incremental update and cache line datums - # # self._ds_line_xy = fx, fy - - # # maybe downsample to line - # # ds = self.maybe_downsample() - # # if ds: - # # # if we downsample to a line don't bother with - # # # any more path generation / updates - # # self._ds_xrange = first_index, last_index - # # profiler('downsampled to line') - # # return - - # # print(in_view.size) - - # # if self.path: - # # self.path = path - # # self.path.reserve(path.capacity()) - # # self.path.swap(path) - - # # path updates - # # if prepend_length: - # # # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path - # # # y value not matching the first value from - # # # ohlc[prepend_length + 1] ??? - # # prepend_path = gen_qpath(prepend_bars, 0, self.w) - # # old_path = self.path - # # self.path = prepend_path - # # self.path.addPath(old_path) - # # profiler('path PREPEND') - - # # if append_length: - # # append_path = gen_qpath(append_bars, 0, self.w) - - # # self.path.moveTo( - # # float(istop - self.w), - # # float(append_bars[0]['open']) - # # ) - # # self.path.addPath(append_path) - - # # profiler('path APPEND') - # # fp = self.fast_path - # # if fp is None: - # # self.fast_path = append_path - - # # else: - # # fp.moveTo( - # # float(istop - self.w), float(new_bars[0]['open']) - # # ) - # # fp.addPath(append_path) - - # # self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # # flip_cache = True - - # self._xrange = first_index, last_index - - # # trigger redraw despite caching - # self.prepareGeometryChange() - - # self.draw_last(last) - - # # # generate new lines objects for updatable "current bar" - # # self._last_bar_lines = bar_from_ohlc_row(last, self.w) - - # # # last bar update - # # i, o, h, l, last, v = last[ - # # ['index', 'open', 'high', 'low', 'close', 'volume'] - # # ] - # # # assert i == self.start_index - 1 - # # # assert i == last_index - # # body, larm, rarm = self._last_bar_lines - - # # # XXX: is there a faster way to modify this? - # # rarm.setLine(rarm.x1(), last, rarm.x2(), last) - - # # # writer is responsible for changing open on "first" volume of bar - # # larm.setLine(larm.x1(), o, larm.x2(), o) - - # # if l != h: # noqa - - # # if body is None: - # # body = self._last_bar_lines[0] = QLineF(i, l, i, h) - # # else: - # # # update body - # # body.setLine(i, l, i, h) - - # # # XXX: pretty sure this is causing an issue where the bar has - # # # a large upward move right before the next sample and the body - # # # is getting set to None since the next bar is flat but the shm - # # # array index update wasn't read by the time this code runs. Iow - # # # we're doing this removal of the body for a bar index that is - # # # now out of date / from some previous sample. It's weird - # # # though because i've seen it do this to bars i - 3 back? - - # profiler('last bar set') - - # self.update() - # profiler('.update()') - - # if flip_cache: - # self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - # # profiler.finish() - def draw_last( self, last: np.ndarray, From bc50db59259d5ec9b59c69c69ff2f73c7eef19e7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 14:30:13 -0400 Subject: [PATCH 048/113] Rename `._ohlc.gen_qpath()` -> `.gen_ohlc_qpath()` --- piker/ui/_flows.py | 4 ++-- piker/ui/_ohlc.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index bf6fc3f3..bfbe8aea 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -52,7 +52,7 @@ from .._profile import ( ) from ._ohlc import ( BarItems, - gen_qpath, + gen_ohlc_qpath, ) from ._curve import ( FastAppendCurve, @@ -365,7 +365,7 @@ class Flow(msgspec.Struct): # , frozen=True): r = self._src_r = Renderer( flow=self, # TODO: rename this to something with ohlc - draw_path=gen_qpath, + draw_path=gen_ohlc_qpath, last_read=read, ) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index c8da5ba5..efd50e95 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -156,7 +156,7 @@ def path_arrays_from_ohlc( return x, y, c -def gen_qpath( +def gen_ohlc_qpath( data: np.ndarray, start: int = 0, # XXX: do we need this? # 0.5 is no overlap between arms, 1.0 is full overlap @@ -274,7 +274,7 @@ class BarItems(pg.GraphicsObject): ''' hist, last = ohlc[:-1], ohlc[-1] - self.path = gen_qpath(hist, start, self.w) + self.path = gen_ohlc_qpath(hist, start, self.w) # save graphics for later reference and keep track # of current internal "last index" From 9c5bc6dedaa06c9d42346431b740628cf0e14ec8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 15:15:14 -0400 Subject: [PATCH 049/113] Add `.ui._pathops` module Starts a module for grouping together all our `QPainterpath` related generation and data format operations for creation of fast curve graphics. To start, drops `FastAppendCurve.downsample()` and moves it to a new `._pathops.xy_downsample()`. --- piker/ui/_curve.py | 40 ++++++----------------------- piker/ui/_flows.py | 31 ---------------------- piker/ui/_pathops.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 63 deletions(-) create mode 100644 piker/ui/_pathops.py diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index d038f085..4aee9ced 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -34,10 +34,11 @@ from PyQt5.QtCore import ( from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor -from ._compression import ( - # ohlc_to_m4_line, - ds_m4, -) +# from ._compression import ( +# # ohlc_to_m4_line, +# ds_m4, +# ) +from ._pathops import xy_downsample from ..log import get_logger @@ -174,32 +175,6 @@ class FastAppendCurve(pg.GraphicsObject): QLineF(lbar, 0, rbar, 0) ).length() - def downsample( - self, - x, - y, - px_width, - uppx, - - ) -> tuple[np.ndarray, np.ndarray]: - - # downsample whenever more then 1 pixels per datum can be shown. - # always refresh data bounds until we get diffing - # working properly, see above.. - bins, x, y = ds_m4( - x, - y, - px_width=px_width, - uppx=uppx, - # log_scale=bool(uppx) - ) - x = np.broadcast_to(x[:, None], y.shape) - # x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() - x = (x + np.array([-0.5, 0, 0, 0.5])).flatten() - y = y.flatten() - - return x, y - def update_from_array( self, @@ -396,7 +371,8 @@ class FastAppendCurve(pg.GraphicsObject): self._in_ds = False elif should_ds and uppx and px_width > 1: - x_out, y_out = self.downsample( + + x_out, y_out = xy_downsample( x_out, y_out, px_width, @@ -461,7 +437,7 @@ class FastAppendCurve(pg.GraphicsObject): ) # if should_ds: - # new_x, new_y = self.downsample( + # new_x, new_y = xy_downsample( # new_x, # new_y, # px_width, diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index bfbe8aea..9f70efea 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -789,37 +789,6 @@ class Flow(msgspec.Struct): # , frozen=True): return graphics -def xy_downsample( - x, - y, - px_width, - uppx, - - x_spacer: float = 0.5, - -) -> tuple[np.ndarray, np.ndarray]: - - # downsample whenever more then 1 pixels per datum can be shown. - # always refresh data bounds until we get diffing - # working properly, see above.. - bins, x, y = ds_m4( - x, - y, - px_width=px_width, - uppx=uppx, - log_scale=bool(uppx) - ) - - # flatten output to 1d arrays suitable for path-graphics generation. - x = np.broadcast_to(x[:, None], y.shape) - x = (x + np.array( - [-x_spacer, 0, 0, x_spacer] - )).flatten() - y = y.flatten() - - return x, y - - class Renderer(msgspec.Struct): flow: Flow diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py new file mode 100644 index 00000000..654b079a --- /dev/null +++ b/piker/ui/_pathops.py @@ -0,0 +1,61 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +""" +Super fast ``QPainterPath`` generation related operator routines. + +""" + +import numpy as np +# from numba import njit, float64, int64 # , optional +# import pyqtgraph as pg +# from PyQt5 import QtCore, QtGui, QtWidgets +# from PyQt5.QtCore import QLineF, QPointF + +from ._compression import ( + # ohlc_flatten, + ds_m4, +) + + +def xy_downsample( + x, + y, + px_width, + uppx, + + x_spacer: float = 0.5, + +) -> tuple[np.ndarray, np.ndarray]: + + # downsample whenever more then 1 pixels per datum can be shown. + # always refresh data bounds until we get diffing + # working properly, see above.. + bins, x, y = ds_m4( + x, + y, + px_width=px_width, + uppx=uppx, + # log_scale=bool(uppx) + ) + + # flatten output to 1d arrays suitable for path-graphics generation. + x = np.broadcast_to(x[:, None], y.shape) + x = (x + np.array( + [-x_spacer, 0, 0, x_spacer] + )).flatten() + y = y.flatten() + + return x, y From 037300ced0d254983d6941d30bceafa2120a3f38 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 15:21:25 -0400 Subject: [PATCH 050/113] Move ohlc lines-curve generators into pathops mod --- piker/ui/_flows.py | 12 ++--- piker/ui/_ohlc.py | 115 +--------------------------------------- piker/ui/_pathops.py | 123 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 126 insertions(+), 124 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 9f70efea..38bcf348 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -48,20 +48,18 @@ from ..data._sharedmem import ( ) from .._profile import ( pg_profile_enabled, - ms_slower_then, + # ms_slower_then, +) +from ._pathops import ( + gen_ohlc_qpath, ) from ._ohlc import ( BarItems, - gen_ohlc_qpath, ) from ._curve import ( FastAppendCurve, # step_path_arrays_from_1d, ) -from ._compression import ( - # ohlc_flatten, - ds_m4, -) from ..log import get_logger @@ -784,7 +782,7 @@ class Flow(msgspec.Struct): # , frozen=True): profiler=profiler, **kwargs ) - profiler(f'`graphics.update_from_array()` complete') + profiler('`graphics.update_from_array()` complete') return graphics diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index efd50e95..abe0cb7b 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -25,7 +25,6 @@ from typing import ( import numpy as np import pyqtgraph as pg -from numba import njit, float64, int64 # , optional from PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtCore import QLineF, QPointF # from numba import types as ntypes @@ -36,6 +35,7 @@ from ._style import hcolor from ..log import get_logger from ._curve import FastAppendCurve from ._compression import ohlc_flatten +from ._pathops import gen_ohlc_qpath if TYPE_CHECKING: from ._chart import LinkedSplits @@ -84,119 +84,6 @@ def bar_from_ohlc_row( return [hl, o, c] -@njit( - # TODO: for now need to construct this manually for readonly arrays, see - # https://github.com/numba/numba/issues/4511 - # ntypes.tuple((float64[:], float64[:], float64[:]))( - # numba_ohlc_dtype[::1], # contiguous - # int64, - # optional(float64), - # ), - nogil=True -) -def path_arrays_from_ohlc( - data: np.ndarray, - start: int64, - bar_gap: float64 = 0.43, - -) -> np.ndarray: - ''' - Generate an array of lines objects from input ohlc data. - - ''' - size = int(data.shape[0] * 6) - - x = np.zeros( - # data, - shape=size, - dtype=float64, - ) - y, c = x.copy(), x.copy() - - # TODO: report bug for assert @ - # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991 - for i, q in enumerate(data[start:], start): - - # TODO: ask numba why this doesn't work.. - # open, high, low, close, index = q[ - # ['open', 'high', 'low', 'close', 'index']] - - open = q['open'] - high = q['high'] - low = q['low'] - close = q['close'] - index = float64(q['index']) - - istart = i * 6 - istop = istart + 6 - - # x,y detail the 6 points which connect all vertexes of a ohlc bar - x[istart:istop] = ( - index - bar_gap, - index, - index, - index, - index, - index + bar_gap, - ) - y[istart:istop] = ( - open, - open, - low, - high, - close, - close, - ) - - # specifies that the first edge is never connected to the - # prior bars last edge thus providing a small "gap"/"space" - # between bars determined by ``bar_gap``. - c[istart:istop] = (1, 1, 1, 1, 1, 0) - - return x, y, c - - -def gen_ohlc_qpath( - data: np.ndarray, - start: int = 0, # XXX: do we need this? - # 0.5 is no overlap between arms, 1.0 is full overlap - w: float = 0.43, - path: Optional[QtGui.QPainterPath] = None, - -) -> QtGui.QPainterPath: - - path_was_none = path is None - - profiler = pg.debug.Profiler( - msg='gen_qpath ohlc', - disabled=not pg_profile_enabled(), - ms_threshold=ms_slower_then, - ) - - x, y, c = path_arrays_from_ohlc( - data, - start, - bar_gap=w, - ) - profiler("generate stream with numba") - - # TODO: numba the internals of this! - path = pg.functions.arrayToQPath( - x, - y, - connect=c, - path=path, - ) - - # avoid mem allocs if possible - if path_was_none: - path.reserve(path.capacity()) - - profiler("generate path with arrayToQPath") - - return path - - class BarItems(pg.GraphicsObject): ''' "Price range" bars graphics rendered from a OHLC sampled sequence. diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index 654b079a..87e3183e 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -17,13 +17,17 @@ Super fast ``QPainterPath`` generation related operator routines. """ +from typing import ( + Optional, +) import numpy as np -# from numba import njit, float64, int64 # , optional -# import pyqtgraph as pg -# from PyQt5 import QtCore, QtGui, QtWidgets +from numba import njit, float64, int64 # , optional +import pyqtgraph as pg +from PyQt5 import QtGui # from PyQt5.QtCore import QLineF, QPointF +from .._profile import pg_profile_enabled, ms_slower_then from ._compression import ( # ohlc_flatten, ds_m4, @@ -59,3 +63,116 @@ def xy_downsample( y = y.flatten() return x, y + + +@njit( + # TODO: for now need to construct this manually for readonly arrays, see + # https://github.com/numba/numba/issues/4511 + # ntypes.tuple((float64[:], float64[:], float64[:]))( + # numba_ohlc_dtype[::1], # contiguous + # int64, + # optional(float64), + # ), + nogil=True +) +def path_arrays_from_ohlc( + data: np.ndarray, + start: int64, + bar_gap: float64 = 0.43, + +) -> np.ndarray: + ''' + Generate an array of lines objects from input ohlc data. + + ''' + size = int(data.shape[0] * 6) + + x = np.zeros( + # data, + shape=size, + dtype=float64, + ) + y, c = x.copy(), x.copy() + + # TODO: report bug for assert @ + # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991 + for i, q in enumerate(data[start:], start): + + # TODO: ask numba why this doesn't work.. + # open, high, low, close, index = q[ + # ['open', 'high', 'low', 'close', 'index']] + + open = q['open'] + high = q['high'] + low = q['low'] + close = q['close'] + index = float64(q['index']) + + istart = i * 6 + istop = istart + 6 + + # x,y detail the 6 points which connect all vertexes of a ohlc bar + x[istart:istop] = ( + index - bar_gap, + index, + index, + index, + index, + index + bar_gap, + ) + y[istart:istop] = ( + open, + open, + low, + high, + close, + close, + ) + + # specifies that the first edge is never connected to the + # prior bars last edge thus providing a small "gap"/"space" + # between bars determined by ``bar_gap``. + c[istart:istop] = (1, 1, 1, 1, 1, 0) + + return x, y, c + + +def gen_ohlc_qpath( + data: np.ndarray, + start: int = 0, # XXX: do we need this? + # 0.5 is no overlap between arms, 1.0 is full overlap + w: float = 0.43, + path: Optional[QtGui.QPainterPath] = None, + +) -> QtGui.QPainterPath: + + path_was_none = path is None + + profiler = pg.debug.Profiler( + msg='gen_qpath ohlc', + disabled=not pg_profile_enabled(), + ms_threshold=ms_slower_then, + ) + + x, y, c = path_arrays_from_ohlc( + data, + start, + bar_gap=w, + ) + profiler("generate stream with numba") + + # TODO: numba the internals of this! + path = pg.functions.arrayToQPath( + x, + y, + connect=c, + path=path, + ) + + # avoid mem allocs if possible + if path_was_none: + path.reserve(path.capacity()) + + profiler("generate path with arrayToQPath") + + return path From ca5a25f9217404d78756121d0015c6a66b8ec209 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 15:44:19 -0400 Subject: [PATCH 051/113] Drop commented `numba` imports --- piker/ui/_ohlc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index abe0cb7b..14d5b926 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -27,8 +27,6 @@ import numpy as np import pyqtgraph as pg from PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtCore import QLineF, QPointF -# from numba import types as ntypes -# from ..data._source import numba_ohlc_dtype from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor From 537b725bf3738c2e455638e4683e92151107177f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 15:45:06 -0400 Subject: [PATCH 052/113] Factor ohlc to line data conversion into `._pathops.ohlc_to_line()` --- piker/ui/_flows.py | 47 ++++++++++++++++++++++------------------- piker/ui/_pathops.py | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 21 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 38bcf348..266b3aeb 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -52,13 +52,13 @@ from .._profile import ( ) from ._pathops import ( gen_ohlc_qpath, + ohlc_to_line, ) from ._ohlc import ( BarItems, ) from ._curve import ( FastAppendCurve, - # step_path_arrays_from_1d, ) from ..log import get_logger @@ -426,29 +426,34 @@ class Flow(msgspec.Struct): # , frozen=True): # create a flattened view onto the OHLC array # which can be read as a line-style format shm = self.shm + ( + self._iflat_first, + self._iflat_last, + self.gx, + self.gy, + ) = ohlc_to_line(shm) - # flat = self.gy = self.shm.unstruct_view(fields) - self.gy = self.shm.ustruct(fields) - first = self._iflat_first = self.shm._first.value - last = self._iflat_last = self.shm._last.value + # self.gy = self.shm.ustruct(fields) + # first = self._iflat_first = self.shm._first.value + # last = self._iflat_last = self.shm._last.value - # write pushed data to flattened copy - self.gy[first:last] = rfn.structured_to_unstructured( - self.shm.array[fields] - ) + # # write pushed data to flattened copy + # self.gy[first:last] = rfn.structured_to_unstructured( + # self.shm.array[fields] + # ) - # generate an flat-interpolated x-domain - self.gx = ( - np.broadcast_to( - shm._array['index'][:, None], - ( - shm._array.size, - # 4, # only ohlc - self.gy.shape[1], - ), - ) + np.array([-0.5, 0, 0, 0.5]) - ) - assert self.gy.any() + # # generate an flat-interpolated x-domain + # self.gx = ( + # np.broadcast_to( + # shm._array['index'][:, None], + # ( + # shm._array.size, + # # 4, # only ohlc + # self.gy.shape[1], + # ), + # ) + np.array([-0.5, 0, 0, 0.5]) + # ) + # assert self.gy.any() # print(f'unstruct diff: {time.time() - start}') # profiler('read unstr view bars to line') diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index 87e3183e..c1ad383c 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -22,11 +22,15 @@ from typing import ( ) import numpy as np +from numpy.lib import recfunctions as rfn from numba import njit, float64, int64 # , optional import pyqtgraph as pg from PyQt5 import QtGui # from PyQt5.QtCore import QLineF, QPointF +from ..data._sharedmem import ( + ShmArray, +) from .._profile import pg_profile_enabled, ms_slower_then from ._compression import ( # ohlc_flatten, @@ -176,3 +180,49 @@ def gen_ohlc_qpath( profiler("generate path with arrayToQPath") return path + + +def ohlc_to_line( + ohlc_shm: ShmArray, + fields: list[str] = ['open', 'high', 'low', 'close'] + +) -> tuple[ + int, # flattened first index + int, # flattened last index + np.ndarray, + np.ndarray, +]: + ''' + Convert an input struct-array holding OHLC samples into a pair of + flattened x, y arrays with the same size (datums wise) as the source + data. + + ''' + y_out = ohlc_shm.ustruct(fields) + first = ohlc_shm._first.value + last = ohlc_shm._last.value + + # write pushed data to flattened copy + y_out[first:last] = rfn.structured_to_unstructured( + ohlc_shm.array[fields] + ) + + # generate an flat-interpolated x-domain + x_out = ( + np.broadcast_to( + ohlc_shm._array['index'][:, None], + ( + ohlc_shm._array.size, + # 4, # only ohlc + y_out.shape[1], + ), + ) + np.array([-0.5, 0, 0, 0.5]) + ) + assert y_out.any() + + return ( + first, + last, + x_out, + y_out, + ) From 5d294031f298578dc071a73e650b4cf57abe77af Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 16:54:50 -0400 Subject: [PATCH 053/113] Factor step format data gen into `to_step_format()` Yet another path ops routine which converts a 1d array into a data format suitable for rendering a "step curve" graphics path (aka a "bar graph" but implemented as a continuous line). Also, factor the `BarItems` rendering logic (which determines whether to render the literal bars lines or a downsampled curve) into a routine `render_baritems()` until we figure out the right abstraction layer for it. --- piker/ui/_flows.py | 516 +++++++++++++++++++++---------------------- piker/ui/_pathops.py | 28 +++ 2 files changed, 282 insertions(+), 262 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 266b3aeb..70839ca0 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -53,6 +53,7 @@ from .._profile import ( from ._pathops import ( gen_ohlc_qpath, ohlc_to_line, + to_step_format, ) from ._ohlc import ( BarItems, @@ -140,6 +141,243 @@ def mk_ohlc_flat_copy( return y +def render_baritems( + flow: Flow, + graphics: BarItems, + read: tuple[ + int, int, np.ndarray, + int, int, np.ndarray, + ], + profiler: pg.debug.Profiler, + **kwargs, + +) -> None: + ''' + Graphics management logic for a ``BarItems`` object. + + Mostly just logic to determine when and how to downsample an OHLC + lines curve into a flattened line graphic and when to display one + graphic or the other. + + TODO: this should likely be moved into some kind of better abstraction + layer, if not a `Renderer` then something just above it? + + ''' + ( + xfirst, xlast, array, + ivl, ivr, in_view, + ) = read + + # if no source data renderer exists create one. + self = flow + r = self._src_r + if not r: + # OHLC bars path renderer + r = self._src_r = Renderer( + flow=self, + # TODO: rename this to something with ohlc + draw_path=gen_ohlc_qpath, + last_read=read, + ) + + ds_curve_r = Renderer( + flow=self, + + # just swap in the flat view + # data_t=lambda array: self.gy.array, + last_read=read, + draw_path=partial( + rowarr_to_path, + x_basis=None, + ), + + ) + curve = FastAppendCurve( + name='OHLC', + color=graphics._color, + ) + curve.hide() + self.plot.addItem(curve) + + # baseline "line" downsampled OHLC curve that should + # kick on only when we reach a certain uppx threshold. + self._render_table[0] = ( + ds_curve_r, + curve, + ) + + dsc_r, curve = self._render_table[0] + + # do checks for whether or not we require downsampling: + # - if we're **not** downsampling then we simply want to + # render the bars graphics curve and update.. + # - if insteam we are in a downsamplig state then we to + x_gt = 6 + uppx = curve.x_uppx() + in_line = should_line = curve.isVisible() + if ( + should_line + and uppx < x_gt + ): + print('FLIPPING TO BARS') + should_line = False + + elif ( + not should_line + and uppx >= x_gt + ): + print('FLIPPING TO LINE') + should_line = True + + profiler(f'ds logic complete line={should_line}') + + # do graphics updates + if should_line: + + fields = ['open', 'high', 'low', 'close'] + if self.gy is None: + # create a flattened view onto the OHLC array + # which can be read as a line-style format + shm = self.shm + ( + self._iflat_first, + self._iflat_last, + self.gx, + self.gy, + ) = ohlc_to_line( + shm, + fields=fields, + ) + + # print(f'unstruct diff: {time.time() - start}') + + gy = self.gy + + # update flatted ohlc copy + ( + iflat_first, + iflat, + ishm_last, + ishm_first, + ) = ( + self._iflat_first, + self._iflat_last, + self.shm._last.value, + self.shm._first.value + ) + + # check for shm prepend updates since last read. + if iflat_first != ishm_first: + + # write newly prepended data to flattened copy + gy[ + ishm_first:iflat_first + ] = rfn.structured_to_unstructured( + self.shm._array[fields][ishm_first:iflat_first] + ) + self._iflat_first = ishm_first + + to_update = rfn.structured_to_unstructured( + self.shm._array[iflat:ishm_last][fields] + ) + + gy[iflat:ishm_last][:] = to_update + profiler('updated ustruct OHLC data') + + # slice out up-to-last step contents + y_flat = gy[ishm_first:ishm_last] + x_flat = self.gx[ishm_first:ishm_last] + + # update local last-index tracking + self._iflat_last = ishm_last + + # reshape to 1d for graphics rendering + y = y_flat.reshape(-1) + x = x_flat.reshape(-1) + profiler('flattened ustruct OHLC data') + + # do all the same for only in-view data + y_iv_flat = y_flat[ivl:ivr] + x_iv_flat = x_flat[ivl:ivr] + y_iv = y_iv_flat.reshape(-1) + x_iv = x_iv_flat.reshape(-1) + profiler('flattened ustruct in-view OHLC data') + + # pass into curve graphics processing + curve.update_from_array( + x, + y, + x_iv=x_iv, + y_iv=y_iv, + view_range=(ivl, ivr), # hack + profiler=profiler, + # should_redraw=False, + + # NOTE: already passed through by display loop? + # do_append=uppx < 16, + **kwargs, + ) + curve.show() + profiler('updated ds curve') + + else: + # render incremental or in-view update + # and apply ouput (path) to graphics. + path, last = r.render( + read, + only_in_view=True, + ) + + graphics.path = path + graphics.draw_last(last) + + # NOTE: on appends we used to have to flip the coords + # cache thought it doesn't seem to be required any more? + # graphics.setCacheMode(QtWidgets.QGraphicsItem.NoCache) + # graphics.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + + # graphics.prepareGeometryChange() + graphics.update() + + if ( + not in_line + and should_line + ): + # change to line graphic + + log.info( + f'downsampling to line graphic {self.name}' + ) + graphics.hide() + # graphics.update() + curve.show() + curve.update() + + elif in_line and not should_line: + log.info(f'showing bars graphic {self.name}') + curve.hide() + graphics.show() + graphics.update() + + # update our pre-downsample-ready data and then pass that + # new data the downsampler algo for incremental update. + + # graphics.update_from_array( + # array, + # in_view, + # view_range=(ivl, ivr) if use_vr else None, + + # **kwargs, + # ) + + # generate and apply path to graphics obj + # graphics.path, last = r.render( + # read, + # only_in_view=True, + # ) + # graphics.draw_last(last) + + class Flow(msgspec.Struct): # , frozen=True): ''' (Financial Signal-)Flow compound type which wraps a real-time @@ -355,276 +593,30 @@ class Flow(msgspec.Struct): # , frozen=True): graphics = self.graphics if isinstance(graphics, BarItems): - - # if no source data renderer exists create one. - r = self._src_r - if not r: - # OHLC bars path renderer - r = self._src_r = Renderer( - flow=self, - # TODO: rename this to something with ohlc - draw_path=gen_ohlc_qpath, - last_read=read, - ) - - ds_curve_r = Renderer( - flow=self, - - # just swap in the flat view - # data_t=lambda array: self.gy.array, - last_read=read, - draw_path=partial( - rowarr_to_path, - x_basis=None, - ), - - ) - curve = FastAppendCurve( - name='OHLC', - color=graphics._color, - ) - curve.hide() - self.plot.addItem(curve) - - # baseline "line" downsampled OHLC curve that should - # kick on only when we reach a certain uppx threshold. - self._render_table[0] = ( - ds_curve_r, - curve, - ) - - dsc_r, curve = self._render_table[0] - - # do checks for whether or not we require downsampling: - # - if we're **not** downsampling then we simply want to - # render the bars graphics curve and update.. - # - if insteam we are in a downsamplig state then we to - x_gt = 6 - uppx = curve.x_uppx() - in_line = should_line = curve.isVisible() - if ( - should_line - and uppx < x_gt - ): - print('FLIPPING TO BARS') - should_line = False - - elif ( - not should_line - and uppx >= x_gt - ): - print('FLIPPING TO LINE') - should_line = True - - profiler(f'ds logic complete line={should_line}') - - # do graphics updates - if should_line: - - fields = ['open', 'high', 'low', 'close'] - if self.gy is None: - # create a flattened view onto the OHLC array - # which can be read as a line-style format - shm = self.shm - ( - self._iflat_first, - self._iflat_last, - self.gx, - self.gy, - ) = ohlc_to_line(shm) - - # self.gy = self.shm.ustruct(fields) - # first = self._iflat_first = self.shm._first.value - # last = self._iflat_last = self.shm._last.value - - # # write pushed data to flattened copy - # self.gy[first:last] = rfn.structured_to_unstructured( - # self.shm.array[fields] - # ) - - # # generate an flat-interpolated x-domain - # self.gx = ( - # np.broadcast_to( - # shm._array['index'][:, None], - # ( - # shm._array.size, - # # 4, # only ohlc - # self.gy.shape[1], - # ), - # ) + np.array([-0.5, 0, 0, 0.5]) - # ) - # assert self.gy.any() - - # print(f'unstruct diff: {time.time() - start}') - # profiler('read unstr view bars to line') - # start = self.gy._first.value - # update flatted ohlc copy - ( - iflat_first, - iflat, - ishm_last, - ishm_first, - ) = ( - self._iflat_first, - self._iflat_last, - self.shm._last.value, - self.shm._first.value - ) - - # check for shm prepend updates since last read. - if iflat_first != ishm_first: - - # write newly prepended data to flattened copy - self.gy[ - ishm_first:iflat_first - ] = rfn.structured_to_unstructured( - self.shm._array[fields][ishm_first:iflat_first] - ) - self._iflat_first = ishm_first - - # # flat = self.gy = self.shm.unstruct_view(fields) - # self.gy = self.shm.ustruct(fields) - # # self._iflat_last = self.shm._last.value - - # # self._iflat_first = self.shm._first.value - # # do an update for the most recent prepend - # # index - # iflat = ishm_first - - to_update = rfn.structured_to_unstructured( - self.shm._array[iflat:ishm_last][fields] - ) - - self.gy[iflat:ishm_last][:] = to_update - profiler('updated ustruct OHLC data') - - # slice out up-to-last step contents - y_flat = self.gy[ishm_first:ishm_last] - x_flat = self.gx[ishm_first:ishm_last] - - # update local last-index tracking - self._iflat_last = ishm_last - - # reshape to 1d for graphics rendering - y = y_flat.reshape(-1) - x = x_flat.reshape(-1) - profiler('flattened ustruct OHLC data') - - # do all the same for only in-view data - y_iv_flat = y_flat[ivl:ivr] - x_iv_flat = x_flat[ivl:ivr] - y_iv = y_iv_flat.reshape(-1) - x_iv = x_iv_flat.reshape(-1) - profiler('flattened ustruct in-view OHLC data') - - # legacy full-recompute-everytime method - # x, y = ohlc_flatten(array) - # x_iv, y_iv = ohlc_flatten(in_view) - # profiler('flattened OHLC data') - - curve.update_from_array( - x, - y, - x_iv=x_iv, - y_iv=y_iv, - view_range=(ivl, ivr), # hack - profiler=profiler, - # should_redraw=False, - - # NOTE: already passed through by display loop? - # do_append=uppx < 16, - **kwargs, - ) - curve.show() - profiler('updated ds curve') - - else: - # render incremental or in-view update - # and apply ouput (path) to graphics. - path, last = r.render( - read, - only_in_view=True, - ) - - graphics.path = path - graphics.draw_last(last) - - # NOTE: on appends we used to have to flip the coords - # cache thought it doesn't seem to be required any more? - # graphics.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # graphics.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - # graphics.prepareGeometryChange() - graphics.update() - - if ( - not in_line - and should_line - ): - # change to line graphic - - log.info( - f'downsampling to line graphic {self.name}' - ) - graphics.hide() - # graphics.update() - curve.show() - curve.update() - - elif in_line and not should_line: - log.info(f'showing bars graphic {self.name}') - curve.hide() - graphics.show() - graphics.update() - - # update our pre-downsample-ready data and then pass that - # new data the downsampler algo for incremental update. - - # graphics.update_from_array( - # array, - # in_view, - # view_range=(ivl, ivr) if use_vr else None, - - # **kwargs, - # ) - - # generate and apply path to graphics obj - # graphics.path, last = r.render( - # read, - # only_in_view=True, - # ) - # graphics.draw_last(last) + render_baritems( + self, + graphics, + read, + profiler, + **kwargs, + ) else: # ``FastAppendCurve`` case: array_key = array_key or self.name uppx = graphics.x_uppx() - profiler('read uppx') + profiler(f'read uppx {uppx}') if graphics._step_mode and self.gy is None: - self._iflat_first = self.shm._first.value - - # create a flattened view onto the OHLC array - # which can be read as a line-style format shm = self.shm - - # fields = ['index', array_key] - i = shm._array['index'].copy() - out = shm._array[array_key].copy() - - self.gx = np.broadcast_to( - i[:, None], - (i.size, 2), - ) + np.array([-0.5, 0.5]) - - # self.gy = np.broadcast_to( - # out[:, None], (out.size, 2), - # ) - self.gy = np.empty((len(out), 2), dtype=out.dtype) - self.gy[:] = out[:, np.newaxis] - - # start y at origin level - self.gy[0, 0] = 0 + ( + self._iflat_first, + self.gx, + self.gy, + ) = to_step_format( + shm, + array_key, + ) profiler('generated step mode data') if graphics._step_mode: diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index c1ad383c..2f491365 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -226,3 +226,31 @@ def ohlc_to_line( x_out, y_out, ) + + +def to_step_format( + shm: ShmArray, + data_field: str, + index_field: str = 'index', + +) -> tuple[int, np.ndarray, np.ndarray]: + ''' + Convert an input 1d shm array to a "step array" format + for use by path graphics generation. + + ''' + first = shm._first.value + i = shm._array['index'].copy() + out = shm._array[data_field].copy() + + x_out = np.broadcast_to( + i[:, None], + (i.size, 2), + ) + np.array([-0.5, 0.5]) + + y_out = np.empty((len(out), 2), dtype=out.dtype) + y_out[:] = out[:, np.newaxis] + + # start y at origin level + y_out[0, 0] = 0 + return first, x_out, y_out From 27ee9fdc81fa5d72e4bb6c596b124a590b2758c9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 17:06:52 -0400 Subject: [PATCH 054/113] Drop old non-working flatten routine --- piker/ui/_flows.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 70839ca0..d7ebe4e6 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -113,34 +113,6 @@ def rowarr_to_path( ) -def mk_ohlc_flat_copy( - ohlc_shm: ShmArray, - - # XXX: we bind this in currently.. - # x_basis: np.ndarray, - - # vr: Optional[slice] = None, - -) -> tuple[np.ndarray, np.ndarray]: - ''' - Return flattened-non-copy view into an OHLC shm array. - - ''' - ohlc = ohlc_shm._array[['open', 'high', 'low', 'close']] - # if vr: - # ohlc = ohlc[vr] - # x = x_basis[vr] - - unstructured = rfn.structured_to_unstructured( - ohlc, - copy=False, - ) - # breakpoint() - y = unstructured.flatten() - # x = x_basis[:y.size] - return y - - def render_baritems( flow: Flow, graphics: BarItems, From b236dc72e441c190f71119999d4dd441fc78d234 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 16 May 2022 14:31:04 -0400 Subject: [PATCH 055/113] Make vlm a float; discrete is so 80s --- piker/data/_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/data/_source.py b/piker/data/_source.py index 2f5f61ed..9afcb191 100644 --- a/piker/data/_source.py +++ b/piker/data/_source.py @@ -33,7 +33,7 @@ ohlc_fields = [ ('high', float), ('low', float), ('close', float), - ('volume', int), + ('volume', float), ('bar_wap', float), ] From 1dca7766d20763f3eeae7c735cf8f9843af875c9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 16 May 2022 14:31:23 -0400 Subject: [PATCH 056/113] Add notes about how to do mkts "trimming" Which is basically just "deleting" rows from a column series. You can only use the trim command from the `.cmd` cli and only with a so called `LocalClient` currently; it's also sketchy af and caused a machine to hang due to mem usage.. Ideally we can patch in this functionality for use by the rpc api and have it not hang like this XD Pertains to https://github.com/alpacahq/marketstore/issues/264 --- piker/data/marketstore.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/piker/data/marketstore.py b/piker/data/marketstore.py index e1fb38d5..43b15671 100644 --- a/piker/data/marketstore.py +++ b/piker/data/marketstore.py @@ -230,8 +230,8 @@ _ohlcv_dt = [ # ohlcv sampling ('Open', 'f4'), ('High', 'f4'), - ('Low', 'i8'), - ('Close', 'i8'), + ('Low', 'f4'), + ('Close', 'f4'), ('Volume', 'f4'), ] @@ -547,6 +547,17 @@ class Storage: if err: raise MarketStoreError(err) + # XXX: currently the only way to do this is through the CLI: + + # sudo ./marketstore connect --dir ~/.config/piker/data + # >> \show mnq.globex.20220617.ib/1Sec/OHLCV 2022-05-15 + # and this seems to block and use up mem.. + # >> \trim mnq.globex.20220617.ib/1Sec/OHLCV 2022-05-15 + + # relevant source code for this is here: + # https://github.com/alpacahq/marketstore/blob/master/cmd/connect/session/trim.go#L14 + # def delete_range(self, start_dt, end_dt) -> None: + # ... @acm async def open_storage_client( From 1f95ba4fd81d2bfceafe599eed15dffeb1a0ead4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 16 May 2022 17:58:44 -0400 Subject: [PATCH 057/113] Drop input xy from constructor, only keep state for cursor stuff.. --- piker/ui/_chart.py | 2 -- piker/ui/_curve.py | 23 ++++++++++------------- piker/ui/_ohlc.py | 2 -- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index f6fc44ec..2eba9a24 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1075,8 +1075,6 @@ class ChartPlotWidget(pg.PlotWidget): # yah, we wrote our own B) data = shm.array curve = FastAppendCurve( - y=data[data_key], - x=data['index'], # antialias=True, name=name, diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 4aee9ced..b404cb76 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -68,9 +68,6 @@ class FastAppendCurve(pg.GraphicsObject): ''' def __init__( self, - - x: np.ndarray = None, - y: np.ndarray = None, *args, step_mode: bool = False, @@ -85,8 +82,8 @@ class FastAppendCurve(pg.GraphicsObject): ) -> None: # brutaaalll, see comments within.. - self._y = self.yData = y - self._x = self.xData = x + self.yData = None + self.xData = None self._vr: Optional[tuple] = None self._avr: Optional[tuple] = None self._br = None @@ -206,7 +203,7 @@ class FastAppendCurve(pg.GraphicsObject): disabled=not pg_profile_enabled(), ms_threshold=ms_slower_then, ) - flip_cache = False + # flip_cache = False if self._xrange: istart, istop = self._xrange @@ -227,9 +224,8 @@ class FastAppendCurve(pg.GraphicsObject): # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. - # self.xData = x - # self.yData = y - # self._x, self._y = x, y + self.xData = x + self.yData = y # downsampling incremental state checking uppx = self.x_uppx() @@ -261,7 +257,7 @@ class FastAppendCurve(pg.GraphicsObject): vl, vr = view_range # last_ivr = self._x_iv_range - # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) + # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) zoom_or_append = False last_vr = self._vr @@ -390,7 +386,9 @@ class FastAppendCurve(pg.GraphicsObject): ) self.prepareGeometryChange() profiler( - f'generated fresh path. (should_redraw: {should_redraw} should_ds: {should_ds} new_sample_rate: {new_sample_rate})' + 'generated fresh path. ' + f'(should_redraw: {should_redraw} ' + f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' ) # profiler(f'DRAW PATH IN VIEW -> {self._name}') @@ -495,7 +493,6 @@ class FastAppendCurve(pg.GraphicsObject): self.draw_last(x, y) profiler('draw last segment') - # if flip_cache: # # # XXX: seems to be needed to avoid artifacts (see above). # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) @@ -545,7 +542,7 @@ class FastAppendCurve(pg.GraphicsObject): # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. def getData(self): - return self._x, self._y + return self.xData, self.yData # TODO: drop the above after ``Cursor`` re-work def get_arrays(self) -> tuple[np.ndarray, np.ndarray]: diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 14d5b926..88fa62f9 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -187,8 +187,6 @@ class BarItems(pg.GraphicsObject): # curve that does not release mem allocs: # https://doc.qt.io/qt-5/qpainterpath.html#clear curve = FastAppendCurve( - y=y, - x=x, name='OHLC', color=self._color, ) From f67fd11a29d5b5eefe46566b57ded4adfb74b698 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 16 May 2022 17:59:10 -0400 Subject: [PATCH 058/113] Little formattito --- piker/ui/_cursor.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/piker/ui/_cursor.py b/piker/ui/_cursor.py index 8f18fe45..fe5fc100 100644 --- a/piker/ui/_cursor.py +++ b/piker/ui/_cursor.py @@ -98,9 +98,11 @@ class LineDot(pg.CurvePoint): ev: QtCore.QEvent, ) -> bool: - if not isinstance( - ev, QtCore.QDynamicPropertyChangeEvent - ) or self.curve() is None: + + if ( + not isinstance(ev, QtCore.QDynamicPropertyChangeEvent) + or self.curve() is None + ): return False # TODO: get rid of this ``.getData()`` and @@ -115,7 +117,10 @@ class LineDot(pg.CurvePoint): i = round(index - x[0]) if i > 0 and i < len(y): newPos = (index, y[i]) - QtWidgets.QGraphicsItem.setPos(self, *newPos) + QtWidgets.QGraphicsItem.setPos( + self, + *newPos, + ) return True return False From df1c89e8118662199131f1fcce4811e09af9fef1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 17 May 2022 19:06:57 -0400 Subject: [PATCH 059/113] Drop all "pixel width" refs (`px_width`) from m4 impl --- piker/ui/_compression.py | 85 ++++++---------------------------------- piker/ui/_pathops.py | 5 +-- 2 files changed, 13 insertions(+), 77 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index 5e8b759a..e9564359 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -138,51 +138,20 @@ def ohlc_flatten( return x, flat -def ohlc_to_m4_line( - ohlc: np.ndarray, - px_width: int, - - downsample: bool = False, - uppx: Optional[float] = None, - pretrace: bool = False, - -) -> tuple[np.ndarray, np.ndarray]: - ''' - Convert an OHLC struct-array to a m4 downsampled 1-d array. - - ''' - xpts, flat = ohlc_flatten( - ohlc, - use_mxmn=pretrace, - ) - - if downsample: - bins, x, y = ds_m4( - xpts, - flat, - px_width=px_width, - uppx=uppx, - # log_scale=bool(uppx) - ) - x = np.broadcast_to(x[:, None], y.shape) - x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() - y = y.flatten() - - return x, y - else: - return xpts, flat - - def ds_m4( x: np.ndarray, y: np.ndarray, + # units-per-pixel-x(dimension) + uppx: float, + + # XXX: troll zone / easter egg.. + # want to mess with ur pal, pass in the actual + # pixel width here instead of uppx-proper (i.e. pass + # in our ``pg.GraphicsObject`` derivative's ``.px_width()`` + # gto mega-trip-out ur bud). Hint, it used to be implemented + # (wrongly) using "pixel width", so check the git history ;) - # this is the width of the data in view - # in display-device-local pixel units. - px_width: int, - uppx: Optional[float] = None, xrange: Optional[float] = None, - # log_scale: bool = True, ) -> tuple[int, np.ndarray, np.ndarray]: ''' @@ -209,29 +178,8 @@ def ds_m4( # "i didn't show it in the sample code, but it's accounted for # in the start and end indices and number of bins" - # optionally log-scale down the "supposed pxs on screen" - # as the units-per-px (uppx) get's large. - # if log_scale: - # assert uppx, 'You must provide a `uppx` value to use log scaling!' - # # uppx = uppx * math.log(uppx, 2) - - # # scaler = 2**7 / (1 + math.log(uppx, 2)) - # scaler = round( - # max( - # # NOTE: found that a 16x px width brought greater - # # detail, likely due to dpi scaling? - # # px_width=px_width * 16, - # 2**7 / (1 + math.log(uppx, 2)), - # 1 - # ) - # ) - # px_width *= scaler - - # else: - # px_width *= 16 - # should never get called unless actually needed - assert px_width > 1 and uppx > 0 + assert uppx > 1 # NOTE: if we didn't pre-slice the data to downsample # you could in theory pass these as the slicing params, @@ -248,16 +196,9 @@ def ds_m4( # uppx *= max(4 / (1 + math.log(uppx, 2)), 1) pxw = math.ceil(xrange / uppx) - # px_width = math.ceil(px_width) - # ratio of indexed x-value to width of raster in pixels. - # this is more or less, uppx: units-per-pixel. - # w = xrange / float(px_width) - # uppx = uppx * math.log(uppx, 2) - # w2 = px_width / uppx - - # scale up the width as the uppx get's large - w = uppx # * math.log(uppx, 666) + # scale up the frame "width" directly with uppx + w = uppx # ensure we make more then enough # frames (windows) for the output pixel @@ -276,9 +217,7 @@ def ds_m4( # print( # f'uppx: {uppx}\n' # f'xrange: {xrange}\n' - # f'px_width: {px_width}\n' # f'pxw: {pxw}\n' - # f'WTF w:{w}, w2:{w2}\n' # f'frames: {frames}\n' # ) assert frames >= (xrange / uppx) diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index 2f491365..f7eaf2a7 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -41,7 +41,6 @@ from ._compression import ( def xy_downsample( x, y, - px_width, uppx, x_spacer: float = 0.5, @@ -54,9 +53,7 @@ def xy_downsample( bins, x, y = ds_m4( x, y, - px_width=px_width, - uppx=uppx, - # log_scale=bool(uppx) + uppx, ) # flatten output to 1d arrays suitable for path-graphics generation. From 81be0b4bd0b3b8e3e65a5552df992317d0ac221e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 17 May 2022 19:14:49 -0400 Subject: [PATCH 060/113] Dont pass `px_width` to m4, add some commented path cap tracking --- piker/ui/_curve.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index b404cb76..8fd31199 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -86,7 +86,7 @@ class FastAppendCurve(pg.GraphicsObject): self.xData = None self._vr: Optional[tuple] = None self._avr: Optional[tuple] = None - self._br = None + self._last_cap: int = 0 self._name = name self.path: Optional[QtGui.QPainterPath] = None @@ -238,8 +238,8 @@ class FastAppendCurve(pg.GraphicsObject): # should_redraw = False # by default we only pull data up to the last (current) index - x_out_full = x_out = x[:slice_to_head] - y_out_full = y_out = y[:slice_to_head] + x_out = x[:slice_to_head] + y_out = y[:slice_to_head] # if a view range is passed, plan to draw the # source ouput that's "in view" of the chart. @@ -319,8 +319,7 @@ class FastAppendCurve(pg.GraphicsObject): # check for downsampling conditions if ( # std m4 downsample conditions - px_width - and abs(uppx_diff) >= 1 + abs(uppx_diff) >= 1 ): log.info( f'{self._name} sampler change: {self._last_uppx} -> {uppx}' @@ -366,12 +365,11 @@ class FastAppendCurve(pg.GraphicsObject): self._in_ds = False - elif should_ds and uppx and px_width > 1: + elif should_ds and uppx > 1: x_out, y_out = xy_downsample( x_out, y_out, - px_width, uppx, ) profiler(f'FULL PATH downsample redraw={should_ds}') @@ -438,7 +436,6 @@ class FastAppendCurve(pg.GraphicsObject): # new_x, new_y = xy_downsample( # new_x, # new_y, - # px_width, # uppx, # ) # profiler(f'fast path downsample redraw={should_ds}') @@ -489,9 +486,9 @@ class FastAppendCurve(pg.GraphicsObject): # self.disable_cache() # flip_cache = True - if draw_last: - self.draw_last(x, y) - profiler('draw last segment') + # if draw_last: + # self.draw_last(x, y) + # profiler('draw last segment') # if flip_cache: # # # XXX: seems to be needed to avoid artifacts (see above). @@ -544,10 +541,6 @@ class FastAppendCurve(pg.GraphicsObject): def getData(self): return self.xData, self.yData - # TODO: drop the above after ``Cursor`` re-work - def get_arrays(self) -> tuple[np.ndarray, np.ndarray]: - return self._x, self._y - def clear(self): ''' Clear internal graphics making object ready for full re-draw. @@ -653,7 +646,6 @@ class FastAppendCurve(pg.GraphicsObject): # hb_size, QSizeF(w, h) ) - self._br = br # print(f'bounding rect: {br}') return br @@ -691,6 +683,11 @@ class FastAppendCurve(pg.GraphicsObject): path = self.path + # cap = path.capacity() + # if cap != self._last_cap: + # print(f'NEW CAPACITY: {self._last_cap} -> {cap}') + # self._last_cap = cap + if path: p.drawPath(path) profiler(f'.drawPath(path): {path.capacity()}') From e258654c862a93dfc7bcbeace9bb617e1d1a2ad7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 17 May 2022 19:18:31 -0400 Subject: [PATCH 061/113] Just drop "line dot" updates for now.. --- piker/ui/_cursor.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/piker/ui/_cursor.py b/piker/ui/_cursor.py index fe5fc100..606ff3f2 100644 --- a/piker/ui/_cursor.py +++ b/piker/ui/_cursor.py @@ -108,20 +108,20 @@ class LineDot(pg.CurvePoint): # TODO: get rid of this ``.getData()`` and # make a more pythonic api to retreive backing # numpy arrays... - (x, y) = self.curve().getData() - index = self.property('index') - # first = self._plot._arrays['ohlc'][0]['index'] - # first = x[0] - # i = index - first - if index: - i = round(index - x[0]) - if i > 0 and i < len(y): - newPos = (index, y[i]) - QtWidgets.QGraphicsItem.setPos( - self, - *newPos, - ) - return True + # (x, y) = self.curve().getData() + # index = self.property('index') + # # first = self._plot._arrays['ohlc'][0]['index'] + # # first = x[0] + # # i = index - first + # if index: + # i = round(index - x[0]) + # if i > 0 and i < len(y): + # newPos = (index, y[i]) + # QtWidgets.QGraphicsItem.setPos( + # self, + # *newPos, + # ) + # return True return False From 4c7661fc2339e59aa4204f34f70002129dd4d648 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 08:24:12 -0400 Subject: [PATCH 062/113] Factor `.update_from_array()` into `Flow.update_graphics()` A bit hacky to get all graphics types working but this is hopefully the first step toward moving all the generic update logic into `Renderer` types which can be themselves managed more compactly and cached per uppx-m4 level. --- piker/ui/_curve.py | 538 +++++++++++++++--------------- piker/ui/_flows.py | 809 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 883 insertions(+), 464 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 8fd31199..9e1f684a 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -172,332 +172,332 @@ class FastAppendCurve(pg.GraphicsObject): QLineF(lbar, 0, rbar, 0) ).length() - def update_from_array( - self, + # def update_from_array( + # self, - # full array input history - x: np.ndarray, - y: np.ndarray, + # # full array input history + # x: np.ndarray, + # y: np.ndarray, - # pre-sliced array data that's "in view" - x_iv: np.ndarray, - y_iv: np.ndarray, + # # pre-sliced array data that's "in view" + # x_iv: np.ndarray, + # y_iv: np.ndarray, - view_range: Optional[tuple[int, int]] = None, - profiler: Optional[pg.debug.Profiler] = None, - draw_last: bool = True, - slice_to_head: int = -1, - do_append: bool = True, - should_redraw: bool = False, + # view_range: Optional[tuple[int, int]] = None, + # profiler: Optional[pg.debug.Profiler] = None, + # draw_last: bool = True, + # slice_to_head: int = -1, + # do_append: bool = True, + # should_redraw: bool = False, - ) -> QtGui.QPainterPath: - ''' - Update curve from input 2-d data. + # ) -> QtGui.QPainterPath: + # ''' + # Update curve from input 2-d data. - Compare with a cached "x-range" state and (pre/a)ppend based on - a length diff. + # Compare with a cached "x-range" state and (pre/a)ppend based on + # a length diff. - ''' - profiler = profiler or pg.debug.Profiler( - msg=f'FastAppendCurve.update_from_array(): `{self._name}`', - disabled=not pg_profile_enabled(), - ms_threshold=ms_slower_then, - ) - # flip_cache = False + # ''' + # profiler = profiler or pg.debug.Profiler( + # msg=f'FastAppendCurve.update_from_array(): `{self._name}`', + # disabled=not pg_profile_enabled(), + # ms_threshold=ms_slower_then, + # ) + # # flip_cache = False - if self._xrange: - istart, istop = self._xrange - else: - self._xrange = istart, istop = x[0], x[-1] + # if self._xrange: + # istart, istop = self._xrange + # else: + # self._xrange = istart, istop = x[0], x[-1] - # compute the length diffs between the first/last index entry in - # the input data and the last indexes we have on record from the - # last time we updated the curve index. - prepend_length = int(istart - x[0]) - append_length = int(x[-1] - istop) + # # compute the length diffs between the first/last index entry in + # # the input data and the last indexes we have on record from the + # # last time we updated the curve index. + # prepend_length = int(istart - x[0]) + # append_length = int(x[-1] - istop) - # this is the diff-mode, "data"-rendered index - # tracking var.. - self._xrange = x[0], x[-1] + # # this is the diff-mode, "data"-rendered index + # # tracking var.. + # self._xrange = x[0], x[-1] - # print(f"xrange: {self._xrange}") + # # print(f"xrange: {self._xrange}") - # XXX: lol brutal, the internals of `CurvePoint` (inherited by - # our `LineDot`) required ``.getData()`` to work.. - self.xData = x - self.yData = y + # # XXX: lol brutal, the internals of `CurvePoint` (inherited by + # # our `LineDot`) required ``.getData()`` to work.. + # self.xData = x + # self.yData = y - # downsampling incremental state checking - uppx = self.x_uppx() - px_width = self.px_width() - uppx_diff = (uppx - self._last_uppx) + # # downsampling incremental state checking + # uppx = self.x_uppx() + # px_width = self.px_width() + # uppx_diff = (uppx - self._last_uppx) - new_sample_rate = False - should_ds = self._in_ds - showing_src_data = self._in_ds - # should_redraw = False + # new_sample_rate = False + # should_ds = self._in_ds + # showing_src_data = self._in_ds + # # should_redraw = False - # by default we only pull data up to the last (current) index - x_out = x[:slice_to_head] - y_out = y[:slice_to_head] + # # by default we only pull data up to the last (current) index + # x_out = x[:slice_to_head] + # y_out = y[:slice_to_head] - # if a view range is passed, plan to draw the - # source ouput that's "in view" of the chart. - if ( - view_range - # and not self._in_ds - # and not prepend_length > 0 - ): - # print(f'{self._name} vr: {view_range}') + # # if a view range is passed, plan to draw the + # # source ouput that's "in view" of the chart. + # if ( + # view_range + # # and not self._in_ds + # # and not prepend_length > 0 + # ): + # # print(f'{self._name} vr: {view_range}') - # by default we only pull data up to the last (current) index - x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head] - profiler(f'view range slice {view_range}') + # # by default we only pull data up to the last (current) index + # x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head] + # profiler(f'view range slice {view_range}') - vl, vr = view_range + # vl, vr = view_range - # last_ivr = self._x_iv_range - # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) + # # last_ivr = self._x_iv_range + # # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) - zoom_or_append = False - last_vr = self._vr - last_ivr = self._avr + # zoom_or_append = False + # last_vr = self._vr + # last_ivr = self._avr - if last_vr: - # relative slice indices - lvl, lvr = last_vr - # abs slice indices - al, ar = last_ivr + # if last_vr: + # # relative slice indices + # lvl, lvr = last_vr + # # abs slice indices + # al, ar = last_ivr - # append_length = int(x[-1] - istop) - # append_length = int(x_iv[-1] - ar) + # # append_length = int(x[-1] - istop) + # # append_length = int(x_iv[-1] - ar) - # left_change = abs(x_iv[0] - al) >= 1 - # right_change = abs(x_iv[-1] - ar) >= 1 + # # left_change = abs(x_iv[0] - al) >= 1 + # # right_change = abs(x_iv[-1] - ar) >= 1 - if ( - # likely a zoom view change - (vr - lvr) > 2 or vl < lvl - # append / prepend update - # we had an append update where the view range - # didn't change but the data-viewed (shifted) - # underneath, so we need to redraw. - # or left_change and right_change and last_vr == view_range + # if ( + # # likely a zoom view change + # (vr - lvr) > 2 or vl < lvl + # # append / prepend update + # # we had an append update where the view range + # # didn't change but the data-viewed (shifted) + # # underneath, so we need to redraw. + # # or left_change and right_change and last_vr == view_range - # not (left_change and right_change) and ivr - # ( - # or abs(x_iv[ivr] - livr) > 1 - ): - zoom_or_append = True + # # not (left_change and right_change) and ivr + # # ( + # # or abs(x_iv[ivr] - livr) > 1 + # ): + # zoom_or_append = True - # if last_ivr: - # liivl, liivr = last_ivr + # # if last_ivr: + # # liivl, liivr = last_ivr - if ( - view_range != last_vr - and ( - append_length > 1 - or zoom_or_append - ) - ): - should_redraw = True - # print("REDRAWING BRUH") + # if ( + # view_range != last_vr + # and ( + # append_length > 1 + # or zoom_or_append + # ) + # ): + # should_redraw = True + # # print("REDRAWING BRUH") - self._vr = view_range - self._avr = x_iv[0], x_iv[slice_to_head] + # self._vr = view_range + # self._avr = x_iv[0], x_iv[slice_to_head] - # x_last = x_iv[-1] - # y_last = y_iv[-1] - # self._last_vr = view_range + # # x_last = x_iv[-1] + # # y_last = y_iv[-1] + # # self._last_vr = view_range - # self.disable_cache() - # flip_cache = True + # # self.disable_cache() + # # flip_cache = True - if prepend_length > 0: - should_redraw = True + # if prepend_length > 0: + # should_redraw = True - # check for downsampling conditions - if ( - # std m4 downsample conditions - abs(uppx_diff) >= 1 - ): - log.info( - f'{self._name} sampler change: {self._last_uppx} -> {uppx}' - ) - self._last_uppx = uppx - new_sample_rate = True - showing_src_data = False - should_redraw = True - should_ds = True + # # check for downsampling conditions + # if ( + # # std m4 downsample conditions + # abs(uppx_diff) >= 1 + # ): + # log.info( + # f'{self._name} sampler change: {self._last_uppx} -> {uppx}' + # ) + # self._last_uppx = uppx + # new_sample_rate = True + # showing_src_data = False + # should_redraw = True + # should_ds = True - elif ( - uppx <= 2 - and self._in_ds - ): - # we should de-downsample back to our original - # source data so we clear our path data in prep - # to generate a new one from original source data. - should_redraw = True - new_sample_rate = True - should_ds = False - showing_src_data = True + # elif ( + # uppx <= 2 + # and self._in_ds + # ): + # # we should de-downsample back to our original + # # source data so we clear our path data in prep + # # to generate a new one from original source data. + # should_redraw = True + # new_sample_rate = True + # should_ds = False + # showing_src_data = True - # no_path_yet = self.path is None - if ( - self.path is None - or should_redraw - or new_sample_rate - or prepend_length > 0 - ): - if should_redraw: - if self.path: - self.path.clear() - profiler('cleared paths due to `should_redraw=True`') + # # no_path_yet = self.path is None + # if ( + # self.path is None + # or should_redraw + # or new_sample_rate + # or prepend_length > 0 + # ): + # if should_redraw: + # if self.path: + # self.path.clear() + # profiler('cleared paths due to `should_redraw=True`') - if self.fast_path: - self.fast_path.clear() + # if self.fast_path: + # self.fast_path.clear() - profiler('cleared paths due to `should_redraw` set') + # profiler('cleared paths due to `should_redraw` set') - if new_sample_rate and showing_src_data: - # if self._in_ds: - log.info(f'DEDOWN -> {self._name}') + # if new_sample_rate and showing_src_data: + # # if self._in_ds: + # log.info(f'DEDOWN -> {self._name}') - self._in_ds = False + # self._in_ds = False - elif should_ds and uppx > 1: + # elif should_ds and uppx > 1: - x_out, y_out = xy_downsample( - x_out, - y_out, - uppx, - ) - profiler(f'FULL PATH downsample redraw={should_ds}') - self._in_ds = True + # x_out, y_out = xy_downsample( + # x_out, + # y_out, + # uppx, + # ) + # profiler(f'FULL PATH downsample redraw={should_ds}') + # self._in_ds = True - self.path = pg.functions.arrayToQPath( - x_out, - y_out, - connect='all', - finiteCheck=False, - path=self.path, - ) - self.prepareGeometryChange() - profiler( - 'generated fresh path. ' - f'(should_redraw: {should_redraw} ' - f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' - ) - # profiler(f'DRAW PATH IN VIEW -> {self._name}') + # self.path = pg.functions.arrayToQPath( + # x_out, + # y_out, + # connect='all', + # finiteCheck=False, + # path=self.path, + # ) + # self.prepareGeometryChange() + # profiler( + # 'generated fresh path. ' + # f'(should_redraw: {should_redraw} ' + # f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' + # ) + # # profiler(f'DRAW PATH IN VIEW -> {self._name}') - # reserve mem allocs see: - # - https://doc.qt.io/qt-5/qpainterpath.html#reserve - # - https://doc.qt.io/qt-5/qpainterpath.html#capacity - # - https://doc.qt.io/qt-5/qpainterpath.html#clear - # XXX: right now this is based on had hoc checks on a - # hidpi 3840x2160 4k monitor but we should optimize for - # the target display(s) on the sys. - # if no_path_yet: - # self.path.reserve(int(500e3)) + # # reserve mem allocs see: + # # - https://doc.qt.io/qt-5/qpainterpath.html#reserve + # # - https://doc.qt.io/qt-5/qpainterpath.html#capacity + # # - https://doc.qt.io/qt-5/qpainterpath.html#clear + # # XXX: right now this is based on had hoc checks on a + # # hidpi 3840x2160 4k monitor but we should optimize for + # # the target display(s) on the sys. + # # if no_path_yet: + # # self.path.reserve(int(500e3)) - # TODO: get this piecewise prepend working - right now it's - # giving heck on vwap... - # elif prepend_length: - # breakpoint() + # # TODO: get this piecewise prepend working - right now it's + # # giving heck on vwap... + # # elif prepend_length: + # # breakpoint() - # prepend_path = pg.functions.arrayToQPath( - # x[0:prepend_length], - # y[0:prepend_length], - # connect='all' - # ) + # # prepend_path = pg.functions.arrayToQPath( + # # x[0:prepend_length], + # # y[0:prepend_length], + # # connect='all' + # # ) - # # swap prepend path in "front" - # old_path = self.path - # self.path = prepend_path - # # self.path.moveTo(new_x[0], new_y[0]) - # self.path.connectPath(old_path) + # # # swap prepend path in "front" + # # old_path = self.path + # # self.path = prepend_path + # # # self.path.moveTo(new_x[0], new_y[0]) + # # self.path.connectPath(old_path) - elif ( - append_length > 0 - and do_append - and not should_redraw - # and not view_range - ): - print(f'{self._name} append len: {append_length}') - new_x = x[-append_length - 2:slice_to_head] - new_y = y[-append_length - 2:slice_to_head] - profiler('sliced append path') + # elif ( + # append_length > 0 + # and do_append + # and not should_redraw + # # and not view_range + # ): + # print(f'{self._name} append len: {append_length}') + # new_x = x[-append_length - 2:slice_to_head] + # new_y = y[-append_length - 2:slice_to_head] + # profiler('sliced append path') - profiler( - f'diffed array input, append_length={append_length}' - ) + # profiler( + # f'diffed array input, append_length={append_length}' + # ) - # if should_ds: - # new_x, new_y = xy_downsample( - # new_x, - # new_y, - # uppx, - # ) - # profiler(f'fast path downsample redraw={should_ds}') + # # if should_ds: + # # new_x, new_y = xy_downsample( + # # new_x, + # # new_y, + # # uppx, + # # ) + # # profiler(f'fast path downsample redraw={should_ds}') - append_path = pg.functions.arrayToQPath( - new_x, - new_y, - connect='all', - finiteCheck=False, - path=self.fast_path, - ) - profiler('generated append qpath') + # append_path = pg.functions.arrayToQPath( + # new_x, + # new_y, + # connect='all', + # finiteCheck=False, + # path=self.fast_path, + # ) + # profiler('generated append qpath') - if self.use_fpath: - # an attempt at trying to make append-updates faster.. - if self.fast_path is None: - self.fast_path = append_path - # self.fast_path.reserve(int(6e3)) - else: - self.fast_path.connectPath(append_path) - size = self.fast_path.capacity() - profiler(f'connected fast path w size: {size}') + # if self.use_fpath: + # # an attempt at trying to make append-updates faster.. + # if self.fast_path is None: + # self.fast_path = append_path + # # self.fast_path.reserve(int(6e3)) + # else: + # self.fast_path.connectPath(append_path) + # size = self.fast_path.capacity() + # profiler(f'connected fast path w size: {size}') - # print(f"append_path br: {append_path.boundingRect()}") - # self.path.moveTo(new_x[0], new_y[0]) - # path.connectPath(append_path) + # # print(f"append_path br: {append_path.boundingRect()}") + # # self.path.moveTo(new_x[0], new_y[0]) + # # path.connectPath(append_path) - # XXX: lol this causes a hang.. - # self.path = self.path.simplified() - else: - size = self.path.capacity() - profiler(f'connected history path w size: {size}') - self.path.connectPath(append_path) + # # XXX: lol this causes a hang.. + # # self.path = self.path.simplified() + # else: + # size = self.path.capacity() + # profiler(f'connected history path w size: {size}') + # self.path.connectPath(append_path) - # other merging ideas: - # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths - # path.addPath(append_path) - # path.closeSubpath() + # # other merging ideas: + # # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths + # # path.addPath(append_path) + # # path.closeSubpath() - # TODO: try out new work from `pyqtgraph` main which - # should repair horrid perf: - # https://github.com/pyqtgraph/pyqtgraph/pull/2032 - # ok, nope still horrible XD - # if self._fill: - # # XXX: super slow set "union" op - # self.path = self.path.united(append_path).simplified() + # # TODO: try out new work from `pyqtgraph` main which + # # should repair horrid perf: + # # https://github.com/pyqtgraph/pyqtgraph/pull/2032 + # # ok, nope still horrible XD + # # if self._fill: + # # # XXX: super slow set "union" op + # # self.path = self.path.united(append_path).simplified() - # self.disable_cache() - # flip_cache = True + # # self.disable_cache() + # # flip_cache = True - # if draw_last: - # self.draw_last(x, y) - # profiler('draw last segment') + # # if draw_last: + # # self.draw_last(x, y) + # # profiler('draw last segment') - # if flip_cache: - # # # XXX: seems to be needed to avoid artifacts (see above). - # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + # # if flip_cache: + # # # # XXX: seems to be needed to avoid artifacts (see above). + # # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - # trigger redraw of path - # do update before reverting to cache mode - self.update() - profiler('.update()') + # # trigger redraw of path + # # do update before reverting to cache mode + # self.update() + # profiler('.update()') def draw_last( self, diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index d7ebe4e6..bc93f648 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -23,7 +23,7 @@ incremental update. ''' from __future__ import annotations -from functools import partial +# from functools import partial from typing import ( Optional, Callable, @@ -54,6 +54,7 @@ from ._pathops import ( gen_ohlc_qpath, ohlc_to_line, to_step_format, + xy_downsample, ) from ._ohlc import ( BarItems, @@ -152,18 +153,18 @@ def render_baritems( last_read=read, ) - ds_curve_r = Renderer( - flow=self, + # ds_curve_r = Renderer( + # flow=self, - # just swap in the flat view - # data_t=lambda array: self.gy.array, - last_read=read, - draw_path=partial( - rowarr_to_path, - x_basis=None, - ), + # # just swap in the flat view + # # data_t=lambda array: self.gy.array, + # last_read=read, + # draw_path=partial( + # rowarr_to_path, + # x_basis=None, + # ), - ) + # ) curve = FastAppendCurve( name='OHLC', color=graphics._color, @@ -173,12 +174,14 @@ def render_baritems( # baseline "line" downsampled OHLC curve that should # kick on only when we reach a certain uppx threshold. - self._render_table[0] = ( - ds_curve_r, - curve, - ) + self._render_table[0] = curve + # ( + # # ds_curve_r, + # curve, + # ) - dsc_r, curve = self._render_table[0] + curve = self._render_table[0] + # dsc_r, curve = self._render_table[0] # do checks for whether or not we require downsampling: # - if we're **not** downsampling then we simply want to @@ -276,19 +279,20 @@ def render_baritems( profiler('flattened ustruct in-view OHLC data') # pass into curve graphics processing - curve.update_from_array( - x, - y, - x_iv=x_iv, - y_iv=y_iv, - view_range=(ivl, ivr), # hack - profiler=profiler, - # should_redraw=False, + # curve.update_from_array( + # x, + # y, + # x_iv=x_iv, + # y_iv=y_iv, + # view_range=(ivl, ivr), # hack + # profiler=profiler, + # # should_redraw=False, - # NOTE: already passed through by display loop? - # do_append=uppx < 16, - **kwargs, - ) + # # NOTE: already passed through by display loop? + # # do_append=uppx < 16, + # **kwargs, + # ) + # curve.draw_last(x, y) curve.show() profiler('updated ds curve') @@ -349,6 +353,130 @@ def render_baritems( # ) # graphics.draw_last(last) + if should_line: + return ( + curve, + x, + y, + x_iv, + y_iv, + ) + + +def update_step_data( + flow: Flow, + shm: ShmArray, + ivl: int, + ivr: int, + array_key: str, + iflat_first: int, + iflat: int, + profiler: pg.debug.Profiler, + +) -> tuple: + + self = flow + ( + # iflat_first, + # iflat, + ishm_last, + ishm_first, + ) = ( + # self._iflat_first, + # self._iflat_last, + shm._last.value, + shm._first.value + ) + il = max(iflat - 1, 0) + profiler('read step mode incr update indices') + + # check for shm prepend updates since last read. + if iflat_first != ishm_first: + + print(f'prepend {array_key}') + + # i_prepend = self.shm._array['index'][ + # ishm_first:iflat_first] + y_prepend = self.shm._array[array_key][ + ishm_first:iflat_first + ] + + y2_prepend = np.broadcast_to( + y_prepend[:, None], (y_prepend.size, 2), + ) + + # write newly prepended data to flattened copy + self.gy[ishm_first:iflat_first] = y2_prepend + self._iflat_first = ishm_first + profiler('prepended step mode history') + + append_diff = ishm_last - iflat + if append_diff: + + # slice up to the last datum since last index/append update + # new_x = self.shm._array[il:ishm_last]['index'] + new_y = self.shm._array[il:ishm_last][array_key] + + new_y2 = np.broadcast_to( + new_y[:, None], (new_y.size, 2), + ) + self.gy[il:ishm_last] = new_y2 + profiler('updated step curve data') + + # print( + # f'append size: {append_diff}\n' + # f'new_x: {new_x}\n' + # f'new_y: {new_y}\n' + # f'new_y2: {new_y2}\n' + # f'new gy: {gy}\n' + # ) + + # update local last-index tracking + self._iflat_last = ishm_last + + # slice out up-to-last step contents + x_step = self.gx[ishm_first:ishm_last+2] + # shape to 1d + x = x_step.reshape(-1) + profiler('sliced step x') + + y_step = self.gy[ishm_first:ishm_last+2] + lasts = self.shm.array[['index', array_key]] + last = lasts[array_key][-1] + y_step[-1] = last + # shape to 1d + y = y_step.reshape(-1) + + # s = 6 + # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') + + profiler('sliced step y') + + # do all the same for only in-view data + ys_iv = y_step[ivl:ivr+1] + xs_iv = x_step[ivl:ivr+1] + y_iv = ys_iv.reshape(ys_iv.size) + x_iv = xs_iv.reshape(xs_iv.size) + # print( + # f'ys_iv : {ys_iv[-s:]}\n' + # f'y_iv: {y_iv[-s:]}\n' + # f'xs_iv: {xs_iv[-s:]}\n' + # f'x_iv: {x_iv[-s:]}\n' + # ) + profiler('sliced in view step data') + + # legacy full-recompute-everytime method + # x, y = ohlc_flatten(array) + # x_iv, y_iv = ohlc_flatten(in_view) + # profiler('flattened OHLC data') + return ( + x, + y, + x_iv, + y_iv, + append_diff, + ) + class Flow(msgspec.Struct): # , frozen=True): ''' @@ -368,11 +496,19 @@ class Flow(msgspec.Struct): # , frozen=True): is_ohlc: bool = False render: bool = True # toggle for display loop + + # pre-graphics formatted data gy: Optional[ShmArray] = None gx: Optional[np.ndarray] = None + # pre-graphics update indices _iflat_last: int = 0 _iflat_first: int = 0 + # view-range incremental state + _vr: Optional[tuple] = None + _avr: Optional[tuple] = None + + # downsampling state _last_uppx: float = 0 _in_ds: bool = False @@ -495,7 +631,11 @@ class Flow(msgspec.Struct): # , frozen=True): start, l, lbar, rbar, r, end, ) - def read(self) -> tuple[ + def read( + self, + array_field: Optional[str] = None, + + ) -> tuple[ int, int, np.ndarray, int, int, np.ndarray, ]: @@ -513,6 +653,9 @@ class Flow(msgspec.Struct): # , frozen=True): lbar_i = max(l, ifirst) - ifirst rbar_i = min(r, ilast) - ifirst + if array_field: + array = array[array_field] + # TODO: we could do it this way as well no? # to_draw = array[lbar - ifirst:(rbar - ifirst) + 1] in_view = array[lbar_i: rbar_i + 1] @@ -532,6 +675,7 @@ class Flow(msgspec.Struct): # , frozen=True): array_key: Optional[str] = None, profiler: Optional[pg.debug.Profiler] = None, + do_append: bool = True, **kwargs, @@ -557,15 +701,20 @@ class Flow(msgspec.Struct): # , frozen=True): ) = self.read() profiler('read src shm data') + graphics = self.graphics + if ( not in_view.size or not render ): - return self.graphics + return graphics - graphics = self.graphics + out: Optional[tuple] = None if isinstance(graphics, BarItems): - render_baritems( + # XXX: special case where we change out graphics + # to a line after a certain uppx threshold. + # render_baritems( + out = render_baritems( self, graphics, read, @@ -573,14 +722,74 @@ class Flow(msgspec.Struct): # , frozen=True): **kwargs, ) - else: - # ``FastAppendCurve`` case: - array_key = array_key or self.name - uppx = graphics.x_uppx() - profiler(f'read uppx {uppx}') + if out is None: + return graphics - if graphics._step_mode and self.gy is None: - shm = self.shm + # return graphics + + r = self._src_r + if not r: + # just using for ``.diff()`` atm.. + r = self._src_r = Renderer( + flow=self, + # TODO: rename this to something with ohlc + # draw_path=gen_ohlc_qpath, + last_read=read, + ) + + # ``FastAppendCurve`` case: + array_key = array_key or self.name + + new_sample_rate = False + should_ds = self._in_ds + showing_src_data = self._in_ds + + # draw_last: bool = True + slice_to_head: int = -1 + should_redraw: bool = False + + shm = self.shm + + # if a view range is passed, plan to draw the + # source ouput that's "in view" of the chart. + view_range = (ivl, ivr) if use_vr else None + + if out is not None: + # hack to handle ds curve from bars above + ( + graphics, # curve + x, + y, + x_iv, + y_iv, + ) = out + + else: + # full input data + x = array['index'] + y = array[array_key] + + # inview data + x_iv = in_view['index'] + y_iv = in_view[array_key] + + # downsampling incremental state checking + uppx = graphics.x_uppx() + # px_width = graphics.px_width() + uppx_diff = (uppx - self._last_uppx) + profiler(f'diffed uppx {uppx}') + + x_last = x[-1] + y_last = y[-1] + + slice_to_head = -1 + + profiler('sliced input arrays') + + if graphics._step_mode: + slice_to_head = -2 + + if self.gy is None: ( self._iflat_first, self.gx, @@ -591,177 +800,324 @@ class Flow(msgspec.Struct): # , frozen=True): ) profiler('generated step mode data') - if graphics._step_mode: - ( - iflat_first, - iflat, - ishm_last, - ishm_first, - ) = ( - self._iflat_first, - self._iflat_last, - self.shm._last.value, - self.shm._first.value + ( + x, + y, + x_iv, + y_iv, + append_diff, + + ) = update_step_data( + self, + shm, + ivl, + ivr, + array_key, + self._iflat_first, + self._iflat_last, + profiler, + ) + + graphics._last_line = QLineF( + x_last - 0.5, 0, + x_last + 0.5, 0, + ) + graphics._last_step_rect = QRectF( + x_last - 0.5, 0, + x_last + 0.5, y_last, + ) + + should_redraw = bool(append_diff) + + # graphics.reset_cache() + # print( + # f"path br: {graphics.path.boundingRect()}\n", + # # f"fast path br: {graphics.fast_path.boundingRect()}", + # f"last rect br: {graphics._last_step_rect}\n", + # f"full br: {graphics._br}\n", + # ) + + # compute the length diffs between the first/last index entry in + # the input data and the last indexes we have on record from the + # last time we updated the curve index. + prepend_length, append_length = r.diff(read) + # print((prepend_length, append_length)) + + # old_prepend_length = int(istart - x[0]) + # old_append_length = int(x[-1] - istop) + + # MAIN RENDER LOGIC: + # - determine in view data and redraw on range change + # - determine downsampling ops if needed + # - (incrementally) update ``QPainterPath`` + + if ( + view_range + # and not self._in_ds + # and not prepend_length > 0 + ): + # print(f'{self._name} vr: {view_range}') + + # by default we only pull data up to the last (current) index + x_out = x_iv[:slice_to_head] + y_out = y_iv[:slice_to_head] + profiler(f'view range slice {view_range}') + + vl, vr = view_range + + zoom_or_append = False + last_vr = self._vr + last_ivr = self._avr + + # incremental in-view data update. + if last_vr: + # relative slice indices + lvl, lvr = last_vr + # abs slice indices + al, ar = last_ivr + + # append_length = int(x[-1] - istop) + # append_length = int(x_iv[-1] - ar) + + # left_change = abs(x_iv[0] - al) >= 1 + # right_change = abs(x_iv[-1] - ar) >= 1 + + if ( + # likely a zoom view change + (vr - lvr) > 2 or vl < lvl + # append / prepend update + # we had an append update where the view range + # didn't change but the data-viewed (shifted) + # underneath, so we need to redraw. + # or left_change and right_change and last_vr == view_range + + # not (left_change and right_change) and ivr + # ( + # or abs(x_iv[ivr] - livr) > 1 + ): + zoom_or_append = True + + # if last_ivr: + # liivl, liivr = last_ivr + + if ( + view_range != last_vr + and ( + append_length > 1 + or zoom_or_append ) + ): + should_redraw = True + # print("REDRAWING BRUH") - il = max(iflat - 1, 0) - profiler('read step mode incr update indices') + self._vr = view_range + self._avr = x_iv[0], x_iv[slice_to_head] - # check for shm prepend updates since last read. - if iflat_first != ishm_first: + if prepend_length > 0: + should_redraw = True - print(f'prepend {array_key}') + # check for downsampling conditions + if ( + # std m4 downsample conditions + # px_width + # and abs(uppx_diff) >= 1 + abs(uppx_diff) >= 1 + ): + log.info( + f'{array_key} sampler change: {self._last_uppx} -> {uppx}' + ) + self._last_uppx = uppx + new_sample_rate = True + showing_src_data = False + should_redraw = True + should_ds = True - # i_prepend = self.shm._array['index'][ - # ishm_first:iflat_first] - y_prepend = self.shm._array[array_key][ - ishm_first:iflat_first - ] + elif ( + uppx <= 2 + and self._in_ds + ): + # we should de-downsample back to our original + # source data so we clear our path data in prep + # to generate a new one from original source data. + should_redraw = True + new_sample_rate = True + should_ds = False + showing_src_data = True - y2_prepend = np.broadcast_to( - y_prepend[:, None], (y_prepend.size, 2), - ) + # no_path_yet = self.path is None + fast_path = graphics.fast_path + if ( + graphics.path is None + or should_redraw + or new_sample_rate + or prepend_length > 0 + ): + if should_redraw: + if graphics.path: + graphics.path.clear() + profiler('cleared paths due to `should_redraw=True`') - # write newly prepended data to flattened copy - self.gy[ishm_first:iflat_first] = y2_prepend - self._iflat_first = ishm_first - profiler('prepended step mode history') + if graphics.fast_path: + graphics.fast_path.clear() - append_diff = ishm_last - iflat - if append_diff: + profiler('cleared paths due to `should_redraw` set') - # slice up to the last datum since last index/append update - # new_x = self.shm._array[il:ishm_last]['index'] - new_y = self.shm._array[il:ishm_last][array_key] + if new_sample_rate and showing_src_data: + # if self._in_ds: + log.info(f'DEDOWN -> {self.name}') - new_y2 = np.broadcast_to( - new_y[:, None], (new_y.size, 2), - ) - self.gy[il:ishm_last] = new_y2 - profiler('updated step curve data') + self._in_ds = False - # print( - # f'append size: {append_diff}\n' - # f'new_x: {new_x}\n' - # f'new_y: {new_y}\n' - # f'new_y2: {new_y2}\n' - # f'new gy: {gy}\n' - # ) + # elif should_ds and uppx and px_width > 1: + elif should_ds and uppx > 1: - # update local last-index tracking - self._iflat_last = ishm_last - - # slice out up-to-last step contents - x_step = self.gx[ishm_first:ishm_last+2] - # shape to 1d - x = x_step.reshape(-1) - profiler('sliced step x') - - y_step = self.gy[ishm_first:ishm_last+2] - lasts = self.shm.array[['index', array_key]] - last = lasts[array_key][-1] - y_step[-1] = last - # shape to 1d - y = y_step.reshape(-1) - - # s = 6 - # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') - - profiler('sliced step y') - - # do all the same for only in-view data - ys_iv = y_step[ivl:ivr+1] - xs_iv = x_step[ivl:ivr+1] - y_iv = ys_iv.reshape(ys_iv.size) - x_iv = xs_iv.reshape(xs_iv.size) - # print( - # f'ys_iv : {ys_iv[-s:]}\n' - # f'y_iv: {y_iv[-s:]}\n' - # f'xs_iv: {xs_iv[-s:]}\n' - # f'x_iv: {x_iv[-s:]}\n' - # ) - profiler('sliced in view step data') - - # legacy full-recompute-everytime method - # x, y = ohlc_flatten(array) - # x_iv, y_iv = ohlc_flatten(in_view) - # profiler('flattened OHLC data') - - x_last = array['index'][-1] - y_last = array[array_key][-1] - graphics._last_line = QLineF( - x_last - 0.5, 0, - x_last + 0.5, 0, + x_out, y_out = xy_downsample( + x_out, + y_out, + uppx, + # px_width, ) - graphics._last_step_rect = QRectF( - x_last - 0.5, 0, - x_last + 0.5, y_last, - ) - # graphics.update() + profiler(f'FULL PATH downsample redraw={should_ds}') + self._in_ds = True - graphics.update_from_array( - x=x, - y=y, + graphics.path = pg.functions.arrayToQPath( + x_out, + y_out, + connect='all', + finiteCheck=False, + path=graphics.path, + ) + graphics.prepareGeometryChange() + profiler( + 'generated fresh path. ' + f'(should_redraw: {should_redraw} ' + f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' + ) + # profiler(f'DRAW PATH IN VIEW -> {self.name}') - x_iv=x_iv, - y_iv=y_iv, + # reserve mem allocs see: + # - https://doc.qt.io/qt-5/qpainterpath.html#reserve + # - https://doc.qt.io/qt-5/qpainterpath.html#capacity + # - https://doc.qt.io/qt-5/qpainterpath.html#clear + # XXX: right now this is based on had hoc checks on a + # hidpi 3840x2160 4k monitor but we should optimize for + # the target display(s) on the sys. + # if no_path_yet: + # graphics.path.reserve(int(500e3)) - view_range=(ivl, ivr) if use_vr else None, + # TODO: get this piecewise prepend working - right now it's + # giving heck on vwap... + # elif prepend_length: + # breakpoint() - draw_last=False, - slice_to_head=-2, + # prepend_path = pg.functions.arrayToQPath( + # x[0:prepend_length], + # y[0:prepend_length], + # connect='all' + # ) - should_redraw=bool(append_diff), + # # swap prepend path in "front" + # old_path = graphics.path + # graphics.path = prepend_path + # # graphics.path.moveTo(new_x[0], new_y[0]) + # graphics.path.connectPath(old_path) - # NOTE: already passed through by display loop? - # do_append=uppx < 16, - profiler=profiler, + elif ( + append_length > 0 + and do_append + and not should_redraw + # and not view_range + ): + print(f'{self.name} append len: {append_length}') + new_x = x[-append_length - 2:slice_to_head] + new_y = y[-append_length - 2:slice_to_head] + profiler('sliced append path') - **kwargs - ) - profiler('updated step mode curve') - # graphics.reset_cache() - # print( - # f"path br: {graphics.path.boundingRect()}\n", - # # f"fast path br: {graphics.fast_path.boundingRect()}", - # f"last rect br: {graphics._last_step_rect}\n", - # f"full br: {graphics._br}\n", - # ) + profiler( + f'diffed array input, append_length={append_length}' + ) + # if should_ds: + # new_x, new_y = xy_downsample( + # new_x, + # new_y, + # px_width, + # uppx, + # ) + # profiler(f'fast path downsample redraw={should_ds}') + + append_path = pg.functions.arrayToQPath( + new_x, + new_y, + connect='all', + finiteCheck=False, + path=graphics.fast_path, + ) + profiler('generated append qpath') + + if graphics.use_fpath: + print("USING FPATH") + # an attempt at trying to make append-updates faster.. + if fast_path is None: + graphics.fast_path = append_path + # self.fast_path.reserve(int(6e3)) + else: + fast_path.connectPath(append_path) + size = fast_path.capacity() + profiler(f'connected fast path w size: {size}') + + # print(f"append_path br: {append_path.boundingRect()}") + # graphics.path.moveTo(new_x[0], new_y[0]) + # path.connectPath(append_path) + + # XXX: lol this causes a hang.. + # graphics.path = graphics.path.simplified() else: - x = array['index'] - y = array[array_key] - x_iv = in_view['index'] - y_iv = in_view[array_key] - profiler('sliced input arrays') + size = graphics.path.capacity() + profiler(f'connected history path w size: {size}') + graphics.path.connectPath(append_path) - # graphics.draw_last(x, y) + # graphics.update_from_array( + # x=x, + # y=y, - graphics.update_from_array( - x=x, - y=y, + # x_iv=x_iv, + # y_iv=y_iv, - x_iv=x_iv, - y_iv=y_iv, + # view_range=(ivl, ivr) if use_vr else None, - view_range=(ivl, ivr) if use_vr else None, + # # NOTE: already passed through by display loop. + # # do_append=uppx < 16, + # do_append=do_append, - # NOTE: already passed through by display loop? - # do_append=uppx < 16, - profiler=profiler, - **kwargs - ) - profiler('`graphics.update_from_array()` complete') + # slice_to_head=slice_to_head, + # should_redraw=should_redraw, + # profiler=profiler, + # **kwargs + # ) + graphics.draw_last(x, y) + profiler('draw last segment') + + graphics.update() + profiler('.update()') + + profiler('`graphics.update_from_array()` complete') return graphics class Renderer(msgspec.Struct): flow: Flow + # last array view read + last_read: Optional[tuple] = None # called to render path graphics - draw_path: Callable[np.ndarray, QPainterPath] + draw_path: Optional[Callable[np.ndarray, QPainterPath]] = None + + # output graphics rendering, the main object + # processed in ``QGraphicsObject.paint()`` + path: Optional[QPainterPath] = None # called on input data but before any graphics format # conversions or processing. @@ -778,25 +1134,66 @@ class Renderer(msgspec.Struct): prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None - # last array view read - last_read: Optional[np.ndarray] = None + # incremental update state(s) + # _in_ds: bool = False + # _last_uppx: float = 0 + _last_vr: Optional[tuple[float, float]] = None + _last_ivr: Optional[tuple[float, float]] = None - # output graphics rendering, the main object - # processed in ``QGraphicsObject.paint()`` - path: Optional[QPainterPath] = None + def diff( + self, + new_read: tuple[np.ndarray], - # def diff( - # self, - # latest_read: tuple[np.ndarray], + ) -> tuple[np.ndarray]: - # ) -> tuple[np.ndarray]: - # # blah blah blah - # # do diffing for prepend, append and last entry - # return ( - # to_prepend - # to_append - # last, - # ) + ( + last_xfirst, + last_xlast, + last_array, + last_ivl, last_ivr, + last_in_view, + ) = self.last_read + + # TODO: can the renderer just call ``Flow.read()`` directly? + # unpack latest source data read + ( + xfirst, + xlast, + array, + ivl, + ivr, + in_view, + ) = new_read + + # compute the length diffs between the first/last index entry in + # the input data and the last indexes we have on record from the + # last time we updated the curve index. + prepend_length = int(last_xfirst - xfirst) + append_length = int(xlast - last_xlast) + + # TODO: eventually maybe we can implement some kind of + # transform on the ``QPainterPath`` that will more or less + # detect the diff in "elements" terms? + # update state + self.last_read = new_read + + # blah blah blah + # do diffing for prepend, append and last entry + return ( + prepend_length, + append_length, + # last, + ) + + def draw_path( + self, + should_redraw: bool = False, + ) -> QPainterPath: + + if should_redraw: + if self.path: + self.path.clear() + # profiler('cleared paths due to `should_redraw=True`') def render( self, @@ -819,11 +1216,30 @@ class Renderer(msgspec.Struct): - blah blah blah (from notes) ''' - # do full source data render to path + # get graphics info + + # TODO: can the renderer just call ``Flow.read()`` directly? + # unpack latest source data read ( - xfirst, xlast, array, - ivl, ivr, in_view, - ) = self.last_read + xfirst, + xlast, + array, + ivl, + ivr, + in_view, + ) = new_read + + ( + prepend_length, + append_length, + ) = self.diff(new_read) + + # do full source data render to path + + # x = array['index'] + # y = array#[array_key] + # x_iv = in_view['index'] + # y_iv = in_view#[array_key] if only_in_view: array = in_view @@ -832,7 +1248,10 @@ class Renderer(msgspec.Struct): # xfirst, xlast, array, ivl, ivr, in_view # ) = new_read - if self.path is None or only_in_view: + if ( + self.path is None + or only_in_view + ): # redraw the entire source data if we have either of: # - no prior path graphic rendered or, # - we always intend to re-render the data only in view From 1dab77ca0b9debc976411ff253a23c58048ce6e1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 08:46:09 -0400 Subject: [PATCH 063/113] Rect wont show on step curves unless we avoid `.draw_last()` --- piker/ui/_flows.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index bc93f648..b0f1bb21 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -739,21 +739,16 @@ class Flow(msgspec.Struct): # , frozen=True): # ``FastAppendCurve`` case: array_key = array_key or self.name + shm = self.shm + # update config new_sample_rate = False should_ds = self._in_ds showing_src_data = self._in_ds - - # draw_last: bool = True + draw_last: bool = True slice_to_head: int = -1 should_redraw: bool = False - shm = self.shm - - # if a view range is passed, plan to draw the - # source ouput that's "in view" of the chart. - view_range = (ivl, ivr) if use_vr else None - if out is not None: # hack to handle ds curve from bars above ( @@ -828,6 +823,7 @@ class Flow(msgspec.Struct): # , frozen=True): ) should_redraw = bool(append_diff) + draw_last = False # graphics.reset_cache() # print( @@ -841,10 +837,6 @@ class Flow(msgspec.Struct): # , frozen=True): # the input data and the last indexes we have on record from the # last time we updated the curve index. prepend_length, append_length = r.diff(read) - # print((prepend_length, append_length)) - - # old_prepend_length = int(istart - x[0]) - # old_append_length = int(x[-1] - istop) # MAIN RENDER LOGIC: # - determine in view data and redraw on range change @@ -852,10 +844,14 @@ class Flow(msgspec.Struct): # , frozen=True): # - (incrementally) update ``QPainterPath`` if ( - view_range + use_vr # and not self._in_ds # and not prepend_length > 0 ): + + # if a view range is passed, plan to draw the + # source ouput that's "in view" of the chart. + view_range = (ivl, ivr) # print(f'{self._name} vr: {view_range}') # by default we only pull data up to the last (current) index @@ -1096,8 +1092,9 @@ class Flow(msgspec.Struct): # , frozen=True): # **kwargs # ) - graphics.draw_last(x, y) - profiler('draw last segment') + if draw_last: + graphics.draw_last(x, y) + profiler('draw last segment') graphics.update() profiler('.update()') From b5b9ecf4b1f3a19734e5654c614f26c92b3a3b29 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 08:53:35 -0400 Subject: [PATCH 064/113] Treat paths like input/output vars --- piker/ui/_flows.py | 48 ++++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index b0f1bb21..a9c3f3d5 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -843,6 +843,9 @@ class Flow(msgspec.Struct): # , frozen=True): # - determine downsampling ops if needed # - (incrementally) update ``QPainterPath`` + path = graphics.path + fast_path = graphics.fast_path + if ( use_vr # and not self._in_ds @@ -940,21 +943,19 @@ class Flow(msgspec.Struct): # , frozen=True): should_ds = False showing_src_data = True - # no_path_yet = self.path is None - fast_path = graphics.fast_path if ( - graphics.path is None + path is None or should_redraw or new_sample_rate or prepend_length > 0 ): if should_redraw: - if graphics.path: - graphics.path.clear() + if path: + path.clear() profiler('cleared paths due to `should_redraw=True`') - if graphics.fast_path: - graphics.fast_path.clear() + if fast_path: + fast_path.clear() profiler('cleared paths due to `should_redraw` set') @@ -976,12 +977,12 @@ class Flow(msgspec.Struct): # , frozen=True): profiler(f'FULL PATH downsample redraw={should_ds}') self._in_ds = True - graphics.path = pg.functions.arrayToQPath( + path = pg.functions.arrayToQPath( x_out, y_out, connect='all', finiteCheck=False, - path=graphics.path, + path=path, ) graphics.prepareGeometryChange() profiler( @@ -1047,7 +1048,7 @@ class Flow(msgspec.Struct): # , frozen=True): new_y, connect='all', finiteCheck=False, - path=graphics.fast_path, + path=fast_path, ) profiler('generated append qpath') @@ -1055,8 +1056,8 @@ class Flow(msgspec.Struct): # , frozen=True): print("USING FPATH") # an attempt at trying to make append-updates faster.. if fast_path is None: - graphics.fast_path = append_path - # self.fast_path.reserve(int(6e3)) + fast_path = append_path + # fast_path.reserve(int(6e3)) else: fast_path.connectPath(append_path) size = fast_path.capacity() @@ -1073,25 +1074,6 @@ class Flow(msgspec.Struct): # , frozen=True): profiler(f'connected history path w size: {size}') graphics.path.connectPath(append_path) - # graphics.update_from_array( - # x=x, - # y=y, - - # x_iv=x_iv, - # y_iv=y_iv, - - # view_range=(ivl, ivr) if use_vr else None, - - # # NOTE: already passed through by display loop. - # # do_append=uppx < 16, - # do_append=do_append, - - # slice_to_head=slice_to_head, - # should_redraw=should_redraw, - # profiler=profiler, - # **kwargs - # ) - if draw_last: graphics.draw_last(x, y) profiler('draw last segment') @@ -1099,6 +1081,10 @@ class Flow(msgspec.Struct): # , frozen=True): graphics.update() profiler('.update()') + # assign output paths to graphicis obj + graphics.path = path + graphics.fast_path = fast_path + profiler('`graphics.update_from_array()` complete') return graphics From b3ae562e4f43d4b72c9e9fa270dfdef3fc2a1781 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 09:08:08 -0400 Subject: [PATCH 065/113] Fully drop `.update_from_array()` --- piker/ui/_curve.py | 341 +-------------------------------------------- 1 file changed, 7 insertions(+), 334 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 9e1f684a..965d682c 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -84,8 +84,8 @@ class FastAppendCurve(pg.GraphicsObject): # brutaaalll, see comments within.. self.yData = None self.xData = None - self._vr: Optional[tuple] = None - self._avr: Optional[tuple] = None + # self._vr: Optional[tuple] = None + # self._avr: Optional[tuple] = None self._last_cap: int = 0 self._name = name @@ -99,12 +99,12 @@ class FastAppendCurve(pg.GraphicsObject): super().__init__(*args, **kwargs) # self._xrange: tuple[int, int] = self.dataBounds(ax=0) - self._xrange: Optional[tuple[int, int]] = None + # self._xrange: Optional[tuple[int, int]] = None # self._x_iv_range = None # self._last_draw = time.time() - self._in_ds: bool = False - self._last_uppx: float = 0 + # self._in_ds: bool = False + # self._last_uppx: float = 0 # all history of curve is drawn in single px thickness pen = pg.mkPen(hcolor(color)) @@ -161,8 +161,8 @@ class FastAppendCurve(pg.GraphicsObject): vr = self.viewRect() l, r = int(vr.left()), int(vr.right()) - if not self._xrange: - return 0 + # if not self._xrange: + # return 0 start, stop = self._xrange lbar = max(l, start) @@ -172,333 +172,6 @@ class FastAppendCurve(pg.GraphicsObject): QLineF(lbar, 0, rbar, 0) ).length() - # def update_from_array( - # self, - - # # full array input history - # x: np.ndarray, - # y: np.ndarray, - - # # pre-sliced array data that's "in view" - # x_iv: np.ndarray, - # y_iv: np.ndarray, - - # view_range: Optional[tuple[int, int]] = None, - # profiler: Optional[pg.debug.Profiler] = None, - # draw_last: bool = True, - # slice_to_head: int = -1, - # do_append: bool = True, - # should_redraw: bool = False, - - # ) -> QtGui.QPainterPath: - # ''' - # Update curve from input 2-d data. - - # Compare with a cached "x-range" state and (pre/a)ppend based on - # a length diff. - - # ''' - # profiler = profiler or pg.debug.Profiler( - # msg=f'FastAppendCurve.update_from_array(): `{self._name}`', - # disabled=not pg_profile_enabled(), - # ms_threshold=ms_slower_then, - # ) - # # flip_cache = False - - # if self._xrange: - # istart, istop = self._xrange - # else: - # self._xrange = istart, istop = x[0], x[-1] - - # # compute the length diffs between the first/last index entry in - # # the input data and the last indexes we have on record from the - # # last time we updated the curve index. - # prepend_length = int(istart - x[0]) - # append_length = int(x[-1] - istop) - - # # this is the diff-mode, "data"-rendered index - # # tracking var.. - # self._xrange = x[0], x[-1] - - # # print(f"xrange: {self._xrange}") - - # # XXX: lol brutal, the internals of `CurvePoint` (inherited by - # # our `LineDot`) required ``.getData()`` to work.. - # self.xData = x - # self.yData = y - - # # downsampling incremental state checking - # uppx = self.x_uppx() - # px_width = self.px_width() - # uppx_diff = (uppx - self._last_uppx) - - # new_sample_rate = False - # should_ds = self._in_ds - # showing_src_data = self._in_ds - # # should_redraw = False - - # # by default we only pull data up to the last (current) index - # x_out = x[:slice_to_head] - # y_out = y[:slice_to_head] - - # # if a view range is passed, plan to draw the - # # source ouput that's "in view" of the chart. - # if ( - # view_range - # # and not self._in_ds - # # and not prepend_length > 0 - # ): - # # print(f'{self._name} vr: {view_range}') - - # # by default we only pull data up to the last (current) index - # x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head] - # profiler(f'view range slice {view_range}') - - # vl, vr = view_range - - # # last_ivr = self._x_iv_range - # # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) - - # zoom_or_append = False - # last_vr = self._vr - # last_ivr = self._avr - - # if last_vr: - # # relative slice indices - # lvl, lvr = last_vr - # # abs slice indices - # al, ar = last_ivr - - # # append_length = int(x[-1] - istop) - # # append_length = int(x_iv[-1] - ar) - - # # left_change = abs(x_iv[0] - al) >= 1 - # # right_change = abs(x_iv[-1] - ar) >= 1 - - # if ( - # # likely a zoom view change - # (vr - lvr) > 2 or vl < lvl - # # append / prepend update - # # we had an append update where the view range - # # didn't change but the data-viewed (shifted) - # # underneath, so we need to redraw. - # # or left_change and right_change and last_vr == view_range - - # # not (left_change and right_change) and ivr - # # ( - # # or abs(x_iv[ivr] - livr) > 1 - # ): - # zoom_or_append = True - - # # if last_ivr: - # # liivl, liivr = last_ivr - - # if ( - # view_range != last_vr - # and ( - # append_length > 1 - # or zoom_or_append - # ) - # ): - # should_redraw = True - # # print("REDRAWING BRUH") - - # self._vr = view_range - # self._avr = x_iv[0], x_iv[slice_to_head] - - # # x_last = x_iv[-1] - # # y_last = y_iv[-1] - # # self._last_vr = view_range - - # # self.disable_cache() - # # flip_cache = True - - # if prepend_length > 0: - # should_redraw = True - - # # check for downsampling conditions - # if ( - # # std m4 downsample conditions - # abs(uppx_diff) >= 1 - # ): - # log.info( - # f'{self._name} sampler change: {self._last_uppx} -> {uppx}' - # ) - # self._last_uppx = uppx - # new_sample_rate = True - # showing_src_data = False - # should_redraw = True - # should_ds = True - - # elif ( - # uppx <= 2 - # and self._in_ds - # ): - # # we should de-downsample back to our original - # # source data so we clear our path data in prep - # # to generate a new one from original source data. - # should_redraw = True - # new_sample_rate = True - # should_ds = False - # showing_src_data = True - - # # no_path_yet = self.path is None - # if ( - # self.path is None - # or should_redraw - # or new_sample_rate - # or prepend_length > 0 - # ): - # if should_redraw: - # if self.path: - # self.path.clear() - # profiler('cleared paths due to `should_redraw=True`') - - # if self.fast_path: - # self.fast_path.clear() - - # profiler('cleared paths due to `should_redraw` set') - - # if new_sample_rate and showing_src_data: - # # if self._in_ds: - # log.info(f'DEDOWN -> {self._name}') - - # self._in_ds = False - - # elif should_ds and uppx > 1: - - # x_out, y_out = xy_downsample( - # x_out, - # y_out, - # uppx, - # ) - # profiler(f'FULL PATH downsample redraw={should_ds}') - # self._in_ds = True - - # self.path = pg.functions.arrayToQPath( - # x_out, - # y_out, - # connect='all', - # finiteCheck=False, - # path=self.path, - # ) - # self.prepareGeometryChange() - # profiler( - # 'generated fresh path. ' - # f'(should_redraw: {should_redraw} ' - # f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' - # ) - # # profiler(f'DRAW PATH IN VIEW -> {self._name}') - - # # reserve mem allocs see: - # # - https://doc.qt.io/qt-5/qpainterpath.html#reserve - # # - https://doc.qt.io/qt-5/qpainterpath.html#capacity - # # - https://doc.qt.io/qt-5/qpainterpath.html#clear - # # XXX: right now this is based on had hoc checks on a - # # hidpi 3840x2160 4k monitor but we should optimize for - # # the target display(s) on the sys. - # # if no_path_yet: - # # self.path.reserve(int(500e3)) - - # # TODO: get this piecewise prepend working - right now it's - # # giving heck on vwap... - # # elif prepend_length: - # # breakpoint() - - # # prepend_path = pg.functions.arrayToQPath( - # # x[0:prepend_length], - # # y[0:prepend_length], - # # connect='all' - # # ) - - # # # swap prepend path in "front" - # # old_path = self.path - # # self.path = prepend_path - # # # self.path.moveTo(new_x[0], new_y[0]) - # # self.path.connectPath(old_path) - - # elif ( - # append_length > 0 - # and do_append - # and not should_redraw - # # and not view_range - # ): - # print(f'{self._name} append len: {append_length}') - # new_x = x[-append_length - 2:slice_to_head] - # new_y = y[-append_length - 2:slice_to_head] - # profiler('sliced append path') - - # profiler( - # f'diffed array input, append_length={append_length}' - # ) - - # # if should_ds: - # # new_x, new_y = xy_downsample( - # # new_x, - # # new_y, - # # uppx, - # # ) - # # profiler(f'fast path downsample redraw={should_ds}') - - # append_path = pg.functions.arrayToQPath( - # new_x, - # new_y, - # connect='all', - # finiteCheck=False, - # path=self.fast_path, - # ) - # profiler('generated append qpath') - - # if self.use_fpath: - # # an attempt at trying to make append-updates faster.. - # if self.fast_path is None: - # self.fast_path = append_path - # # self.fast_path.reserve(int(6e3)) - # else: - # self.fast_path.connectPath(append_path) - # size = self.fast_path.capacity() - # profiler(f'connected fast path w size: {size}') - - # # print(f"append_path br: {append_path.boundingRect()}") - # # self.path.moveTo(new_x[0], new_y[0]) - # # path.connectPath(append_path) - - # # XXX: lol this causes a hang.. - # # self.path = self.path.simplified() - # else: - # size = self.path.capacity() - # profiler(f'connected history path w size: {size}') - # self.path.connectPath(append_path) - - # # other merging ideas: - # # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths - # # path.addPath(append_path) - # # path.closeSubpath() - - # # TODO: try out new work from `pyqtgraph` main which - # # should repair horrid perf: - # # https://github.com/pyqtgraph/pyqtgraph/pull/2032 - # # ok, nope still horrible XD - # # if self._fill: - # # # XXX: super slow set "union" op - # # self.path = self.path.united(append_path).simplified() - - # # self.disable_cache() - # # flip_cache = True - - # # if draw_last: - # # self.draw_last(x, y) - # # profiler('draw last segment') - - # # if flip_cache: - # # # # XXX: seems to be needed to avoid artifacts (see above). - # # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - - # # trigger redraw of path - # # do update before reverting to cache mode - # self.update() - # profiler('.update()') - def draw_last( self, x: np.ndarray, From 72e849c6518c77365ab34ef12b01e60f577286a3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 09:08:38 -0400 Subject: [PATCH 066/113] Drop commented cruft from update logic --- piker/ui/_flows.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index a9c3f3d5..bae4b5a9 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -825,14 +825,6 @@ class Flow(msgspec.Struct): # , frozen=True): should_redraw = bool(append_diff) draw_last = False - # graphics.reset_cache() - # print( - # f"path br: {graphics.path.boundingRect()}\n", - # # f"fast path br: {graphics.fast_path.boundingRect()}", - # f"last rect br: {graphics._last_step_rect}\n", - # f"full br: {graphics._br}\n", - # ) - # compute the length diffs between the first/last index entry in # the input data and the last indexes we have on record from the # last time we updated the curve index. @@ -848,8 +840,6 @@ class Flow(msgspec.Struct): # , frozen=True): if ( use_vr - # and not self._in_ds - # and not prepend_length > 0 ): # if a view range is passed, plan to draw the @@ -875,9 +865,6 @@ class Flow(msgspec.Struct): # , frozen=True): # abs slice indices al, ar = last_ivr - # append_length = int(x[-1] - istop) - # append_length = int(x_iv[-1] - ar) - # left_change = abs(x_iv[0] - al) >= 1 # right_change = abs(x_iv[-1] - ar) >= 1 @@ -896,9 +883,6 @@ class Flow(msgspec.Struct): # , frozen=True): ): zoom_or_append = True - # if last_ivr: - # liivl, liivr = last_ivr - if ( view_range != last_vr and ( @@ -915,11 +899,8 @@ class Flow(msgspec.Struct): # , frozen=True): if prepend_length > 0: should_redraw = True - # check for downsampling conditions + # check for and set std m4 downsample conditions if ( - # std m4 downsample conditions - # px_width - # and abs(uppx_diff) >= 1 abs(uppx_diff) >= 1 ): log.info( @@ -965,14 +946,12 @@ class Flow(msgspec.Struct): # , frozen=True): self._in_ds = False - # elif should_ds and uppx and px_width > 1: elif should_ds and uppx > 1: x_out, y_out = xy_downsample( x_out, y_out, uppx, - # px_width, ) profiler(f'FULL PATH downsample redraw={should_ds}') self._in_ds = True @@ -1023,7 +1002,6 @@ class Flow(msgspec.Struct): # , frozen=True): append_length > 0 and do_append and not should_redraw - # and not view_range ): print(f'{self.name} append len: {append_length}') new_x = x[-append_length - 2:slice_to_head] From 876add4fc21dad2def1b23e145f6d838a0622d4f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 15:16:01 -0400 Subject: [PATCH 067/113] Drop `.update()` call from `.draw_last()` --- piker/ui/_curve.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 965d682c..fa073d37 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -207,8 +207,6 @@ class FastAppendCurve(pg.GraphicsObject): x_last, y_last ) - self.update() - # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. def getData(self): From 664a208ae51ac1edba35b27d2701c09efc10456b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 15:17:48 -0400 Subject: [PATCH 068/113] Drop path generation from `gen_ohlc_qpath()` --- piker/ui/_pathops.py | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index f7eaf2a7..4cb5b86e 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -33,7 +33,6 @@ from ..data._sharedmem import ( ) from .._profile import pg_profile_enabled, ms_slower_then from ._compression import ( - # ohlc_flatten, ds_m4, ) @@ -140,43 +139,24 @@ def path_arrays_from_ohlc( def gen_ohlc_qpath( data: np.ndarray, + array_key: str, # we ignore this + start: int = 0, # XXX: do we need this? # 0.5 is no overlap between arms, 1.0 is full overlap w: float = 0.43, - path: Optional[QtGui.QPainterPath] = None, ) -> QtGui.QPainterPath: + ''' + More or less direct proxy to ``path_arrays_from_ohlc()`` + but with closed in kwargs for line spacing. - path_was_none = path is None - - profiler = pg.debug.Profiler( - msg='gen_qpath ohlc', - disabled=not pg_profile_enabled(), - ms_threshold=ms_slower_then, - ) - + ''' x, y, c = path_arrays_from_ohlc( data, start, bar_gap=w, ) - profiler("generate stream with numba") - - # TODO: numba the internals of this! - path = pg.functions.arrayToQPath( - x, - y, - connect=c, - path=path, - ) - - # avoid mem allocs if possible - if path_was_none: - path.reserve(path.capacity()) - - profiler("generate path with arrayToQPath") - - return path + return x, y, c def ohlc_to_line( From aa0efe15230f3240b9fed0b81a9d52b6ee9949e0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 15:21:21 -0400 Subject: [PATCH 069/113] Drop `BarItems.draw_from_data()` --- piker/ui/_chart.py | 5 ----- piker/ui/_ohlc.py | 56 ---------------------------------------------- 2 files changed, 61 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 2eba9a24..31ca0604 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -979,11 +979,6 @@ class ChartPlotWidget(pg.PlotWidget): graphics=graphics, ) - # TODO: i think we can eventually remove this if - # we write the ``Flow.update_graphics()`` method right? - # draw after to allow self.scene() to work... - graphics.draw_from_data(shm.array) - self._add_sticky(name, bg_color='davies') return graphics, data_key diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 88fa62f9..fb57d6ff 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -146,62 +146,6 @@ class BarItems(pg.GraphicsObject): self._dsi: tuple[int, int] = 0, 0 self._xs_in_px: float = 0 - def draw_from_data( - self, - ohlc: np.ndarray, - start: int = 0, - - ) -> QtGui.QPainterPath: - ''' - Draw OHLC datum graphics from a ``np.ndarray``. - - This routine is usually only called to draw the initial history. - - ''' - hist, last = ohlc[:-1], ohlc[-1] - self.path = gen_ohlc_qpath(hist, start, self.w) - - # save graphics for later reference and keep track - # of current internal "last index" - # self.start_index = len(ohlc) - index = ohlc['index'] - self._xrange = (index[0], index[-1]) - # self._yrange = ( - # np.nanmax(ohlc['high']), - # np.nanmin(ohlc['low']), - # ) - - # up to last to avoid double draw of last bar - self._last_bar_lines = bar_from_ohlc_row(last, self.w) - - x, y = self._ds_line_xy = ohlc_flatten(ohlc) - - # TODO: figuring out the most optimial size for the ideal - # curve-path by, - # - calcing the display's max px width `.screen()` - # - drawing a curve and figuring out it's capacity: - # https://doc.qt.io/qt-5/qpainterpath.html#capacity - # - reserving that cap for each curve-mapped-to-shm with - - # - leveraging clearing when needed to redraw the entire - # curve that does not release mem allocs: - # https://doc.qt.io/qt-5/qpainterpath.html#clear - curve = FastAppendCurve( - name='OHLC', - color=self._color, - ) - curve.hide() - self._pi.addItem(curve) - self._ds_line = curve - - # self._ds_xrange = (index[0], index[-1]) - - # trigger render - # https://doc.qt.io/qt-5/qgraphicsitem.html#update - self.update() - - return self.path - def x_uppx(self) -> int: if self._ds_line: return self._ds_line.x_uppx() From 167ae965665ee132a484eaccc90c49bd8bd743f3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 15:23:14 -0400 Subject: [PATCH 070/113] Move graphics update logic into `Renderer.render()` Finally this gets us much closer to a generic incremental update system for graphics wherein the input array diffing, pre-graphical format data processing, downsampler activation and incremental update and storage of any of these data flow stages can be managed in one modular sub-system :surfer_boi:. Dirty deatz: - reorg and move all path logic into `Renderer.render()` and have it take in pretty much the same flags as the old `FastAppendCurve.update_from_array()` and instead storing all update state vars (even copies of the downsampler related ones) on the renderer instance: - new state vars: `._last_uppx, ._in_ds, ._vr, ._avr` - `.render()` input bools: `new_sample_rate, should_redraw, should_ds, showing_src_data` - add a hack-around for passing in incremental update data (for now) via a `input_data: tuple` of numpy arrays - a default `uppx: float = 1` - add new render interface attrs: - `.format_xy()` which takes in the source data array and produces out x, y arrays (and maybe a `connect` array) that can be passed to `.draw_path()` (the default for this is just to slice out the index and `array_key: str` columns from the input struct array), - `.draw_path()` which takes in the x, y, connect arrays and generates a `QPainterPath` - `.fast_path`, for "appendable" updates like there was on the fast append curve - move redraw (aka `.clear()` calls) into `.draw_path()` and trigger via `redraw: bool` flag. - our graphics objects no longer set their own `.path` state, it's done by the `Flow.update_graphics()` method using output from `Renderer.render()` (and it's state if necessary) --- piker/ui/_flows.py | 737 ++++++++++++++++++++++++++++----------------- 1 file changed, 458 insertions(+), 279 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index bae4b5a9..c974878d 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -27,6 +27,7 @@ from __future__ import annotations from typing import ( Optional, Callable, + Union, ) import msgspec @@ -144,12 +145,13 @@ def render_baritems( # if no source data renderer exists create one. self = flow r = self._src_r + show_bars: bool = False if not r: + show_bars = True # OHLC bars path renderer r = self._src_r = Renderer( flow=self, - # TODO: rename this to something with ohlc - draw_path=gen_ohlc_qpath, + format_xy=gen_ohlc_qpath, last_read=read, ) @@ -292,20 +294,28 @@ def render_baritems( # # do_append=uppx < 16, # **kwargs, # ) - # curve.draw_last(x, y) + curve.draw_last(x, y) curve.show() profiler('updated ds curve') else: # render incremental or in-view update # and apply ouput (path) to graphics. - path, last = r.render( + path, data = r.render( read, - only_in_view=True, + 'ohlc', + profiler=profiler, + # uppx=1, + use_vr=True, + # graphics=graphics, + # should_redraw=True, # always ) + assert path graphics.path = path - graphics.draw_last(last) + graphics.draw_last(data[-1]) + if show_bars: + graphics.show() # NOTE: on appends we used to have to flip the coords # cache thought it doesn't seem to be required any more? @@ -699,6 +709,7 @@ class Flow(msgspec.Struct): # , frozen=True): xfirst, xlast, array, ivl, ivr, in_view, ) = self.read() + profiler('read src shm data') graphics = self.graphics @@ -709,8 +720,13 @@ class Flow(msgspec.Struct): # , frozen=True): ): return graphics + draw_last: bool = True + slice_to_head: int = -1 + input_data = None + out: Optional[tuple] = None if isinstance(graphics, BarItems): + draw_last = False # XXX: special case where we change out graphics # to a line after a certain uppx threshold. # render_baritems( @@ -741,14 +757,6 @@ class Flow(msgspec.Struct): # , frozen=True): array_key = array_key or self.name shm = self.shm - # update config - new_sample_rate = False - should_ds = self._in_ds - showing_src_data = self._in_ds - draw_last: bool = True - slice_to_head: int = -1 - should_redraw: bool = False - if out is not None: # hack to handle ds curve from bars above ( @@ -758,32 +766,60 @@ class Flow(msgspec.Struct): # , frozen=True): x_iv, y_iv, ) = out + input_data = out[1:] + # breakpoint() - else: + # ds update config + new_sample_rate: bool = False + should_redraw: bool = False + should_ds: bool = r._in_ds + showing_src_data: bool = not r._in_ds + + # downsampling incremental state checking + # check for and set std m4 downsample conditions + uppx = graphics.x_uppx() + uppx_diff = (uppx - self._last_uppx) + profiler(f'diffed uppx {uppx}') + if ( + uppx > 1 + and abs(uppx_diff) >= 1 + ): + log.info( + f'{array_key} sampler change: {self._last_uppx} -> {uppx}' + ) + self._last_uppx = uppx + new_sample_rate = True + showing_src_data = False + should_redraw = True + should_ds = True + + elif ( + uppx <= 2 + and self._in_ds + ): + # we should de-downsample back to our original + # source data so we clear our path data in prep + # to generate a new one from original source data. + should_redraw = True + new_sample_rate = True + should_ds = False + showing_src_data = True + + if graphics._step_mode: + slice_to_head = -2 + + # TODO: remove this and instead place all step curve + # updating into pre-path data render callbacks. # full input data x = array['index'] y = array[array_key] + x_last = x[-1] + y_last = y[-1] # inview data x_iv = in_view['index'] y_iv = in_view[array_key] - # downsampling incremental state checking - uppx = graphics.x_uppx() - # px_width = graphics.px_width() - uppx_diff = (uppx - self._last_uppx) - profiler(f'diffed uppx {uppx}') - - x_last = x[-1] - y_last = y[-1] - - slice_to_head = -1 - - profiler('sliced input arrays') - - if graphics._step_mode: - slice_to_head = -2 - if self.gy is None: ( self._iflat_first, @@ -824,261 +860,104 @@ class Flow(msgspec.Struct): # , frozen=True): should_redraw = bool(append_diff) draw_last = False + input_data = ( + x, + y, + x_iv, + y_iv, + ) # compute the length diffs between the first/last index entry in # the input data and the last indexes we have on record from the # last time we updated the curve index. - prepend_length, append_length = r.diff(read) + # prepend_length, append_length = r.diff(read) # MAIN RENDER LOGIC: # - determine in view data and redraw on range change # - determine downsampling ops if needed # - (incrementally) update ``QPainterPath`` - path = graphics.path - fast_path = graphics.fast_path + # path = graphics.path + # fast_path = graphics.fast_path - if ( - use_vr - ): + path, data = r.render( + read, + array_key, + profiler, + uppx=uppx, + input_data=input_data, + # use_vr=True, - # if a view range is passed, plan to draw the - # source ouput that's "in view" of the chart. - view_range = (ivl, ivr) - # print(f'{self._name} vr: {view_range}') + # TODO: better way to detect and pass this? + # if we want to eventually cache renderers for a given uppx + # we should probably use this as a key + state? + should_redraw=should_redraw, + new_sample_rate=new_sample_rate, + should_ds=should_ds, + showing_src_data=showing_src_data, - # by default we only pull data up to the last (current) index - x_out = x_iv[:slice_to_head] - y_out = y_iv[:slice_to_head] - profiler(f'view range slice {view_range}') - - vl, vr = view_range - - zoom_or_append = False - last_vr = self._vr - last_ivr = self._avr - - # incremental in-view data update. - if last_vr: - # relative slice indices - lvl, lvr = last_vr - # abs slice indices - al, ar = last_ivr - - # left_change = abs(x_iv[0] - al) >= 1 - # right_change = abs(x_iv[-1] - ar) >= 1 - - if ( - # likely a zoom view change - (vr - lvr) > 2 or vl < lvl - # append / prepend update - # we had an append update where the view range - # didn't change but the data-viewed (shifted) - # underneath, so we need to redraw. - # or left_change and right_change and last_vr == view_range - - # not (left_change and right_change) and ivr - # ( - # or abs(x_iv[ivr] - livr) > 1 - ): - zoom_or_append = True - - if ( - view_range != last_vr - and ( - append_length > 1 - or zoom_or_append - ) - ): - should_redraw = True - # print("REDRAWING BRUH") - - self._vr = view_range - self._avr = x_iv[0], x_iv[slice_to_head] - - if prepend_length > 0: - should_redraw = True - - # check for and set std m4 downsample conditions - if ( - abs(uppx_diff) >= 1 - ): - log.info( - f'{array_key} sampler change: {self._last_uppx} -> {uppx}' - ) - self._last_uppx = uppx - new_sample_rate = True - showing_src_data = False - should_redraw = True - should_ds = True - - elif ( - uppx <= 2 - and self._in_ds - ): - # we should de-downsample back to our original - # source data so we clear our path data in prep - # to generate a new one from original source data. - should_redraw = True - new_sample_rate = True - should_ds = False - showing_src_data = True - - if ( - path is None - or should_redraw - or new_sample_rate - or prepend_length > 0 - ): - if should_redraw: - if path: - path.clear() - profiler('cleared paths due to `should_redraw=True`') - - if fast_path: - fast_path.clear() - - profiler('cleared paths due to `should_redraw` set') - - if new_sample_rate and showing_src_data: - # if self._in_ds: - log.info(f'DEDOWN -> {self.name}') - - self._in_ds = False - - elif should_ds and uppx > 1: - - x_out, y_out = xy_downsample( - x_out, - y_out, - uppx, - ) - profiler(f'FULL PATH downsample redraw={should_ds}') - self._in_ds = True - - path = pg.functions.arrayToQPath( - x_out, - y_out, - connect='all', - finiteCheck=False, - path=path, - ) - graphics.prepareGeometryChange() - profiler( - 'generated fresh path. ' - f'(should_redraw: {should_redraw} ' - f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' - ) - # profiler(f'DRAW PATH IN VIEW -> {self.name}') - - # reserve mem allocs see: - # - https://doc.qt.io/qt-5/qpainterpath.html#reserve - # - https://doc.qt.io/qt-5/qpainterpath.html#capacity - # - https://doc.qt.io/qt-5/qpainterpath.html#clear - # XXX: right now this is based on had hoc checks on a - # hidpi 3840x2160 4k monitor but we should optimize for - # the target display(s) on the sys. - # if no_path_yet: - # graphics.path.reserve(int(500e3)) - - # TODO: get this piecewise prepend working - right now it's - # giving heck on vwap... - # elif prepend_length: - # breakpoint() - - # prepend_path = pg.functions.arrayToQPath( - # x[0:prepend_length], - # y[0:prepend_length], - # connect='all' - # ) - - # # swap prepend path in "front" - # old_path = graphics.path - # graphics.path = prepend_path - # # graphics.path.moveTo(new_x[0], new_y[0]) - # graphics.path.connectPath(old_path) - - elif ( - append_length > 0 - and do_append - and not should_redraw - ): - print(f'{self.name} append len: {append_length}') - new_x = x[-append_length - 2:slice_to_head] - new_y = y[-append_length - 2:slice_to_head] - profiler('sliced append path') - - profiler( - f'diffed array input, append_length={append_length}' - ) - - # if should_ds: - # new_x, new_y = xy_downsample( - # new_x, - # new_y, - # px_width, - # uppx, - # ) - # profiler(f'fast path downsample redraw={should_ds}') - - append_path = pg.functions.arrayToQPath( - new_x, - new_y, - connect='all', - finiteCheck=False, - path=fast_path, - ) - profiler('generated append qpath') - - if graphics.use_fpath: - print("USING FPATH") - # an attempt at trying to make append-updates faster.. - if fast_path is None: - fast_path = append_path - # fast_path.reserve(int(6e3)) - else: - fast_path.connectPath(append_path) - size = fast_path.capacity() - profiler(f'connected fast path w size: {size}') - - # print(f"append_path br: {append_path.boundingRect()}") - # graphics.path.moveTo(new_x[0], new_y[0]) - # path.connectPath(append_path) - - # XXX: lol this causes a hang.. - # graphics.path = graphics.path.simplified() - else: - size = graphics.path.capacity() - profiler(f'connected history path w size: {size}') - graphics.path.connectPath(append_path) + slice_to_head=slice_to_head, + do_append=do_append, + graphics=graphics, + ) + # graphics.prepareGeometryChange() + # assign output paths to graphicis obj + graphics.path = r.path + graphics.fast_path = r.fast_path if draw_last: + x = data['index'] + y = data[array_key] graphics.draw_last(x, y) profiler('draw last segment') graphics.update() profiler('.update()') - # assign output paths to graphicis obj - graphics.path = path - graphics.fast_path = fast_path - profiler('`graphics.update_from_array()` complete') return graphics +def by_index_and_key( + array: np.ndarray, + array_key: str, + +) -> tuple[ + np.ndarray, + np.ndarray, + np.ndarray, +]: + # full input data + x = array['index'] + y = array[array_key] + + # # inview data + # x_iv = in_view['index'] + # y_iv = in_view[array_key] + + return tuple({ + 'x': x, + 'y': y, + # 'x_iv': x_iv, + # 'y_iv': y_iv, + 'connect': 'all', + }.values()) + + class Renderer(msgspec.Struct): flow: Flow # last array view read last_read: Optional[tuple] = None + format_xy: Callable[np.ndarray, tuple[np.ndarray]] = by_index_and_key # called to render path graphics - draw_path: Optional[Callable[np.ndarray, QPainterPath]] = None + # draw_path: Optional[Callable[np.ndarray, QPainterPath]] = None # output graphics rendering, the main object # processed in ``QGraphicsObject.paint()`` path: Optional[QPainterPath] = None + fast_path: Optional[QPainterPath] = None # called on input data but before any graphics format # conversions or processing. @@ -1095,12 +974,18 @@ class Renderer(msgspec.Struct): prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None + # downsampling state + _last_uppx: float = 0 + _in_ds: bool = False + # incremental update state(s) - # _in_ds: bool = False - # _last_uppx: float = 0 _last_vr: Optional[tuple[float, float]] = None _last_ivr: Optional[tuple[float, float]] = None + # view-range incremental state + _vr: Optional[tuple] = None + _avr: Optional[tuple] = None + def diff( self, new_read: tuple[np.ndarray], @@ -1146,23 +1031,80 @@ class Renderer(msgspec.Struct): # last, ) + # def gen_path_data( + # self, + # redraw: bool = False, + # ) -> np.ndarray: + # ... + def draw_path( self, - should_redraw: bool = False, + x: np.ndarray, + y: np.ndarray, + connect: Union[str, np.ndarray] = 'all', + path: Optional[QPainterPath] = None, + redraw: bool = False, + ) -> QPainterPath: - if should_redraw: - if self.path: - self.path.clear() - # profiler('cleared paths due to `should_redraw=True`') + path_was_none = path is None + + if redraw and path: + path.clear() + + # TODO: avoid this? + if self.fast_path: + self.fast_path.clear() + + # profiler('cleared paths due to `should_redraw=True`') + + path = pg.functions.arrayToQPath( + x, + y, + connect=connect, + finiteCheck=False, + + # reserve mem allocs see: + # - https://doc.qt.io/qt-5/qpainterpath.html#reserve + # - https://doc.qt.io/qt-5/qpainterpath.html#capacity + # - https://doc.qt.io/qt-5/qpainterpath.html#clear + # XXX: right now this is based on had hoc checks on a + # hidpi 3840x2160 4k monitor but we should optimize for + # the target display(s) on the sys. + # if no_path_yet: + # graphics.path.reserve(int(500e3)) + path=path, # path re-use / reserving + ) + + # avoid mem allocs if possible + if path_was_none: + path.reserve(path.capacity()) + + return path def render( self, new_read, + array_key: str, + profiler: pg.debug.Profiler, + uppx: float = 1, + + input_data: Optional[tuple[np.ndarray]] = None, + + # redraw and ds flags + should_redraw: bool = True, + new_sample_rate: bool = False, + should_ds: bool = False, + showing_src_data: bool = True, + + do_append: bool = True, + slice_to_head: int = -1, + use_fpath: bool = True, # only render datums "in view" of the ``ChartView`` - only_in_view: bool = False, + use_vr: bool = True, + graphics: Optional[pg.GraphicObject] = None, ) -> list[QPainterPath]: ''' @@ -1177,8 +1119,6 @@ class Renderer(msgspec.Struct): - blah blah blah (from notes) ''' - # get graphics info - # TODO: can the renderer just call ``Flow.read()`` directly? # unpack latest source data read ( @@ -1190,29 +1130,268 @@ class Renderer(msgspec.Struct): in_view, ) = new_read + if use_vr: + array = in_view + + if input_data: + # allow input data passing for now from alt curve updaters. + ( + x_out, + y_out, + x_iv, + y_iv, + ) = input_data + connect = 'all' + + if use_vr: + x_out = x_iv + y_out = y_iv + + # last = y_out[slice_to_head] + + else: + hist = array[:slice_to_head] + # last = array[slice_to_head] + + ( + x_out, + y_out, + # x_iv, + # y_iv, + connect, + ) = self.format_xy(hist, array_key) + + # print(f'{array_key} len x,y: {(len(x_out), len(y_out))}') +# # full input data +# x = array['index'] +# y = array[array_key] + +# # inview data +# x_iv = in_view['index'] +# y_iv = in_view[array_key] + + profiler('sliced input arrays') + ( prepend_length, append_length, ) = self.diff(new_read) - # do full source data render to path + if ( + use_vr + ): + # if a view range is passed, plan to draw the + # source ouput that's "in view" of the chart. + view_range = (ivl, ivr) + # print(f'{self._name} vr: {view_range}') - # x = array['index'] - # y = array#[array_key] - # x_iv = in_view['index'] - # y_iv = in_view#[array_key] + # by default we only pull data up to the last (current) index + # x_out = x_iv[:slice_to_head] + # y_out = y_iv[:slice_to_head] - if only_in_view: - array = in_view + profiler(f'view range slice {view_range}') + + vl, vr = view_range + + zoom_or_append = False + last_vr = self._vr + last_ivr = self._avr + + # incremental in-view data update. + if last_vr: + # relative slice indices + lvl, lvr = last_vr + # abs slice indices + al, ar = last_ivr + + # left_change = abs(x_iv[0] - al) >= 1 + # right_change = abs(x_iv[-1] - ar) >= 1 + + if ( + # likely a zoom view change + (vr - lvr) > 2 or vl < lvl + # append / prepend update + # we had an append update where the view range + # didn't change but the data-viewed (shifted) + # underneath, so we need to redraw. + # or left_change and right_change and last_vr == view_range + + # not (left_change and right_change) and ivr + # ( + # or abs(x_iv[ivr] - livr) > 1 + ): + zoom_or_append = True + + if ( + view_range != last_vr + and ( + append_length > 1 + or zoom_or_append + ) + ): + should_redraw = True + # print("REDRAWING BRUH") + + self._vr = view_range + if len(x_out): + self._avr = x_out[0], x_out[slice_to_head] + + if prepend_length > 0: + should_redraw = True + + # # last datums + # x_last = x_out[-1] + # y_last = y_out[-1] + + path = self.path + fast_path = self.fast_path + + if ( + path is None + or should_redraw + or new_sample_rate + or prepend_length > 0 + ): + # if should_redraw: + # if path: + # path.clear() + # profiler('cleared paths due to `should_redraw=True`') + + # if fast_path: + # fast_path.clear() + + # profiler('cleared paths due to `should_redraw` set') + + if new_sample_rate and showing_src_data: + # if self._in_ds: + log.info(f'DEDOWN -> {array_key}') + + self._in_ds = False + + elif should_ds and uppx > 1: + + x_out, y_out = xy_downsample( + x_out, + y_out, + uppx, + ) + profiler(f'FULL PATH downsample redraw={should_ds}') + self._in_ds = True + # else: + # print(f"NOT DOWNSAMPLING {array_key}") + + path = self.draw_path( + x=x_out, + y=y_out, + connect=connect, + path=path, + redraw=True, + ) + # path = pg.functions.arrayToQPath( + # x_out, + # y_out, + # connect='all', + # finiteCheck=False, + # path=path, + # ) + if graphics: + graphics.prepareGeometryChange() + + profiler( + 'generated fresh path. ' + f'(should_redraw: {should_redraw} ' + f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' + ) + # profiler(f'DRAW PATH IN VIEW -> {self.name}') + + # TODO: get this piecewise prepend working - right now it's + # giving heck on vwap... + # elif prepend_length: + # breakpoint() + + # prepend_path = pg.functions.arrayToQPath( + # x[0:prepend_length], + # y[0:prepend_length], + # connect='all' + # ) + + # # swap prepend path in "front" + # old_path = graphics.path + # graphics.path = prepend_path + # # graphics.path.moveTo(new_x[0], new_y[0]) + # graphics.path.connectPath(old_path) + + elif ( + append_length > 0 + and do_append + and not should_redraw + ): + # print(f'{self.name} append len: {append_length}') + print(f'{array_key} append len: {append_length}') + new_x = x_out[-append_length - 2:] # slice_to_head] + new_y = y_out[-append_length - 2:] # slice_to_head] + profiler('sliced append path') + + profiler( + f'diffed array input, append_length={append_length}' + ) + + # if should_ds: + # new_x, new_y = xy_downsample( + # new_x, + # new_y, + # uppx, + # ) + # profiler(f'fast path downsample redraw={should_ds}') + + append_path = self.draw_path( + x=new_x, + y=new_y, + connect=connect, + # path=fast_path, + ) + + # append_path = pg.functions.arrayToQPath( + # connect='all', + # finiteCheck=False, + # path=fast_path, + # ) + profiler('generated append qpath') + + # if graphics.use_fpath: + if use_fpath: + print("USING FPATH") + # an attempt at trying to make append-updates faster.. + if fast_path is None: + fast_path = append_path + # fast_path.reserve(int(6e3)) + else: + fast_path.connectPath(append_path) + size = fast_path.capacity() + profiler(f'connected fast path w size: {size}') + + # print(f"append_path br: {append_path.boundingRect()}") + # graphics.path.moveTo(new_x[0], new_y[0]) + # path.connectPath(append_path) + + # XXX: lol this causes a hang.. + # graphics.path = graphics.path.simplified() + else: + size = path.capacity() + profiler(f'connected history path w size: {size}') + path.connectPath(append_path) + + # if use_vr: + # array = in_view # # get latest data from flow shm # self.last_read = ( # xfirst, xlast, array, ivl, ivr, in_view # ) = new_read - if ( - self.path is None - or only_in_view - ): + # if ( + # self.path is None + # or use_vr + # ): # redraw the entire source data if we have either of: # - no prior path graphic rendered or, # - we always intend to re-render the data only in view @@ -1220,8 +1399,8 @@ class Renderer(msgspec.Struct): # data transform: convert source data to a format # expected to be incrementally updates and later rendered # to a more graphics native format. - if self.data_t: - array = self.data_t(array) + # if self.data_t: + # array = self.data_t(array) # maybe allocate shm for data transform output # if self.data_t_shm is None: @@ -1237,18 +1416,18 @@ class Renderer(msgspec.Struct): # shm.push(array) # self.data_t_shm = shm - elif self.path: - print(f'inremental update not supported yet {self.flow.name}') + # elif self.path: + # print(f'inremental update not supported yet {self.flow.name}') # TODO: do incremental update # prepend, append, last = self.diff(self.flow.read()) # do path generation for each segment # and then push into graphics object. - hist, last = array[:-1], array[-1] - # call path render func on history - self.path = self.draw_path(hist) + # self.path = self.draw_path(hist) + self.path = path + self.fast_path = fast_path self.last_read = new_read - return self.path, last + return self.path, array From 17456d96e0d85a425be38658879d35057865b70a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 19 May 2022 10:23:59 -0400 Subject: [PATCH 071/113] Drop tons of old cruft, move around some commented ideas --- piker/ui/_flows.py | 186 ++++++++++++--------------------------------- 1 file changed, 48 insertions(+), 138 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index c974878d..d7c5c2e6 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -510,14 +510,11 @@ class Flow(msgspec.Struct): # , frozen=True): # pre-graphics formatted data gy: Optional[ShmArray] = None gx: Optional[np.ndarray] = None + # pre-graphics update indices _iflat_last: int = 0 _iflat_first: int = 0 - # view-range incremental state - _vr: Optional[tuple] = None - _avr: Optional[tuple] = None - # downsampling state _last_uppx: float = 0 _in_ds: bool = False @@ -551,13 +548,12 @@ class Flow(msgspec.Struct): # , frozen=True): # private ``._shm`` attr? @shm.setter def shm(self, shm: ShmArray) -> ShmArray: - print(f'{self.name} DO NOT SET SHM THIS WAY!?') self._shm = shm def maxmin( self, - lbar, - rbar, + lbar: int, + rbar: int, ) -> tuple[float, float]: ''' @@ -813,8 +809,6 @@ class Flow(msgspec.Struct): # , frozen=True): # full input data x = array['index'] y = array[array_key] - x_last = x[-1] - y_last = y[-1] # inview data x_iv = in_view['index'] @@ -849,6 +843,8 @@ class Flow(msgspec.Struct): # , frozen=True): profiler, ) + x_last = x[-1] + y_last = y[-1] graphics._last_line = QLineF( x_last - 0.5, 0, x_last + 0.5, 0, @@ -898,9 +894,11 @@ class Flow(msgspec.Struct): # , frozen=True): slice_to_head=slice_to_head, do_append=do_append, - graphics=graphics, ) + # TODO: does this actuallly help us in any way (prolly should + # look at the source / ask ogi). # graphics.prepareGeometryChange() + # assign output paths to graphicis obj graphics.path = r.path graphics.fast_path = r.fast_path @@ -931,15 +929,9 @@ def by_index_and_key( x = array['index'] y = array[array_key] - # # inview data - # x_iv = in_view['index'] - # y_iv = in_view[array_key] - return tuple({ 'x': x, 'y': y, - # 'x_iv': x_iv, - # 'y_iv': y_iv, 'connect': 'all', }.values()) @@ -951,9 +943,6 @@ class Renderer(msgspec.Struct): last_read: Optional[tuple] = None format_xy: Callable[np.ndarray, tuple[np.ndarray]] = by_index_and_key - # called to render path graphics - # draw_path: Optional[Callable[np.ndarray, QPainterPath]] = None - # output graphics rendering, the main object # processed in ``QGraphicsObject.paint()`` path: Optional[QPainterPath] = None @@ -961,18 +950,18 @@ class Renderer(msgspec.Struct): # called on input data but before any graphics format # conversions or processing. - data_t: Optional[Callable[ShmArray, np.ndarray]] = None - data_t_shm: Optional[ShmArray] = None + format_data: Optional[Callable[ShmArray, np.ndarray]] = None + # XXX: just ideas.. # called on the final data (transform) output to convert # to "graphical data form" a format that can be passed to # the ``.draw()`` implementation. - graphics_t: Optional[Callable[ShmArray, np.ndarray]] = None - graphics_t_shm: Optional[ShmArray] = None + # graphics_t: Optional[Callable[ShmArray, np.ndarray]] = None + # graphics_t_shm: Optional[ShmArray] = None # path graphics update implementation methods - prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None - append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None + # prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None + # append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None # downsampling state _last_uppx: float = 0 @@ -982,21 +971,20 @@ class Renderer(msgspec.Struct): _last_vr: Optional[tuple[float, float]] = None _last_ivr: Optional[tuple[float, float]] = None - # view-range incremental state - _vr: Optional[tuple] = None - _avr: Optional[tuple] = None - def diff( self, new_read: tuple[np.ndarray], - ) -> tuple[np.ndarray]: - + ) -> tuple[ + np.ndarray, + np.ndarray, + ]: ( last_xfirst, last_xlast, last_array, - last_ivl, last_ivr, + last_ivl, + last_ivr, last_in_view, ) = self.last_read @@ -1028,15 +1016,8 @@ class Renderer(msgspec.Struct): return ( prepend_length, append_length, - # last, ) - # def gen_path_data( - # self, - # redraw: bool = False, - # ) -> np.ndarray: - # ... - def draw_path( self, x: np.ndarray, @@ -1104,7 +1085,6 @@ class Renderer(msgspec.Struct): # only render datums "in view" of the ``ChartView`` use_vr: bool = True, - graphics: Optional[pg.GraphicObject] = None, ) -> list[QPainterPath]: ''' @@ -1153,23 +1133,32 @@ class Renderer(msgspec.Struct): hist = array[:slice_to_head] # last = array[slice_to_head] + # maybe allocate shm for data transform output + # if self.format_data is None: + # fshm = self.flow.shm + + # shm, opened = maybe_open_shm_array( + # f'{self.flow.name}_data_t', + # # TODO: create entry for each time frame + # dtype=array.dtype, + # readonly=False, + # ) + # assert opened + # shm.push(array) + # self.data_t_shm = shm + + # xy-path data transform: convert source data to a format + # able to be passed to a `QPainterPath` rendering routine. + # expected to be incrementally updates and later rendered to + # a more graphics native format. + # if self.data_t: + # array = self.data_t(array) ( x_out, y_out, - # x_iv, - # y_iv, connect, ) = self.format_xy(hist, array_key) - # print(f'{array_key} len x,y: {(len(x_out), len(y_out))}') -# # full input data -# x = array['index'] -# y = array[array_key] - -# # inview data -# x_iv = in_view['index'] -# y_iv = in_view[array_key] - profiler('sliced input arrays') ( @@ -1185,17 +1174,13 @@ class Renderer(msgspec.Struct): view_range = (ivl, ivr) # print(f'{self._name} vr: {view_range}') - # by default we only pull data up to the last (current) index - # x_out = x_iv[:slice_to_head] - # y_out = y_iv[:slice_to_head] - profiler(f'view range slice {view_range}') vl, vr = view_range zoom_or_append = False - last_vr = self._vr - last_ivr = self._avr + last_vr = self._last_vr + last_ivr = self._last_ivr # incremental in-view data update. if last_vr: @@ -1216,7 +1201,7 @@ class Renderer(msgspec.Struct): # underneath, so we need to redraw. # or left_change and right_change and last_vr == view_range - # not (left_change and right_change) and ivr + # not (left_change and right_change) and ivr # ( # or abs(x_iv[ivr] - livr) > 1 ): @@ -1232,40 +1217,28 @@ class Renderer(msgspec.Struct): should_redraw = True # print("REDRAWING BRUH") - self._vr = view_range + self._last_vr = view_range if len(x_out): - self._avr = x_out[0], x_out[slice_to_head] + self._last_ivr = x_out[0], x_out[slice_to_head] if prepend_length > 0: should_redraw = True - # # last datums - # x_last = x_out[-1] - # y_last = y_out[-1] - path = self.path fast_path = self.fast_path + # redraw the entire source data if we have either of: + # - no prior path graphic rendered or, + # - we always intend to re-render the data only in view if ( path is None or should_redraw or new_sample_rate or prepend_length > 0 ): - # if should_redraw: - # if path: - # path.clear() - # profiler('cleared paths due to `should_redraw=True`') - - # if fast_path: - # fast_path.clear() - - # profiler('cleared paths due to `should_redraw` set') if new_sample_rate and showing_src_data: - # if self._in_ds: log.info(f'DEDOWN -> {array_key}') - self._in_ds = False elif should_ds and uppx > 1: @@ -1277,8 +1250,6 @@ class Renderer(msgspec.Struct): ) profiler(f'FULL PATH downsample redraw={should_ds}') self._in_ds = True - # else: - # print(f"NOT DOWNSAMPLING {array_key}") path = self.draw_path( x=x_out, @@ -1287,15 +1258,6 @@ class Renderer(msgspec.Struct): path=path, redraw=True, ) - # path = pg.functions.arrayToQPath( - # x_out, - # y_out, - # connect='all', - # finiteCheck=False, - # path=path, - # ) - if graphics: - graphics.prepareGeometryChange() profiler( 'generated fresh path. ' @@ -1350,15 +1312,8 @@ class Renderer(msgspec.Struct): connect=connect, # path=fast_path, ) - - # append_path = pg.functions.arrayToQPath( - # connect='all', - # finiteCheck=False, - # path=fast_path, - # ) profiler('generated append qpath') - # if graphics.use_fpath: if use_fpath: print("USING FPATH") # an attempt at trying to make append-updates faster.. @@ -1381,51 +1336,6 @@ class Renderer(msgspec.Struct): profiler(f'connected history path w size: {size}') path.connectPath(append_path) - # if use_vr: - # array = in_view - # # get latest data from flow shm - # self.last_read = ( - # xfirst, xlast, array, ivl, ivr, in_view - # ) = new_read - - # if ( - # self.path is None - # or use_vr - # ): - # redraw the entire source data if we have either of: - # - no prior path graphic rendered or, - # - we always intend to re-render the data only in view - - # data transform: convert source data to a format - # expected to be incrementally updates and later rendered - # to a more graphics native format. - # if self.data_t: - # array = self.data_t(array) - - # maybe allocate shm for data transform output - # if self.data_t_shm is None: - # fshm = self.flow.shm - - # shm, opened = maybe_open_shm_array( - # f'{self.flow.name}_data_t', - # # TODO: create entry for each time frame - # dtype=array.dtype, - # readonly=False, - # ) - # assert opened - # shm.push(array) - # self.data_t_shm = shm - - # elif self.path: - # print(f'inremental update not supported yet {self.flow.name}') - # TODO: do incremental update - # prepend, append, last = self.diff(self.flow.read()) - - # do path generation for each segment - # and then push into graphics object. - - # call path render func on history - # self.path = self.draw_path(hist) self.path = path self.fast_path = fast_path From fa30df36ba840805ed11a7ee1573a4d1cfa3d449 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 19 May 2022 10:35:22 -0400 Subject: [PATCH 072/113] Simplify default xy formatter --- piker/ui/_flows.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index d7c5c2e6..282231e9 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -925,15 +925,7 @@ def by_index_and_key( np.ndarray, np.ndarray, ]: - # full input data - x = array['index'] - y = array[array_key] - - return tuple({ - 'x': x, - 'y': y, - 'connect': 'all', - }.values()) + return array['index'], array[array_key], 'all' class Renderer(msgspec.Struct): @@ -941,7 +933,12 @@ class Renderer(msgspec.Struct): flow: Flow # last array view read last_read: Optional[tuple] = None - format_xy: Callable[np.ndarray, tuple[np.ndarray]] = by_index_and_key + + # default just returns index, and named array from data + format_xy: Callable[ + [np.ndarray, str], + tuple[np.ndarray] + ] = by_index_and_key # output graphics rendering, the main object # processed in ``QGraphicsObject.paint()`` From 432d4545c22da356f8e31ee6fe89c61422f012ab Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 20 May 2022 16:52:44 -0400 Subject: [PATCH 073/113] Fix last values, must be pulled from source data in step mode --- piker/ui/_flows.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 282231e9..c0da2739 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -809,6 +809,8 @@ class Flow(msgspec.Struct): # , frozen=True): # full input data x = array['index'] y = array[array_key] + x_last = x[-1] + y_last = y[-1] # inview data x_iv = in_view['index'] @@ -825,7 +827,7 @@ class Flow(msgspec.Struct): # , frozen=True): ) profiler('generated step mode data') - ( + out = ( x, y, x_iv, @@ -842,9 +844,9 @@ class Flow(msgspec.Struct): # , frozen=True): self._iflat_last, profiler, ) + input_data = out[:-1] - x_last = x[-1] - y_last = y[-1] + w = 0.5 graphics._last_line = QLineF( x_last - 0.5, 0, x_last + 0.5, 0, @@ -856,12 +858,6 @@ class Flow(msgspec.Struct): # , frozen=True): should_redraw = bool(append_diff) draw_last = False - input_data = ( - x, - y, - x_iv, - y_iv, - ) # compute the length diffs between the first/last index entry in # the input data and the last indexes we have on record from the From f5de361f497190d0cdf553ab370b694ed50201f6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 21 May 2022 11:44:20 -0400 Subject: [PATCH 074/113] Import directly from `tractor.trionics` --- piker/data/feed.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/data/feed.py b/piker/data/feed.py index d5e5d3b3..561d063b 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -40,6 +40,7 @@ from trio.abc import ReceiveChannel from trio_typing import TaskStatus import trimeter import tractor +from tractor.trionics import maybe_open_context from pydantic import BaseModel import pendulum import numpy as np From c256d3bdc0ea6eab37da525957d71482d563f0e9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 21 May 2022 11:45:16 -0400 Subject: [PATCH 075/113] Type annot name in put to log routine --- piker/log.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/piker/log.py b/piker/log.py index 7c8bb798..804e09dc 100644 --- a/piker/log.py +++ b/piker/log.py @@ -25,10 +25,13 @@ from pygments import highlight, lexers, formatters # Makes it so we only see the full module name when using ``__name__`` # without the extra "piker." prefix. -_proj_name = 'piker' +_proj_name: str = 'piker' -def get_logger(name: str = None) -> logging.Logger: +def get_logger( + name: str = None, + +) -> logging.Logger: '''Return the package log or a sub-log for `name` if provided. ''' return tractor.log.get_logger(name=name, _root_name=_proj_name) From b985b48eb379e85bea0a03c1a52afc8419eaa9ad Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 21 May 2022 11:45:44 -0400 Subject: [PATCH 076/113] Add `._last_bar_lines` guard to `.paint()` --- piker/ui/_ohlc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index fb57d6ff..f10a4998 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -274,8 +274,9 @@ class BarItems(pg.GraphicsObject): # lead to any perf gains other then when zoomed in to less bars # in view. p.setPen(self.last_bar_pen) - p.drawLines(*tuple(filter(bool, self._last_bar_lines))) - profiler('draw last bar') + if self._last_bar_lines: + p.drawLines(*tuple(filter(bool, self._last_bar_lines))) + profiler('draw last bar') p.setPen(self.bars_pen) p.drawPath(self.path) From 5d91516b41f4ec34bc8d4dfbdfebf2d8045a4bf8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 21 May 2022 11:46:56 -0400 Subject: [PATCH 077/113] Drop step mode "last datum" graphics creation from `.draw_last()` We're doing this in `Flow.update_graphics()` atm and probably are going to in general want custom graphics objects for all the diff curve / path types. The new flows work seems to fix the bounding rect width calcs to not require the ad-hoc extra `+ 1` in the step mode case; before it was always a bit hacky anyway. This also tries to add a more correct bounding rect adjustment for the `._last_line` segment. --- piker/ui/_curve.py | 46 ++++++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index fa073d37..4cffcd25 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -121,6 +121,7 @@ class FastAppendCurve(pg.GraphicsObject): self._last_line: Optional[QLineF] = None self._last_step_rect: Optional[QRectF] = None + self._last_w: float = 1 # flat-top style histogram-like discrete curve self._step_mode: bool = step_mode @@ -183,29 +184,11 @@ class FastAppendCurve(pg.GraphicsObject): # draw the "current" step graphic segment so it lines up with # the "middle" of the current (OHLC) sample. - if self._step_mode: - self._last_line = QLineF( - x_last - 0.5, 0, - x_last + 0.5, 0, - # x_last, 0, - # x_last, 0, - ) - self._last_step_rect = QRectF( - x_last - 0.5, 0, - x_last + 0.5, y_last - # x_last, 0, - # x_last, y_last - ) - # print( - # f"path br: {self.path.boundingRect()}", - # f"fast path br: {self.fast_path.boundingRect()}", - # f"last rect br: {self._last_step_rect}", - # ) - else: - self._last_line = QLineF( - x[-2], y[-2], - x_last, y_last - ) + self._last_line = QLineF( + x[-2], y[-2], + x_last, y_last + ) + # self._last_w = x_last - x[-2] # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. @@ -290,13 +273,20 @@ class FastAppendCurve(pg.GraphicsObject): # # hb_size.height() + 1 # ) - # if self._last_step_rect: # br = self._last_step_rect.bottomRight() - # else: - # hb_size += QSizeF(1, 1) - w = hb_size.width() + 1 - h = hb_size.height() + 1 + w = hb_size.width() + h = hb_size.height() + + if not self._last_step_rect: + # only on a plane line do we include + # and extra index step's worth of width + # since in the step case the end of the curve + # actually terminates earlier so we don't need + # this for the last step. + w += self._last_w + ll = self._last_line + h += ll.y2() - ll.y1() # br = QPointF( # self._vr[-1], From eca2401ab52ff77caf5367c04154622315a608d3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 24 May 2022 08:55:45 -0400 Subject: [PATCH 078/113] Lul, well that heigh did not work.. --- piker/ui/_curve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 4cffcd25..abe48929 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -286,7 +286,7 @@ class FastAppendCurve(pg.GraphicsObject): # this for the last step. w += self._last_w ll = self._last_line - h += ll.y2() - ll.y1() + h += 1 #ll.y2() - ll.y1() # br = QPointF( # self._vr[-1], From bbe1ff19ef1e94363447fb83cf95d539b7f06011 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 24 May 2022 10:35:42 -0400 Subject: [PATCH 079/113] Don't kill all containers on teardown XD --- piker/data/_ahab.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/piker/data/_ahab.py b/piker/data/_ahab.py index 0f96ecaa..fea19a4d 100644 --- a/piker/data/_ahab.py +++ b/piker/data/_ahab.py @@ -98,8 +98,6 @@ async def open_docker( finally: if client: client.close() - for c in client.containers.list(): - c.kill() class Container: From 1b38628b09c853ca906cb541f48dab5f04d4f1c6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 24 May 2022 10:36:17 -0400 Subject: [PATCH 080/113] Handle teardown race, add comment about shm subdirs --- piker/data/_sharedmem.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index 8848ec1c..47d58d3e 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -98,7 +98,12 @@ class SharedInt: if _USE_POSIX: # We manually unlink to bypass all the "resource tracker" # nonsense meant for non-SC systems. - shm_unlink(self._shm.name) + name = self._shm.name + try: + shm_unlink(name) + except FileNotFoundError: + # might be a teardown race here? + log.warning(f'Shm for {name} already unlinked?') class _Token(BaseModel): @@ -536,6 +541,10 @@ def attach_shm_array( if key in _known_tokens: assert _Token.from_msg(_known_tokens[key]) == token, "WTF" + # XXX: ugh, looks like due to the ``shm_open()`` C api we can't + # actually place files in a subdir, see discussion here: + # https://stackoverflow.com/a/11103289 + # attach to array buffer and view as per dtype shm = SharedMemory(name=key) shmarr = np.ndarray( From 8ce7e99210db10c5bb41797183d5571ff0f37eed Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 24 May 2022 11:41:08 -0400 Subject: [PATCH 081/113] Drop prints --- piker/ui/_flows.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index c0da2739..1219627a 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -1281,8 +1281,7 @@ class Renderer(msgspec.Struct): and do_append and not should_redraw ): - # print(f'{self.name} append len: {append_length}') - print(f'{array_key} append len: {append_length}') + # print(f'{array_key} append len: {append_length}') new_x = x_out[-append_length - 2:] # slice_to_head] new_y = y_out[-append_length - 2:] # slice_to_head] profiler('sliced append path') @@ -1308,7 +1307,6 @@ class Renderer(msgspec.Struct): profiler('generated append qpath') if use_fpath: - print("USING FPATH") # an attempt at trying to make append-updates faster.. if fast_path is None: fast_path = append_path From 42572d38087919daa079e3517a4cc74596c1b426 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 24 May 2022 14:22:30 -0400 Subject: [PATCH 082/113] Add back linked plots/views y-range autoscaling --- piker/ui/_interaction.py | 70 +++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 90242c99..8e95855e 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -570,6 +570,13 @@ class ChartView(ViewBox): self._resetTarget() self.scaleBy(s, focal) + # XXX: the order of the next 2 lines i'm pretty sure + # matters, we want the resize to trigger before the graphics + # update, but i gotta feelin that because this one is signal + # based (and thus not necessarily sync invoked right away) + # that calling the resize method manually might work better. + self.sigRangeChangedManually.emit(mask) + # XXX: without this is seems as though sometimes # when zooming in from far out (and maybe vice versa?) # the signal isn't being fired enough since if you pan @@ -580,12 +587,6 @@ class ChartView(ViewBox): # fires don't happen? self.maybe_downsample_graphics() - self.sigRangeChangedManually.emit(mask) - - # self._ic.set() - # self._ic = None - # self.chart.resume_all_feeds() - ev.accept() def mouseDragEvent( @@ -746,7 +747,7 @@ class ChartView(ViewBox): # flag to prevent triggering sibling charts from the same linked # set from recursion errors. - autoscale_linked_plots: bool = False, + autoscale_linked_plots: bool = True, name: Optional[str] = None, # autoscale_overlays: bool = False, @@ -759,6 +760,7 @@ class ChartView(ViewBox): data set. ''' + # print(f'YRANGE ON {self.name}') profiler = pg.debug.Profiler( msg=f'`ChartView._set_yrange()`: `{self.name}`', disabled=not pg_profile_enabled(), @@ -788,42 +790,11 @@ class ChartView(ViewBox): elif yrange is not None: ylow, yhigh = yrange - # calculate max, min y values in viewable x-range from data. - # Make sure min bars/datums on screen is adhered. - # else: - # TODO: eventually we should point to the - # ``FlowsTable`` (or wtv) which should perform - # the group operations? - - # flow = chart._flows[name or chart.name] - # br = bars_range or chart.bars_range() - # br = bars_range or chart.bars_range() - # profiler(f'got bars range: {br}') - - # TODO: maybe should be a method on the - # chart widget/item? - # if False: - # if autoscale_linked_plots: - # # avoid recursion by sibling plots - # linked = self.linkedsplits - # plots = list(linked.subplots.copy().values()) - # main = linked.chart - # if main: - # plots.append(main) - - # for chart in plots: - # if chart and not chart._static_yrange: - # chart.cv._set_yrange( - # # bars_range=br, - # autoscale_linked_plots=False, - # ) - # profiler('autoscaled linked plots') - if set_range: + # XXX: only compute the mxmn range + # if none is provided as input! if not yrange: - # XXX: only compute the mxmn range - # if none is provided as input! yrange = self._maxmin() if yrange is None: @@ -850,6 +821,25 @@ class ChartView(ViewBox): self.setYRange(ylow, yhigh) profiler(f'set limits: {(ylow, yhigh)}') + # TODO: maybe should be a method on the + # chart widget/item? + if autoscale_linked_plots: + # avoid recursion by sibling plots + linked = self.linkedsplits + plots = list(linked.subplots.copy().values()) + main = linked.chart + if main: + plots.append(main) + + # print(f'autoscaling linked: {plots}') + for chart in plots: + if chart and not chart._static_yrange: + chart.cv._set_yrange( + # bars_range=br, + autoscale_linked_plots=False, + ) + profiler('autoscaled linked plots') + profiler.finish() def enable_auto_yrange( From 04897fd402024cdbd63ca594e9009bc5f0831a46 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 25 May 2022 11:15:46 -0400 Subject: [PATCH 083/113] Implement pre-graphics format incremental update Adds a new pre-graphics data-format callback incremental update api to our `Renderer`. `Renderer` instance can now overload these custom routines: - `.update_xy()` a routine which accepts the latest [pre/a]pended data sliced out from shm and returns it in a format suitable to store in the optional `.[x/y]_data` arrays. - `.allocate_xy()` which initially does the work of pre-allocating the `.[x/y]_data` arrays based on the source shm sizing such that new data can be filled in (to memory). - `._xy_[first/last]: int` attrs to track index diffs between src shm and the xy format data updates. Implement the step curve data format with 3 super simple routines: - `.allocate_xy()` -> `._pathops.to_step_format()` - `.update_xy()` -> `._flows.update_step_xy()` - `.format_xy()` -> `._flows.step_to_xy()` Further, adjust `._pathops.gen_ohlc_qpath()` to adhere to the new call signature. --- piker/ui/_flows.py | 408 +++++++++++++++++++++---------------------- piker/ui/_pathops.py | 16 +- 2 files changed, 215 insertions(+), 209 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 1219627a..043f9243 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -144,8 +144,9 @@ def render_baritems( # if no source data renderer exists create one. self = flow - r = self._src_r show_bars: bool = False + + r = self._src_r if not r: show_bars = True # OHLC bars path renderer @@ -188,7 +189,7 @@ def render_baritems( # do checks for whether or not we require downsampling: # - if we're **not** downsampling then we simply want to # render the bars graphics curve and update.. - # - if insteam we are in a downsamplig state then we to + # - if instead we are in a downsamplig state then we to x_gt = 6 uppx = curve.x_uppx() in_line = should_line = curve.isVisible() @@ -212,6 +213,7 @@ def render_baritems( if should_line: fields = ['open', 'high', 'low', 'close'] + if self.gy is None: # create a flattened view onto the OHLC array # which can be read as a line-style format @@ -373,119 +375,75 @@ def render_baritems( ) -def update_step_data( - flow: Flow, - shm: ShmArray, - ivl: int, - ivr: int, +def update_step_xy( + src_shm: ShmArray, array_key: str, - iflat_first: int, - iflat: int, - profiler: pg.debug.Profiler, + y_update: np.ndarray, + slc: slice, + ln: int, + first: int, + last: int, + is_append: bool, -) -> tuple: +) -> np.ndarray: - self = flow - ( - # iflat_first, - # iflat, - ishm_last, - ishm_first, - ) = ( - # self._iflat_first, - # self._iflat_last, - shm._last.value, - shm._first.value - ) - il = max(iflat - 1, 0) - profiler('read step mode incr update indices') + # for a step curve we slice from one datum prior + # to the current "update slice" to get the previous + # "level". + if is_append: + start = max(last - 1, 0) + end = src_shm._last.value + new_y = src_shm._array[start:end][array_key] + slc = slice(start, end) - # check for shm prepend updates since last read. - if iflat_first != ishm_first: + else: + new_y = y_update - print(f'prepend {array_key}') - - # i_prepend = self.shm._array['index'][ - # ishm_first:iflat_first] - y_prepend = self.shm._array[array_key][ - ishm_first:iflat_first - ] - - y2_prepend = np.broadcast_to( - y_prepend[:, None], (y_prepend.size, 2), - ) - - # write newly prepended data to flattened copy - self.gy[ishm_first:iflat_first] = y2_prepend - self._iflat_first = ishm_first - profiler('prepended step mode history') - - append_diff = ishm_last - iflat - if append_diff: - - # slice up to the last datum since last index/append update - # new_x = self.shm._array[il:ishm_last]['index'] - new_y = self.shm._array[il:ishm_last][array_key] - - new_y2 = np.broadcast_to( + return ( + np.broadcast_to( new_y[:, None], (new_y.size, 2), - ) - self.gy[il:ishm_last] = new_y2 - profiler('updated step curve data') + ), + slc, + ) - # print( - # f'append size: {append_diff}\n' - # f'new_x: {new_x}\n' - # f'new_y: {new_y}\n' - # f'new_y2: {new_y2}\n' - # f'new gy: {gy}\n' - # ) - # update local last-index tracking - self._iflat_last = ishm_last +def step_to_xy( + r: Renderer, + array: np.ndarray, + array_key: str, + vr: tuple[int, int], - # slice out up-to-last step contents - x_step = self.gx[ishm_first:ishm_last+2] - # shape to 1d - x = x_step.reshape(-1) - profiler('sliced step x') +) -> tuple[ + np.ndarray, + np.nd.array, + str, +]: - y_step = self.gy[ishm_first:ishm_last+2] - lasts = self.shm.array[['index', array_key]] + # 2 more datum-indexes to capture zero at end + x_step = r.x_data[r._xy_first:r._xy_last+2] + y_step = r.y_data[r._xy_first:r._xy_last+2] + + lasts = array[['index', array_key]] last = lasts[array_key][-1] y_step[-1] = last - # shape to 1d - y = y_step.reshape(-1) - # s = 6 - # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') - - profiler('sliced step y') - - # do all the same for only in-view data + # slice out in-view data + ivl, ivr = vr ys_iv = y_step[ivl:ivr+1] xs_iv = x_step[ivl:ivr+1] + + # flatten to 1d y_iv = ys_iv.reshape(ys_iv.size) x_iv = xs_iv.reshape(xs_iv.size) + # print( # f'ys_iv : {ys_iv[-s:]}\n' # f'y_iv: {y_iv[-s:]}\n' # f'xs_iv: {xs_iv[-s:]}\n' # f'x_iv: {x_iv[-s:]}\n' # ) - profiler('sliced in view step data') - # legacy full-recompute-everytime method - # x, y = ohlc_flatten(array) - # x_iv, y_iv = ohlc_flatten(in_view) - # profiler('flattened OHLC data') - return ( - x, - y, - x_iv, - y_iv, - append_diff, - ) + return x_iv, y_iv, 'all' class Flow(msgspec.Struct): # , frozen=True): @@ -508,7 +466,7 @@ class Flow(msgspec.Struct): # , frozen=True): render: bool = True # toggle for display loop # pre-graphics formatted data - gy: Optional[ShmArray] = None + gy: Optional[np.ndarray] = None gx: Optional[np.ndarray] = None # pre-graphics update indices @@ -723,9 +681,9 @@ class Flow(msgspec.Struct): # , frozen=True): out: Optional[tuple] = None if isinstance(graphics, BarItems): draw_last = False + # XXX: special case where we change out graphics # to a line after a certain uppx threshold. - # render_baritems( out = render_baritems( self, graphics, @@ -739,19 +697,8 @@ class Flow(msgspec.Struct): # , frozen=True): # return graphics - r = self._src_r - if not r: - # just using for ``.diff()`` atm.. - r = self._src_r = Renderer( - flow=self, - # TODO: rename this to something with ohlc - # draw_path=gen_ohlc_qpath, - last_read=read, - ) - # ``FastAppendCurve`` case: array_key = array_key or self.name - shm = self.shm if out is not None: # hack to handle ds curve from bars above @@ -763,7 +710,49 @@ class Flow(msgspec.Struct): # , frozen=True): y_iv, ) = out input_data = out[1:] - # breakpoint() + + r = self._src_r + if not r: + # just using for ``.diff()`` atm.. + r = self._src_r = Renderer( + flow=self, + # TODO: rename this to something with ohlc + # draw_path=gen_ohlc_qpath, + last_read=read, + ) + + if graphics._step_mode: + + r.allocate_xy = to_step_format + r.update_xy = update_step_xy + r.format_xy = step_to_xy + + slice_to_head = -2 + + # TODO: append logic inside ``.render()`` isn't + # corrent yet for step curves.. remove this to see it. + should_redraw = True + + # TODO: remove this and instead place all step curve + # updating into pre-path data render callbacks. + # full input data + x = array['index'] + y = array[array_key] + x_last = x[-1] + y_last = y[-1] + + w = 0.5 + graphics._last_line = QLineF( + x_last - w, 0, + x_last + w, 0, + ) + graphics._last_step_rect = QRectF( + x_last - w, 0, + x_last + w, y_last, + ) + + # should_redraw = bool(append_diff) + draw_last = False # ds update config new_sample_rate: bool = False @@ -780,7 +769,7 @@ class Flow(msgspec.Struct): # , frozen=True): uppx > 1 and abs(uppx_diff) >= 1 ): - log.info( + log.debug( f'{array_key} sampler change: {self._last_uppx} -> {uppx}' ) self._last_uppx = uppx @@ -801,69 +790,6 @@ class Flow(msgspec.Struct): # , frozen=True): should_ds = False showing_src_data = True - if graphics._step_mode: - slice_to_head = -2 - - # TODO: remove this and instead place all step curve - # updating into pre-path data render callbacks. - # full input data - x = array['index'] - y = array[array_key] - x_last = x[-1] - y_last = y[-1] - - # inview data - x_iv = in_view['index'] - y_iv = in_view[array_key] - - if self.gy is None: - ( - self._iflat_first, - self.gx, - self.gy, - ) = to_step_format( - shm, - array_key, - ) - profiler('generated step mode data') - - out = ( - x, - y, - x_iv, - y_iv, - append_diff, - - ) = update_step_data( - self, - shm, - ivl, - ivr, - array_key, - self._iflat_first, - self._iflat_last, - profiler, - ) - input_data = out[:-1] - - w = 0.5 - graphics._last_line = QLineF( - x_last - 0.5, 0, - x_last + 0.5, 0, - ) - graphics._last_step_rect = QRectF( - x_last - 0.5, 0, - x_last + 0.5, y_last, - ) - - should_redraw = bool(append_diff) - draw_last = False - - # compute the length diffs between the first/last index entry in - # the input data and the last indexes we have on record from the - # last time we updated the curve index. - # prepend_length, append_length = r.diff(read) - # MAIN RENDER LOGIC: # - determine in view data and redraw on range change # - determine downsampling ops if needed @@ -913,8 +839,10 @@ class Flow(msgspec.Struct): # , frozen=True): def by_index_and_key( + renderer: Renderer, array: np.ndarray, array_key: str, + vr: tuple[int, int], ) -> tuple[ np.ndarray, @@ -936,15 +864,31 @@ class Renderer(msgspec.Struct): tuple[np.ndarray] ] = by_index_and_key + # optional pre-graphics xy formatted data which + # is incrementally updated in sync with the source data. + allocate_xy: Optional[Callable[ + [int, slice], + tuple[np.ndarray, np.nd.array] + ]] = None + + update_xy: Optional[Callable[ + [int, slice], None] + ] = None + + x_data: Optional[np.ndarray] = None + y_data: Optional[np.ndarray] = None + + # indexes which slice into the above arrays (which are allocated + # based on source data shm input size) and allow retrieving + # incrementally updated data. + _xy_first: int = 0 + _xy_last: int = 0 + # output graphics rendering, the main object # processed in ``QGraphicsObject.paint()`` path: Optional[QPainterPath] = None fast_path: Optional[QPainterPath] = None - # called on input data but before any graphics format - # conversions or processing. - format_data: Optional[Callable[ShmArray, np.ndarray]] = None - # XXX: just ideas.. # called on the final data (transform) output to convert # to "graphical data form" a format that can be passed to @@ -998,17 +942,13 @@ class Renderer(msgspec.Struct): prepend_length = int(last_xfirst - xfirst) append_length = int(xlast - last_xlast) - # TODO: eventually maybe we can implement some kind of - # transform on the ``QPainterPath`` that will more or less - # detect the diff in "elements" terms? - # update state - self.last_read = new_read - # blah blah blah # do diffing for prepend, append and last entry return ( + slice(xfirst, last_xfirst), prepend_length, append_length, + slice(last_xlast, xlast), ) def draw_path( @@ -1103,6 +1043,75 @@ class Renderer(msgspec.Struct): in_view, ) = new_read + ( + pre_slice, + prepend_length, + append_length, + post_slice, + ) = self.diff(new_read) + + if self.update_xy: + + shm = self.flow.shm + + if self.y_data is None: + # we first need to allocate xy data arrays + # from the source data. + assert self.allocate_xy + self.x_data, self.y_data = self.allocate_xy( + shm, + array_key, + ) + self._xy_first = shm._first.value + self._xy_last = shm._last.value + profiler('allocated xy history') + + if prepend_length: + y_prepend = shm._array[array_key][pre_slice] + + xy_data, xy_slice = self.update_xy( + shm, + array_key, + + # this is the pre-sliced, "normally expected" + # new data that an updater would normally be + # expected to process, however in some cases (like + # step curves) the updater routine may want to do + # the source history-data reading itself, so we pass + # both here. + y_prepend, + + pre_slice, + prepend_length, + self._xy_first, + self._xy_last, + is_append=False, + ) + self.y_data[xy_slice] = xy_data + self._xy_first = shm._first.value + profiler('prepended xy history: {prepend_length}') + + if append_length: + y_append = shm._array[array_key][post_slice] + + xy_data, xy_slice = self.update_xy( + shm, + array_key, + + y_append, + post_slice, + append_length, + + self._xy_first, + self._xy_last, + is_append=True, + ) + # self.y_data[post_slice] = xy_data + # self.y_data[xy_slice or post_slice] = xy_data + self.y_data[xy_slice] = xy_data + self._xy_last = shm._last.value + profiler('appened xy history: {append_length}') + if use_vr: array = in_view @@ -1120,45 +1129,31 @@ class Renderer(msgspec.Struct): x_out = x_iv y_out = y_iv - # last = y_out[slice_to_head] - else: - hist = array[:slice_to_head] - # last = array[slice_to_head] - - # maybe allocate shm for data transform output - # if self.format_data is None: - # fshm = self.flow.shm - - # shm, opened = maybe_open_shm_array( - # f'{self.flow.name}_data_t', - # # TODO: create entry for each time frame - # dtype=array.dtype, - # readonly=False, - # ) - # assert opened - # shm.push(array) - # self.data_t_shm = shm - # xy-path data transform: convert source data to a format # able to be passed to a `QPainterPath` rendering routine. # expected to be incrementally updates and later rendered to # a more graphics native format. # if self.data_t: # array = self.data_t(array) + + hist = array[:slice_to_head] ( x_out, y_out, connect, - ) = self.format_xy(hist, array_key) + ) = self.format_xy( + self, + # TODO: hist here should be the pre-sliced + # x/y_data in the case where allocate_xy is + # defined? + hist, + array_key, + (ivl, ivr), + ) profiler('sliced input arrays') - ( - prepend_length, - append_length, - ) = self.diff(new_read) - if ( use_vr ): @@ -1330,5 +1325,10 @@ class Renderer(msgspec.Struct): self.path = path self.fast_path = fast_path + # TODO: eventually maybe we can implement some kind of + # transform on the ``QPainterPath`` that will more or less + # detect the diff in "elements" terms? + # update diff state since we've now rendered paths. self.last_read = new_read + return self.path, array diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index 4cb5b86e..89f7c5dc 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -17,25 +17,30 @@ Super fast ``QPainterPath`` generation related operator routines. """ +from __future__ import annotations from typing import ( - Optional, + # Optional, + TYPE_CHECKING, ) import numpy as np from numpy.lib import recfunctions as rfn from numba import njit, float64, int64 # , optional -import pyqtgraph as pg +# import pyqtgraph as pg from PyQt5 import QtGui # from PyQt5.QtCore import QLineF, QPointF from ..data._sharedmem import ( ShmArray, ) -from .._profile import pg_profile_enabled, ms_slower_then +# from .._profile import pg_profile_enabled, ms_slower_then from ._compression import ( ds_m4, ) +if TYPE_CHECKING: + from ._flows import Renderer + def xy_downsample( x, @@ -138,8 +143,10 @@ def path_arrays_from_ohlc( def gen_ohlc_qpath( + r: Renderer, data: np.ndarray, array_key: str, # we ignore this + vr: tuple[int, int], start: int = 0, # XXX: do we need this? # 0.5 is no overlap between arms, 1.0 is full overlap @@ -216,7 +223,6 @@ def to_step_format( for use by path graphics generation. ''' - first = shm._first.value i = shm._array['index'].copy() out = shm._array[data_field].copy() @@ -230,4 +236,4 @@ def to_step_format( # start y at origin level y_out[0, 0] = 0 - return first, x_out, y_out + return x_out, y_out From d4f31f2b3c569337c67d6cc72f683bfd3794dedc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 25 May 2022 11:41:52 -0400 Subject: [PATCH 084/113] Move update-state-vars defaults above step mode block --- piker/ui/_flows.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 043f9243..7714dd67 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -721,6 +721,12 @@ class Flow(msgspec.Struct): # , frozen=True): last_read=read, ) + # ds update config + new_sample_rate: bool = False + should_redraw: bool = False + should_ds: bool = r._in_ds + showing_src_data: bool = not r._in_ds + if graphics._step_mode: r.allocate_xy = to_step_format @@ -754,12 +760,6 @@ class Flow(msgspec.Struct): # , frozen=True): # should_redraw = bool(append_diff) draw_last = False - # ds update config - new_sample_rate: bool = False - should_redraw: bool = False - should_ds: bool = r._in_ds - showing_src_data: bool = not r._in_ds - # downsampling incremental state checking # check for and set std m4 downsample conditions uppx = graphics.x_uppx() From 066b8df619dc820ab8079e135bf429246abc2a5f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 25 May 2022 19:45:09 -0400 Subject: [PATCH 085/113] Implement OHLC downsampled curve via renderer, drop old bypass code --- piker/ui/_flows.py | 440 +++++++++++++++---------------------------- piker/ui/_pathops.py | 5 +- 2 files changed, 157 insertions(+), 288 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 7714dd67..e74ee123 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -23,7 +23,7 @@ incremental update. ''' from __future__ import annotations -# from functools import partial +from functools import partial from typing import ( Optional, Callable, @@ -45,7 +45,6 @@ from PyQt5.QtCore import ( from ..data._sharedmem import ( ShmArray, - # open_shm_array, ) from .._profile import ( pg_profile_enabled, @@ -68,6 +67,7 @@ from ..log import get_logger log = get_logger(__name__) + # class FlowsTable(msgspec.Struct): # ''' # Data-AGGRegate: high level API onto multiple (categorized) @@ -77,44 +77,56 @@ log = get_logger(__name__) # ''' # flows: dict[str, np.ndarray] = {} -# @classmethod -# def from_token( -# cls, -# shm_token: tuple[ -# str, -# str, -# tuple[str, str], -# ], -# ) -> Renderer: +def update_ohlc_to_line( + src_shm: ShmArray, + array_key: str, + src_update: np.ndarray, + slc: slice, + ln: int, + first: int, + last: int, + is_append: bool, -# shm = attach_shm_array(token) -# return cls(shm) +) -> np.ndarray: - -def rowarr_to_path( - rows_array: np.ndarray, - x_basis: np.ndarray, - flow: Flow, - -) -> QPainterPath: - - # TODO: we could in theory use ``numba`` to flatten - # if needed? - - # to 1d - y = rows_array.flatten() - - return pg.functions.arrayToQPath( - # these get passed at render call time - x=x_basis[:y.size], - y=y, - connect='all', - finiteCheck=False, - path=flow.path, + fields = ['open', 'high', 'low', 'close'] + return ( + rfn.structured_to_unstructured(src_update[fields]), + slc, ) +def ohlc_flat_to_xy( + r: Renderer, + array: np.ndarray, + array_key: str, + vr: tuple[int, int], + +) -> tuple[ + np.ndarray, + np.nd.array, + str, +]: + # TODO: in the case of an existing ``.update_xy()`` + # should we be passing in array as an xy arrays tuple? + + # 2 more datum-indexes to capture zero at end + x_flat = r.x_data[r._xy_first:r._xy_last] + y_flat = r.y_data[r._xy_first:r._xy_last] + + # slice to view + ivl, ivr = vr + x_iv_flat = x_flat[ivl:ivr] + y_iv_flat = y_flat[ivl:ivr] + + # reshape to 1d for graphics rendering + y_iv = y_iv_flat.reshape(-1) + x_iv = x_iv_flat.reshape(-1) + + return x_iv, y_iv, 'all' + + def render_baritems( flow: Flow, graphics: BarItems, @@ -137,10 +149,7 @@ def render_baritems( layer, if not a `Renderer` then something just above it? ''' - ( - xfirst, xlast, array, - ivl, ivr, in_view, - ) = read + bars = graphics # if no source data renderer exists create one. self = flow @@ -156,35 +165,28 @@ def render_baritems( last_read=read, ) - # ds_curve_r = Renderer( - # flow=self, + ds_curve_r = Renderer( + flow=self, + last_read=read, - # # just swap in the flat view - # # data_t=lambda array: self.gy.array, - # last_read=read, - # draw_path=partial( - # rowarr_to_path, - # x_basis=None, - # ), + # incr update routines + allocate_xy=ohlc_to_line, + update_xy=update_ohlc_to_line, + format_xy=ohlc_flat_to_xy, + ) - # ) curve = FastAppendCurve( - name='OHLC', - color=graphics._color, + name=f'{flow.name}_ds_ohlc', + color=bars._color, ) curve.hide() self.plot.addItem(curve) # baseline "line" downsampled OHLC curve that should # kick on only when we reach a certain uppx threshold. - self._render_table[0] = curve - # ( - # # ds_curve_r, - # curve, - # ) + self._render_table = (ds_curve_r, curve) - curve = self._render_table[0] - # dsc_r, curve = self._render_table[0] + ds_r, curve = self._render_table # do checks for whether or not we require downsampling: # - if we're **not** downsampling then we simply want to @@ -197,183 +199,74 @@ def render_baritems( should_line and uppx < x_gt ): - print('FLIPPING TO BARS') + # print('FLIPPING TO BARS') should_line = False elif ( not should_line and uppx >= x_gt ): - print('FLIPPING TO LINE') + # print('FLIPPING TO LINE') should_line = True profiler(f'ds logic complete line={should_line}') # do graphics updates if should_line: - - fields = ['open', 'high', 'low', 'close'] - - if self.gy is None: - # create a flattened view onto the OHLC array - # which can be read as a line-style format - shm = self.shm - ( - self._iflat_first, - self._iflat_last, - self.gx, - self.gy, - ) = ohlc_to_line( - shm, - fields=fields, - ) - - # print(f'unstruct diff: {time.time() - start}') - - gy = self.gy - - # update flatted ohlc copy - ( - iflat_first, - iflat, - ishm_last, - ishm_first, - ) = ( - self._iflat_first, - self._iflat_last, - self.shm._last.value, - self.shm._first.value - ) - - # check for shm prepend updates since last read. - if iflat_first != ishm_first: - - # write newly prepended data to flattened copy - gy[ - ishm_first:iflat_first - ] = rfn.structured_to_unstructured( - self.shm._array[fields][ishm_first:iflat_first] - ) - self._iflat_first = ishm_first - - to_update = rfn.structured_to_unstructured( - self.shm._array[iflat:ishm_last][fields] - ) - - gy[iflat:ishm_last][:] = to_update - profiler('updated ustruct OHLC data') - - # slice out up-to-last step contents - y_flat = gy[ishm_first:ishm_last] - x_flat = self.gx[ishm_first:ishm_last] - - # update local last-index tracking - self._iflat_last = ishm_last - - # reshape to 1d for graphics rendering - y = y_flat.reshape(-1) - x = x_flat.reshape(-1) - profiler('flattened ustruct OHLC data') - - # do all the same for only in-view data - y_iv_flat = y_flat[ivl:ivr] - x_iv_flat = x_flat[ivl:ivr] - y_iv = y_iv_flat.reshape(-1) - x_iv = x_iv_flat.reshape(-1) - profiler('flattened ustruct in-view OHLC data') - - # pass into curve graphics processing - # curve.update_from_array( - # x, - # y, - # x_iv=x_iv, - # y_iv=y_iv, - # view_range=(ivl, ivr), # hack - # profiler=profiler, - # # should_redraw=False, - - # # NOTE: already passed through by display loop? - # # do_append=uppx < 16, - # **kwargs, - # ) - curve.draw_last(x, y) - curve.show() + r = ds_r + graphics = curve profiler('updated ds curve') else: - # render incremental or in-view update - # and apply ouput (path) to graphics. - path, data = r.render( - read, - 'ohlc', - profiler=profiler, - # uppx=1, - use_vr=True, - # graphics=graphics, - # should_redraw=True, # always - ) - assert path + graphics = bars - graphics.path = path - graphics.draw_last(data[-1]) - if show_bars: - graphics.show() - - # NOTE: on appends we used to have to flip the coords - # cache thought it doesn't seem to be required any more? - # graphics.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # graphics.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - # graphics.prepareGeometryChange() - graphics.update() + if show_bars: + bars.show() + changed_to_line = False if ( not in_line and should_line ): # change to line graphic - log.info( f'downsampling to line graphic {self.name}' ) - graphics.hide() - # graphics.update() + bars.hide() curve.show() curve.update() + changed_to_line = True elif in_line and not should_line: + # change to bars graphic log.info(f'showing bars graphic {self.name}') curve.hide() - graphics.show() - graphics.update() + bars.show() + bars.update() - # update our pre-downsample-ready data and then pass that - # new data the downsampler algo for incremental update. - - # graphics.update_from_array( - # array, - # in_view, - # view_range=(ivl, ivr) if use_vr else None, - - # **kwargs, - # ) - - # generate and apply path to graphics obj - # graphics.path, last = r.render( - # read, - # only_in_view=True, - # ) - # graphics.draw_last(last) + draw_last = False + lasts = self.shm.array[-2:] + last = lasts[-1] if should_line: - return ( - curve, - x, - y, - x_iv, - y_iv, + def draw_last(): + x, y = lasts['index'], lasts['close'] + curve.draw_last(x, y) + else: + draw_last = partial( + bars.draw_last, + last, ) + return ( + graphics, + r, + {'read_from_key': False}, + draw_last, + should_line, + changed_to_line, + ) + def update_step_xy( src_shm: ShmArray, @@ -484,7 +377,7 @@ class Flow(msgspec.Struct): # , frozen=True): _render_table: dict[ Optional[int], tuple[Renderer, pg.GraphicsItem], - ] = {} + ] = (None, None) # TODO: hackery to be able to set a shm later # but whilst also allowing this type to hashable, @@ -672,62 +565,58 @@ class Flow(msgspec.Struct): # , frozen=True): not in_view.size or not render ): + # print('exiting early') return graphics draw_last: bool = True slice_to_head: int = -1 - input_data = None + # input_data = None + + should_redraw: bool = False + + rkwargs = {} + bars = False - out: Optional[tuple] = None if isinstance(graphics, BarItems): - draw_last = False - # XXX: special case where we change out graphics # to a line after a certain uppx threshold. - out = render_baritems( + ( + graphics, + r, + rkwargs, + draw_last, + should_line, + changed_to_line, + ) = render_baritems( self, graphics, read, profiler, **kwargs, ) + bars = True + should_redraw = changed_to_line or not should_line - if out is None: - return graphics - - # return graphics + else: + r = self._src_r + if not r: + # just using for ``.diff()`` atm.. + r = self._src_r = Renderer( + flow=self, + # TODO: rename this to something with ohlc + last_read=read, + ) # ``FastAppendCurve`` case: array_key = array_key or self.name - - if out is not None: - # hack to handle ds curve from bars above - ( - graphics, # curve - x, - y, - x_iv, - y_iv, - ) = out - input_data = out[1:] - - r = self._src_r - if not r: - # just using for ``.diff()`` atm.. - r = self._src_r = Renderer( - flow=self, - # TODO: rename this to something with ohlc - # draw_path=gen_ohlc_qpath, - last_read=read, - ) + # print(array_key) # ds update config new_sample_rate: bool = False - should_redraw: bool = False should_ds: bool = r._in_ds showing_src_data: bool = not r._in_ds - if graphics._step_mode: + if getattr(graphics, '_step_mode', False): r.allocate_xy = to_step_format r.update_xy = update_step_xy @@ -756,8 +645,6 @@ class Flow(msgspec.Struct): # , frozen=True): x_last - w, 0, x_last + w, y_last, ) - - # should_redraw = bool(append_diff) draw_last = False # downsampling incremental state checking @@ -795,15 +682,11 @@ class Flow(msgspec.Struct): # , frozen=True): # - determine downsampling ops if needed # - (incrementally) update ``QPainterPath`` - # path = graphics.path - # fast_path = graphics.fast_path - path, data = r.render( read, array_key, profiler, uppx=uppx, - input_data=input_data, # use_vr=True, # TODO: better way to detect and pass this? @@ -816,6 +699,8 @@ class Flow(msgspec.Struct): # , frozen=True): slice_to_head=slice_to_head, do_append=do_append, + + **rkwargs, ) # TODO: does this actuallly help us in any way (prolly should # look at the source / ask ogi). @@ -825,11 +710,16 @@ class Flow(msgspec.Struct): # , frozen=True): graphics.path = r.path graphics.fast_path = r.fast_path - if draw_last: + if draw_last and not bars: + # TODO: how to handle this draw last stuff.. x = data['index'] y = data[array_key] graphics.draw_last(x, y) - profiler('draw last segment') + + elif bars and draw_last: + draw_last() + + profiler('draw last segment') graphics.update() profiler('.update()') @@ -1004,10 +894,8 @@ class Renderer(msgspec.Struct): profiler: pg.debug.Profiler, uppx: float = 1, - input_data: Optional[tuple[np.ndarray]] = None, - # redraw and ds flags - should_redraw: bool = True, + should_redraw: bool = False, new_sample_rate: bool = False, should_ds: bool = False, showing_src_data: bool = True, @@ -1018,6 +906,7 @@ class Renderer(msgspec.Struct): # only render datums "in view" of the ``ChartView`` use_vr: bool = True, + read_from_key: bool = True, ) -> list[QPainterPath]: ''' @@ -1067,7 +956,10 @@ class Renderer(msgspec.Struct): profiler('allocated xy history') if prepend_length: - y_prepend = shm._array[array_key][pre_slice] + y_prepend = shm._array[pre_slice] + + if read_from_key: + y_prepend = y_prepend[array_key] xy_data, xy_slice = self.update_xy( shm, @@ -1092,7 +984,10 @@ class Renderer(msgspec.Struct): profiler('prepended xy history: {prepend_length}') if append_length: - y_append = shm._array[array_key][post_slice] + y_append = shm._array[post_slice] + + if read_from_key: + y_append = y_append[array_key] xy_data, xy_slice = self.update_xy( shm, @@ -1114,43 +1009,22 @@ class Renderer(msgspec.Struct): if use_vr: array = in_view - - if input_data: - # allow input data passing for now from alt curve updaters. - ( - x_out, - y_out, - x_iv, - y_iv, - ) = input_data - connect = 'all' - - if use_vr: - x_out = x_iv - y_out = y_iv - else: - # xy-path data transform: convert source data to a format - # able to be passed to a `QPainterPath` rendering routine. - # expected to be incrementally updates and later rendered to - # a more graphics native format. - # if self.data_t: - # array = self.data_t(array) + ivl, ivr = xfirst, xlast - hist = array[:slice_to_head] - ( - x_out, - y_out, - connect, - ) = self.format_xy( - self, - # TODO: hist here should be the pre-sliced - # x/y_data in the case where allocate_xy is - # defined? - hist, - array_key, - (ivl, ivr), - ) + hist = array[:slice_to_head] + + # xy-path data transform: convert source data to a format + # able to be passed to a `QPainterPath` rendering routine. + x_out, y_out, connect = self.format_xy( + self, + # TODO: hist here should be the pre-sliced + # x/y_data in the case where allocate_xy is + # defined? + hist, + array_key, + (ivl, ivr), + ) profiler('sliced input arrays') @@ -1168,7 +1042,7 @@ class Renderer(msgspec.Struct): zoom_or_append = False last_vr = self._last_vr - last_ivr = self._last_ivr + last_ivr = self._last_ivr or vl, vr # incremental in-view data update. if last_vr: @@ -1203,7 +1077,6 @@ class Renderer(msgspec.Struct): ) ): should_redraw = True - # print("REDRAWING BRUH") self._last_vr = view_range if len(x_out): @@ -1224,7 +1097,7 @@ class Renderer(msgspec.Struct): or new_sample_rate or prepend_length > 0 ): - + # print("REDRAWING BRUH") if new_sample_rate and showing_src_data: log.info(f'DEDOWN -> {array_key}') self._in_ds = False @@ -1252,7 +1125,6 @@ class Renderer(msgspec.Struct): f'(should_redraw: {should_redraw} ' f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' ) - # profiler(f'DRAW PATH IN VIEW -> {self.name}') # TODO: get this piecewise prepend working - right now it's # giving heck on vwap... @@ -1297,7 +1169,7 @@ class Renderer(msgspec.Struct): x=new_x, y=new_y, connect=connect, - # path=fast_path, + path=fast_path, ) profiler('generated append qpath') diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index 89f7c5dc..83b46f43 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -168,11 +168,10 @@ def gen_ohlc_qpath( def ohlc_to_line( ohlc_shm: ShmArray, + data_field: str, fields: list[str] = ['open', 'high', 'low', 'close'] ) -> tuple[ - int, # flattened first index - int, # flattened last index np.ndarray, np.ndarray, ]: @@ -205,8 +204,6 @@ def ohlc_to_line( assert y_out.any() return ( - first, - last, x_out, y_out, ) From 08c83afa9006c4e7516f90f9301b8dd4418c6544 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 26 May 2022 18:32:47 -0400 Subject: [PATCH 086/113] Rejig config helpers for arbitrary named files --- piker/config.py | 82 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 18 deletions(-) diff --git a/piker/config.py b/piker/config.py index cf946405..d1926dec 100644 --- a/piker/config.py +++ b/piker/config.py @@ -1,5 +1,5 @@ # piker: trading gear for hackers -# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) +# Copyright (C) 2018-present Tyler Goodlet (in stewardship for pikers) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,6 +16,7 @@ """ Broker configuration mgmt. + """ import platform import sys @@ -50,7 +51,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): Unix (POSIX): ``~/.foo-bar`` Win XP (roaming): - ``C:\Documents and Settings\\Local Settings\Application Data\Foo Bar`` + ``C:\Documents and Settings\\Local Settings\Application Data\Foo`` Win XP (not roaming): ``C:\Documents and Settings\\Application Data\Foo Bar`` Win 7 (roaming): @@ -81,7 +82,8 @@ def get_app_dir(app_name, roaming=True, force_posix=False): folder = os.path.expanduser("~") return os.path.join(folder, app_name) if force_posix: - return os.path.join(os.path.expanduser("~/.{}".format(_posixify(app_name)))) + return os.path.join( + os.path.expanduser("~/.{}".format(_posixify(app_name)))) if sys.platform == "darwin": return os.path.join( os.path.expanduser("~/Library/Application Support"), app_name @@ -107,7 +109,12 @@ if _parent_user: ] ) -_file_name = 'brokers.toml' +_conf_names: set[str] = { + 'brokers', + 'trades', + 'watchlists', +} + _watchlists_data_path = os.path.join(_config_dir, 'watchlists.json') _context_defaults = dict( default_map={ @@ -129,23 +136,43 @@ def _override_config_dir( _config_dir = path -def get_broker_conf_path(): +def _conf_fn_w_ext( + name: str, +) -> str: + # change this if we ever change the config file format. + return f'{name}.toml' + + +def get_conf_path( + conf_name: str = 'brokers', + +) -> str: """Return the default config path normally under ``~/.config/piker`` on linux. Contains files such as: - brokers.toml - watchlists.toml + - trades.toml + + # maybe coming soon ;) - signals.toml - strats.toml """ - return os.path.join(_config_dir, _file_name) + assert conf_name in _conf_names + fn = _conf_fn_w_ext(conf_name) + return os.path.join( + _config_dir, + fn, + ) def repodir(): - """Return the abspath to the repo directory. - """ + ''' + Return the abspath to the repo directory. + + ''' dirpath = os.path.abspath( # we're 3 levels down in **this** module file dirname(dirname(os.path.realpath(__file__))) @@ -154,16 +181,27 @@ def repodir(): def load( + conf_name: str = 'brokers', path: str = None + ) -> (dict, str): - """Load broker config. - """ - path = path or get_broker_conf_path() + ''' + Load config file by name. + + ''' + path = path or get_conf_path(conf_name) if not os.path.isfile(path): - shutil.copyfile( - os.path.join(repodir(), 'config', 'brokers.toml'), - path, + fn = _conf_fn_w_ext(conf_name) + + template = os.path.join( + repodir(), + 'config', + fn ) + # try to copy in a template config to the user's directory + # if one exists. + if os.path.isfile(template): + shutil.copyfile(template, path) config = toml.load(path) log.debug(f"Read config file {path}") @@ -172,13 +210,17 @@ def load( def write( config: dict, # toml config as dict + name: str = 'brokers', path: str = None, + ) -> None: - """Write broker config to disk. + '''' + Write broker config to disk. Create a ``brokers.ini`` file if one does not exist. - """ - path = path or get_broker_conf_path() + + ''' + path = path or get_conf_path(name) dirname = os.path.dirname(path) if not os.path.isdir(dirname): log.debug(f"Creating config dir {_config_dir}") @@ -188,7 +230,10 @@ def write( raise ValueError( "Watch out you're trying to write a blank config!") - log.debug(f"Writing config file {path}") + log.debug( + f"Writing config `{name}` file to:\n" + f"{path}" + ) with open(path, 'w') as cf: return toml.dump(config, cf) @@ -218,4 +263,5 @@ def load_accounts( # our default paper engine entry accounts['paper'] = None + return accounts From 88ac2fda52a882e60b0d7d211bf1bc9bfeb60afd Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 28 May 2022 15:41:11 -0400 Subject: [PATCH 087/113] Aggretate cache resetting into a single ctx mngr method --- piker/ui/_curve.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index abe48929..11901987 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -18,6 +18,7 @@ Fast, smooth, sexy curves. """ +from contextlib import contextmanager as cm from typing import Optional import numpy as np @@ -38,7 +39,6 @@ from ._style import hcolor # # ohlc_to_m4_line, # ds_m4, # ) -from ._pathops import xy_downsample from ..log import get_logger @@ -216,22 +216,11 @@ class FastAppendCurve(pg.GraphicsObject): # self.fast_path.clear() self.fast_path = None - # self.disable_cache() - # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - + @cm def reset_cache(self) -> None: - self.disable_cache() - self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - - def disable_cache(self) -> None: - ''' - Disable the use of the pixel coordinate cache and trigger a geo event. - - ''' - # XXX: pretty annoying but, without this there's little - # artefacts on the append updates to the curve... self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # self.prepareGeometryChange() + yield + self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) def boundingRect(self): ''' @@ -285,8 +274,8 @@ class FastAppendCurve(pg.GraphicsObject): # actually terminates earlier so we don't need # this for the last step. w += self._last_w - ll = self._last_line - h += 1 #ll.y2() - ll.y1() + # ll = self._last_line + h += 1 # ll.y2() - ll.y1() # br = QPointF( # self._vr[-1], From d61b6364879f8c2ceae2cfd402c4c3ae06e3c26e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 29 May 2022 14:49:53 -0400 Subject: [PATCH 088/113] Auto-yrange overlays in interaction (downsampler) handler --- piker/ui/_interaction.py | 46 +++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 8e95855e..aa2cefc2 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -747,9 +747,8 @@ class ChartView(ViewBox): # flag to prevent triggering sibling charts from the same linked # set from recursion errors. - autoscale_linked_plots: bool = True, + autoscale_linked_plots: bool = False, name: Optional[str] = None, - # autoscale_overlays: bool = False, ) -> None: ''' @@ -760,7 +759,7 @@ class ChartView(ViewBox): data set. ''' - # print(f'YRANGE ON {self.name}') + # log.info(f'YRANGE ON {self.name}') profiler = pg.debug.Profiler( msg=f'`ChartView._set_yrange()`: `{self.name}`', disabled=not pg_profile_enabled(), @@ -795,7 +794,8 @@ class ChartView(ViewBox): # XXX: only compute the mxmn range # if none is provided as input! if not yrange: - yrange = self._maxmin() + yrange = self._maxmin( + ) if yrange is None: log.warning(f'No yrange provided for {self.name}!?') @@ -821,25 +821,6 @@ class ChartView(ViewBox): self.setYRange(ylow, yhigh) profiler(f'set limits: {(ylow, yhigh)}') - # TODO: maybe should be a method on the - # chart widget/item? - if autoscale_linked_plots: - # avoid recursion by sibling plots - linked = self.linkedsplits - plots = list(linked.subplots.copy().values()) - main = linked.chart - if main: - plots.append(main) - - # print(f'autoscaling linked: {plots}') - for chart in plots: - if chart and not chart._static_yrange: - chart.cv._set_yrange( - # bars_range=br, - autoscale_linked_plots=False, - ) - profiler('autoscaled linked plots') - profiler.finish() def enable_auto_yrange( @@ -911,7 +892,10 @@ class ChartView(ViewBox): else: return 0 - def maybe_downsample_graphics(self): + def maybe_downsample_graphics( + self, + autoscale_linked_plots: bool = True, + ): profiler = pg.debug.Profiler( msg=f'ChartView.maybe_downsample_graphics() for {self.name}', @@ -945,9 +929,17 @@ class ChartView(ViewBox): chart.update_graphics_from_flow( name, use_vr=True, - - # gets passed down into graphics obj - # profiler=profiler, ) + # for each overlay on this chart auto-scale the + # y-range to max-min values. + if autoscale_linked_plots: + overlay = chart.pi_overlay + if overlay: + for pi in overlay.overlays: + pi.vb._set_yrange( + # bars_range=br, + ) + profiler('autoscaled linked plots') + profiler(f'<{chart_name}>.update_graphics_from_flow({name})') From a9ec1a97dd54844dc8fc7de6a37360b71ce2780a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 29 May 2022 23:43:31 -0400 Subject: [PATCH 089/113] Vlm "rate" fsps, change maxmin callback name to include `multi_` --- piker/ui/_fsp.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 3d90f014..af03a9c6 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -635,7 +635,7 @@ async def open_vlm_displays( ) # force 0 to always be in view - def maxmin( + def multi_maxmin( names: list[str], ) -> tuple[float, float]: @@ -651,7 +651,7 @@ async def open_vlm_displays( return 0, mx - chart.view.maxmin = partial(maxmin, names=['volume']) + chart.view.maxmin = partial(multi_maxmin, names=['volume']) # TODO: fix the x-axis label issue where if you put # the axis on the left it's totally not lined up... @@ -741,19 +741,20 @@ async def open_vlm_displays( 'dolla_vlm', 'dark_vlm', ] - dvlm_rate_fields = [ - 'dvlm_rate', - 'dark_dvlm_rate', - ] + # dvlm_rate_fields = [ + # 'dvlm_rate', + # 'dark_dvlm_rate', + # ] trade_rate_fields = [ 'trade_rate', 'dark_trade_rate', ] group_mxmn = partial( - maxmin, + multi_maxmin, # keep both regular and dark vlm in view - names=fields + dvlm_rate_fields, + names=fields, + # names=fields + dvlm_rate_fields, ) # add custom auto range handler @@ -820,11 +821,11 @@ async def open_vlm_displays( ) await started.wait() - chart_curves( - dvlm_rate_fields, - dvlm_pi, - fr_shm, - ) + # chart_curves( + # dvlm_rate_fields, + # dvlm_pi, + # fr_shm, + # ) # TODO: is there a way to "sync" the dual axes such that only # one curve is needed? @@ -862,7 +863,7 @@ async def open_vlm_displays( ) # add custom auto range handler tr_pi.vb.maxmin = partial( - maxmin, + multi_maxmin, # keep both regular and dark vlm in view names=trade_rate_fields, ) From ab0def22c1d126925b91eca11b1de0ea2eaa44b3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 30 May 2022 09:26:41 -0400 Subject: [PATCH 090/113] Change flag name to `autoscale_overlays` --- piker/ui/_interaction.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index aa2cefc2..9d7159c3 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -759,9 +759,10 @@ class ChartView(ViewBox): data set. ''' - # log.info(f'YRANGE ON {self.name}') + name = self.name + # print(f'YRANGE ON {name}') profiler = pg.debug.Profiler( - msg=f'`ChartView._set_yrange()`: `{self.name}`', + msg=f'`ChartView._set_yrange()`: `{name}`', disabled=not pg_profile_enabled(), ms_threshold=ms_slower_then, delayed=True, @@ -794,12 +795,12 @@ class ChartView(ViewBox): # XXX: only compute the mxmn range # if none is provided as input! if not yrange: - yrange = self._maxmin( - ) + # flow = chart._flows[name] + yrange = self._maxmin() if yrange is None: - log.warning(f'No yrange provided for {self.name}!?') - print(f"WTF NO YRANGE {self.name}") + log.warning(f'No yrange provided for {name}!?') + print(f"WTF NO YRANGE {name}") return ylow, yhigh = yrange @@ -894,7 +895,7 @@ class ChartView(ViewBox): def maybe_downsample_graphics( self, - autoscale_linked_plots: bool = True, + autoscale_overlays: bool = True, ): profiler = pg.debug.Profiler( @@ -933,7 +934,7 @@ class ChartView(ViewBox): # for each overlay on this chart auto-scale the # y-range to max-min values. - if autoscale_linked_plots: + if autoscale_overlays: overlay = chart.pi_overlay if overlay: for pi in overlay.overlays: From 360643b32f7770ba5c6927269c45bb45e1b51b23 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 30 May 2022 09:37:33 -0400 Subject: [PATCH 091/113] Fix optional input `bars_range` type to match `Flow.datums_range()` --- piker/ui/_chart.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 31ca0604..329f15f5 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1251,7 +1251,9 @@ class ChartPlotWidget(pg.PlotWidget): def maxmin( self, name: Optional[str] = None, - bars_range: Optional[tuple[int, int, int, int]] = None, + bars_range: Optional[tuple[ + int, int, int, int, int, int + ]] = None, ) -> tuple[float, float]: ''' @@ -1260,6 +1262,7 @@ class ChartPlotWidget(pg.PlotWidget): If ``bars_range`` is provided use that range. ''' + # print(f'Chart[{self.name}].maxmin()') profiler = pg.debug.Profiler( msg=f'`{str(self)}.maxmin(name={name})`: `{self.name}`', disabled=not pg_profile_enabled(), @@ -1279,7 +1282,14 @@ class ChartPlotWidget(pg.PlotWidget): key = res = 0, 0 else: - first, l, lbar, rbar, r, last = bars_range or flow.datums_range() + ( + first, + l, + lbar, + rbar, + r, + last, + ) = bars_range or flow.datums_range() profiler(f'{self.name} got bars range') key = round(lbar), round(rbar) From 2c2c453932e4675f27265d25297a32183ca01007 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 30 May 2022 19:02:06 -0400 Subject: [PATCH 092/113] Reset line graphics on downsample step.. This was a bit of a nightmare to figure out but, it seems that the coordinate caching system will really be a dick (like the nickname for richard for you serious types) about leaving stale graphics if we don't reset the cache on downsample full-redraw updates...Sooo, instead we do this manual reset to avoid such artifacts and consequently (for now) return a `reset: bool` flag in the return tuple from `Renderer.render()` to indicate as such. Some further shite: - move the step mode `.draw_last()` equivalent graphics updates down with the rest.. - drop some superfluous `should_redraw` logic from `Renderer.render()` and compound it in the full path redraw block. --- piker/ui/_flows.py | 105 ++++++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index e74ee123..8610bb68 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -570,7 +570,6 @@ class Flow(msgspec.Struct): # , frozen=True): draw_last: bool = True slice_to_head: int = -1 - # input_data = None should_redraw: bool = False @@ -616,7 +615,8 @@ class Flow(msgspec.Struct): # , frozen=True): should_ds: bool = r._in_ds showing_src_data: bool = not r._in_ds - if getattr(graphics, '_step_mode', False): + step_mode = getattr(graphics, '_step_mode', False) + if step_mode: r.allocate_xy = to_step_format r.update_xy = update_step_xy @@ -636,16 +636,7 @@ class Flow(msgspec.Struct): # , frozen=True): x_last = x[-1] y_last = y[-1] - w = 0.5 - graphics._last_line = QLineF( - x_last - w, 0, - x_last + w, 0, - ) - graphics._last_step_rect = QRectF( - x_last - w, 0, - x_last + w, y_last, - ) - draw_last = False + draw_last = True # downsampling incremental state checking # check for and set std m4 downsample conditions @@ -660,10 +651,11 @@ class Flow(msgspec.Struct): # , frozen=True): f'{array_key} sampler change: {self._last_uppx} -> {uppx}' ) self._last_uppx = uppx + new_sample_rate = True showing_src_data = False - should_redraw = True should_ds = True + should_redraw = True elif ( uppx <= 2 @@ -672,17 +664,17 @@ class Flow(msgspec.Struct): # , frozen=True): # we should de-downsample back to our original # source data so we clear our path data in prep # to generate a new one from original source data. - should_redraw = True new_sample_rate = True - should_ds = False showing_src_data = True + should_ds = False + should_redraw = True # MAIN RENDER LOGIC: # - determine in view data and redraw on range change # - determine downsampling ops if needed # - (incrementally) update ``QPainterPath`` - path, data = r.render( + path, data, reset = r.render( read, array_key, profiler, @@ -702,29 +694,49 @@ class Flow(msgspec.Struct): # , frozen=True): **rkwargs, ) - # TODO: does this actuallly help us in any way (prolly should - # look at the source / ask ogi). - # graphics.prepareGeometryChange() - # assign output paths to graphicis obj - graphics.path = r.path - graphics.fast_path = r.fast_path + # XXX: SUPER UGGGHHH... without this we get stale cache + # graphics that don't update until you downsampler again.. + if reset: + with graphics.reset_cache(): + # assign output paths to graphicis obj + graphics.path = r.path + graphics.fast_path = r.fast_path + else: + # assign output paths to graphicis obj + graphics.path = r.path + graphics.fast_path = r.fast_path if draw_last and not bars: - # TODO: how to handle this draw last stuff.. - x = data['index'] - y = data[array_key] - graphics.draw_last(x, y) + # default line draw last call + if not step_mode: + with graphics.reset_cache(): + x = data['index'] + y = data[array_key] + graphics.draw_last(x, y) + + else: + w = 0.5 + graphics._last_line = QLineF( + x_last - w, 0, + x_last + w, 0, + ) + graphics._last_step_rect = QRectF( + x_last - w, 0, + x_last + w, y_last, + ) + + # TODO: does this actuallly help us in any way (prolly should + # look at the source / ask ogi). I think it avoid artifacts on + # wheel-scroll downsampling curve updates? + graphics.update() + profiler('.prepareGeometryChange()') elif bars and draw_last: draw_last() + graphics.update() + profiler('.update()') - profiler('draw last segment') - - graphics.update() - profiler('.update()') - - profiler('`graphics.update_from_array()` complete') return graphics @@ -1009,8 +1021,8 @@ class Renderer(msgspec.Struct): if use_vr: array = in_view - else: - ivl, ivr = xfirst, xlast + # else: + # ivl, ivr = xfirst, xlast hist = array[:slice_to_head] @@ -1069,24 +1081,22 @@ class Renderer(msgspec.Struct): ): zoom_or_append = True - if ( - view_range != last_vr - and ( - append_length > 1 - or zoom_or_append - ) - ): - should_redraw = True - self._last_vr = view_range if len(x_out): self._last_ivr = x_out[0], x_out[slice_to_head] - if prepend_length > 0: + # redraw conditions + if ( + prepend_length > 0 + or new_sample_rate + or append_length > 0 + or zoom_or_append + ): should_redraw = True path = self.path fast_path = self.fast_path + reset = False # redraw the entire source data if we have either of: # - no prior path graphic rendered or, @@ -1094,10 +1104,8 @@ class Renderer(msgspec.Struct): if ( path is None or should_redraw - or new_sample_rate - or prepend_length > 0 ): - # print("REDRAWING BRUH") + # print(f"{self.flow.name} -> REDRAWING BRUH") if new_sample_rate and showing_src_data: log.info(f'DEDOWN -> {array_key}') self._in_ds = False @@ -1109,6 +1117,7 @@ class Renderer(msgspec.Struct): y_out, uppx, ) + reset = True profiler(f'FULL PATH downsample redraw={should_ds}') self._in_ds = True @@ -1203,4 +1212,4 @@ class Renderer(msgspec.Struct): # update diff state since we've now rendered paths. self.last_read = new_read - return self.path, array + return self.path, array, reset From 6f00617bd3a341a9cc15138c9cd124efa491973e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 30 May 2022 20:01:40 -0400 Subject: [PATCH 093/113] Only do new "datum append" when visible in pixels The basic logic is now this: - when zooming out, uppx (units per pixel in x) can be >= 1 - if the uppx is `n` then the next pixel in view becomes occupied by a new datum-x-coordinate-value when the diff between the last datum step (since the last such update) is greater then the current uppx -> `datums_diff >= n` - if we're less then some constant uppx we just always update (because it's not costly enough and we're not downsampling. More or less this just avoids unnecessary real-time updates to flow graphics until they would actually be noticeable via the next pixel column on screen. --- piker/ui/_display.py | 58 +++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index c551fc98..e33c2c74 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -32,7 +32,7 @@ import trio import pendulum import pyqtgraph as pg -from .. import brokers +# from .. import brokers from ..data.feed import open_feed from ._axes import YAxisLabel from ._chart import ( @@ -263,6 +263,7 @@ async def graphics_update_loop( 'vars': { 'tick_margin': tick_margin, 'i_last': i_last, + 'i_last_append': i_last, 'last_mx_vlm': last_mx_vlm, 'last_mx': last_mx, 'last_mn': last_mn, @@ -320,8 +321,8 @@ def graphics_update_cycle( profiler = pg.debug.Profiler( msg=f'Graphics loop cycle for: `{chart.name}`', delayed=True, - # disabled=not pg_profile_enabled(), - disabled=True, + disabled=not pg_profile_enabled(), + # disabled=True, ms_threshold=ms_slower_then, # ms_threshold=1/12 * 1e3, @@ -340,7 +341,7 @@ def graphics_update_cycle( for sym, quote in ds.quotes.items(): # compute the first available graphic's x-units-per-pixel - xpx = vlm_chart.view.x_uppx() + uppx = vlm_chart.view.x_uppx() # NOTE: vlm may be written by the ``brokerd`` backend # event though a tick sample is not emitted. @@ -359,13 +360,32 @@ def graphics_update_cycle( i_diff = i_step - vars['i_last'] vars['i_last'] = i_step + append_diff = i_step - vars['i_last_append'] + + # update the "last datum" (aka extending the flow graphic with + # new data) only if the number of unit steps is >= the number of + # such unit steps per pixel (aka uppx). Iow, if the zoom level + # is such that a datum(s) update to graphics wouldn't span + # to a new pixel, we don't update yet. + do_append = (append_diff >= uppx) + if do_append: + vars['i_last_append'] = i_step + + do_rt_update = uppx < update_uppx + # print( + # f'append_diff:{append_diff}\n' + # f'uppx:{uppx}\n' + # f'do_append: {do_append}' + # ) + + # TODO: we should only run mxmn when we know + # an update is due via ``do_append`` above. ( brange, mx_in_view, mn_in_view, mx_vlm_in_view, ) = ds.maxmin() - l, lbar, rbar, r = brange mx = mx_in_view + tick_margin mn = mn_in_view - tick_margin @@ -389,8 +409,9 @@ def graphics_update_cycle( # left unless we get one of the following: if ( ( - i_diff > 0 # no new sample step - and xpx < 4 # chart is zoomed out very far + # i_diff > 0 # no new sample step + do_append + # and uppx < 4 # chart is zoomed out very far and liv ) or trigger_all @@ -399,6 +420,10 @@ def graphics_update_cycle( # pixel in a curve should show new data based on uppx # and then iff update curves and shift? chart.increment_view(steps=i_diff) + + if vlm_chart: + vlm_chart.increment_view(steps=i_diff) + profiler('view incremented') if vlm_chart: @@ -409,8 +434,8 @@ def graphics_update_cycle( if ( ( - xpx < update_uppx - or i_diff > 0 + do_rt_update + or do_append and liv ) or trigger_all @@ -454,14 +479,15 @@ def graphics_update_cycle( flow, curve_name, array_key=curve_name, - do_append=xpx < update_uppx, + # do_append=uppx < update_uppx, + do_append=do_append, ) # is this even doing anything? # (pretty sure it's the real-time # resizing from last quote?) fvb = flow.plot.vb fvb._set_yrange( - autoscale_linked_plots=False, + # autoscale_linked_plots=False, name=curve_name, ) @@ -510,13 +536,17 @@ def graphics_update_cycle( # update ohlc sampled price bars if ( - xpx < update_uppx - or i_diff > 0 + do_rt_update + or do_append or trigger_all ): + # TODO: we should always update the "last" datum + # since the current range should at least be updated + # to it's max/min on the last pixel. chart.update_graphics_from_flow( chart.name, - do_append=xpx < update_uppx, + # do_append=uppx < update_uppx, + do_append=do_append, ) # iterate in FIFO order per tick-frame From 3ab91deaec5d0d59894b6dc63906f6d909fd26cc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 31 May 2022 12:46:20 -0400 Subject: [PATCH 094/113] Drop all (old) unused state instance vars --- piker/ui/_curve.py | 13 ------------- piker/ui/_flows.py | 12 ++---------- piker/ui/_ohlc.py | 18 ------------------ 3 files changed, 2 insertions(+), 41 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 11901987..65e60428 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -84,8 +84,6 @@ class FastAppendCurve(pg.GraphicsObject): # brutaaalll, see comments within.. self.yData = None self.xData = None - # self._vr: Optional[tuple] = None - # self._avr: Optional[tuple] = None self._last_cap: int = 0 self._name = name @@ -98,14 +96,6 @@ class FastAppendCurve(pg.GraphicsObject): # we're basically only using the pen setting now... super().__init__(*args, **kwargs) - # self._xrange: tuple[int, int] = self.dataBounds(ax=0) - # self._xrange: Optional[tuple[int, int]] = None - # self._x_iv_range = None - - # self._last_draw = time.time() - # self._in_ds: bool = False - # self._last_uppx: float = 0 - # all history of curve is drawn in single px thickness pen = pg.mkPen(hcolor(color)) pen.setStyle(_line_styles[style]) @@ -162,9 +152,6 @@ class FastAppendCurve(pg.GraphicsObject): vr = self.viewRect() l, r = int(vr.left()), int(vr.right()) - # if not self._xrange: - # return 0 - start, stop = self._xrange lbar = max(l, start) rbar = min(r, stop) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 8610bb68..2086a3ec 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -358,20 +358,10 @@ class Flow(msgspec.Struct): # , frozen=True): is_ohlc: bool = False render: bool = True # toggle for display loop - # pre-graphics formatted data - gy: Optional[np.ndarray] = None - gx: Optional[np.ndarray] = None - - # pre-graphics update indices - _iflat_last: int = 0 - _iflat_first: int = 0 - # downsampling state _last_uppx: float = 0 _in_ds: bool = False - _graphics_tranform_fn: Optional[Callable[ShmArray, np.ndarray]] = None - # map from uppx -> (downsampled data, incremental graphics) _src_r: Optional[Renderer] = None _render_table: dict[ @@ -717,6 +707,8 @@ class Flow(msgspec.Struct): # , frozen=True): else: w = 0.5 + # lol, commenting this makes step curves + # all "black" for me :eyeroll:.. graphics._last_line = QLineF( x_last - w, 0, x_last + w, 0, diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index f10a4998..e5542609 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -129,11 +129,7 @@ class BarItems(pg.GraphicsObject): self.fast_path = QtGui.QPainterPath() self._xrange: tuple[int, int] - # self._yrange: tuple[float, float] self._vrange = None - - # TODO: don't render the full backing array each time - # self._path_data = None self._last_bar_lines: Optional[tuple[QLineF, ...]] = None # track the current length of drawable lines within the larger array @@ -212,16 +208,6 @@ class BarItems(pg.GraphicsObject): hb.bottomRight(), ) - # fp = self.fast_path - # if fp: - # fhb = fp.controlPointRect() - # print((hb_tl, hb_br)) - # print(fhb) - # hb_tl, hb_br = ( - # fhb.topLeft() + hb.topLeft(), - # fhb.bottomRight() + hb.bottomRight(), - # ) - # need to include last bar height or BR will be off mx_y = hb_br.y() mn_y = hb_tl.y() @@ -281,7 +267,3 @@ class BarItems(pg.GraphicsObject): p.setPen(self.bars_pen) p.drawPath(self.path) profiler(f'draw history path: {self.path.capacity()}') - - # if self.fast_path: - # p.drawPath(self.fast_path) - # profiler('draw fast path') From 8f1faf97ee5ebe484ca3b79f94ac38de3f1e1656 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 31 May 2022 12:46:50 -0400 Subject: [PATCH 095/113] Add todo for bars range reuse in interaction handler --- piker/ui/_interaction.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 9d7159c3..a659612a 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -939,6 +939,7 @@ class ChartView(ViewBox): if overlay: for pi in overlay.overlays: pi.vb._set_yrange( + # TODO: get the range once up front... # bars_range=br, ) profiler('autoscaled linked plots') From 57acc3bd29417923cb4e376b6034c6a8941335b5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 31 May 2022 13:57:10 -0400 Subject: [PATCH 096/113] Factor all per graphic `.draw_last()` methods into closures --- piker/ui/_curve.py | 17 ----- piker/ui/_flows.py | 173 ++++++++++++++++++++++++++++++++++----------- piker/ui/_ohlc.py | 40 ----------- 3 files changed, 133 insertions(+), 97 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 65e60428..89180200 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -160,23 +160,6 @@ class FastAppendCurve(pg.GraphicsObject): QLineF(lbar, 0, rbar, 0) ).length() - def draw_last( - self, - x: np.ndarray, - y: np.ndarray, - - ) -> None: - x_last = x[-1] - y_last = y[-1] - - # draw the "current" step graphic segment so it lines up with - # the "middle" of the current (OHLC) sample. - self._last_line = QLineF( - x[-2], y[-2], - x_last, y_last - ) - # self._last_w = x_last - x[-2] - # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. def getData(self): diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 2086a3ec..907446a2 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -23,7 +23,6 @@ incremental update. ''' from __future__ import annotations -from functools import partial from typing import ( Optional, Callable, @@ -58,6 +57,7 @@ from ._pathops import ( ) from ._ohlc import ( BarItems, + bar_from_ohlc_row, ) from ._curve import ( FastAppendCurve, @@ -245,18 +245,73 @@ def render_baritems( bars.update() draw_last = False - lasts = self.shm.array[-2:] - last = lasts[-1] if should_line: - def draw_last(): + + def draw_last_flattened_ohlc_line( + graphics: pg.GraphicsObject, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + ) -> None: + lasts = src_data[-2:] x, y = lasts['index'], lasts['close'] - curve.draw_last(x, y) + + # draw the "current" step graphic segment so it + # lines up with the "middle" of the current + # (OHLC) sample. + graphics._last_line = QLineF( + x[-2], y[-2], + x[-1], y[-1] + ) + + draw_last = draw_last_flattened_ohlc_line + else: - draw_last = partial( - bars.draw_last, - last, - ) + def draw_last_ohlc_bar( + graphics: pg.GraphicsObject, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + ) -> None: + last = src_data[-1] + # generate new lines objects for updatable "current bar" + graphics._last_bar_lines = bar_from_ohlc_row(last, graphics.w) + + # last bar update + i, o, h, l, last, v = last[ + ['index', 'open', 'high', 'low', 'close', 'volume'] + ] + # assert i == graphics.start_index - 1 + # assert i == last_index + body, larm, rarm = graphics._last_bar_lines + + # XXX: is there a faster way to modify this? + rarm.setLine(rarm.x1(), last, rarm.x2(), last) + + # writer is responsible for changing open on "first" volume of bar + larm.setLine(larm.x1(), o, larm.x2(), o) + + if l != h: # noqa + + if body is None: + body = graphics._last_bar_lines[0] = QLineF(i, l, i, h) + else: + # update body + body.setLine(i, l, i, h) + + # XXX: pretty sure this is causing an issue where the + # bar has a large upward move right before the next + # sample and the body is getting set to None since the + # next bar is flat but the shm array index update wasn't + # read by the time this code runs. Iow we're doing this + # removal of the body for a bar index that is now out of + # date / from some previous sample. It's weird though + # because i've seen it do this to bars i - 3 back? + + draw_last = draw_last_ohlc_bar return ( graphics, @@ -355,6 +410,12 @@ class Flow(msgspec.Struct): # , frozen=True): graphics: pg.GraphicsObject _shm: ShmArray + draw_last_datum: Optional[ + Callable[ + [np.ndarray, str], + tuple[np.ndarray] + ] + ] = None is_ohlc: bool = False render: bool = True # toggle for display loop @@ -543,7 +604,7 @@ class Flow(msgspec.Struct): # , frozen=True): ) # shm read and slice to view read = ( - xfirst, xlast, array, + xfirst, xlast, src_array, ivl, ivr, in_view, ) = self.read() @@ -618,14 +679,6 @@ class Flow(msgspec.Struct): # , frozen=True): # corrent yet for step curves.. remove this to see it. should_redraw = True - # TODO: remove this and instead place all step curve - # updating into pre-path data render callbacks. - # full input data - x = array['index'] - y = array[array_key] - x_last = x[-1] - y_last = y[-1] - draw_last = True # downsampling incremental state checking @@ -698,34 +751,74 @@ class Flow(msgspec.Struct): # , frozen=True): graphics.fast_path = r.fast_path if draw_last and not bars: - # default line draw last call + if not step_mode: - with graphics.reset_cache(): - x = data['index'] - y = data[array_key] - graphics.draw_last(x, y) + + def draw_last_line( + graphics: pg.GraphicsObject, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + + ) -> None: + # default line draw last call + with graphics.reset_cache(): + x = render_data['index'] + y = render_data[array_key] + x_last = x[-1] + y_last = y[-1] + + # draw the "current" step graphic segment so it + # lines up with the "middle" of the current + # (OHLC) sample. + graphics._last_line = QLineF( + x[-2], y[-2], + x_last, y_last + ) + + draw_last_line(graphics, path, src_array, data, reset) else: - w = 0.5 - # lol, commenting this makes step curves - # all "black" for me :eyeroll:.. - graphics._last_line = QLineF( - x_last - w, 0, - x_last + w, 0, - ) - graphics._last_step_rect = QRectF( - x_last - w, 0, - x_last + w, y_last, - ) - # TODO: does this actuallly help us in any way (prolly should - # look at the source / ask ogi). I think it avoid artifacts on - # wheel-scroll downsampling curve updates? - graphics.update() - profiler('.prepareGeometryChange()') + def draw_last_step( + graphics: pg.GraphicsObject, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + + ) -> None: + w = 0.5 + # TODO: remove this and instead place all step curve + # updating into pre-path data render callbacks. + # full input data + x = src_array['index'] + y = src_array[array_key] + x_last = x[-1] + y_last = y[-1] + + # lol, commenting this makes step curves + # all "black" for me :eyeroll:.. + graphics._last_line = QLineF( + x_last - w, 0, + x_last + w, 0, + ) + graphics._last_step_rect = QRectF( + x_last - w, 0, + x_last + w, y_last, + ) + + draw_last_step(graphics, path, src_array, data, reset) + + # TODO: does this actuallly help us in any way (prolly should + # look at the source / ask ogi). I think it avoid artifacts on + # wheel-scroll downsampling curve updates? + graphics.update() + profiler('.prepareGeometryChange()') elif bars and draw_last: - draw_last() + draw_last(graphics, path, src_array, data, reset) graphics.update() profiler('.update()') diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index e5542609..ad449597 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -32,8 +32,6 @@ from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor from ..log import get_logger from ._curve import FastAppendCurve -from ._compression import ohlc_flatten -from ._pathops import gen_ohlc_qpath if TYPE_CHECKING: from ._chart import LinkedSplits @@ -148,44 +146,6 @@ class BarItems(pg.GraphicsObject): else: return 0 - def draw_last( - self, - last: np.ndarray, - - ) -> None: - # generate new lines objects for updatable "current bar" - self._last_bar_lines = bar_from_ohlc_row(last, self.w) - - # last bar update - i, o, h, l, last, v = last[ - ['index', 'open', 'high', 'low', 'close', 'volume'] - ] - # assert i == self.start_index - 1 - # assert i == last_index - body, larm, rarm = self._last_bar_lines - - # XXX: is there a faster way to modify this? - rarm.setLine(rarm.x1(), last, rarm.x2(), last) - - # writer is responsible for changing open on "first" volume of bar - larm.setLine(larm.x1(), o, larm.x2(), o) - - if l != h: # noqa - - if body is None: - body = self._last_bar_lines[0] = QLineF(i, l, i, h) - else: - # update body - body.setLine(i, l, i, h) - - # XXX: pretty sure this is causing an issue where the bar has - # a large upward move right before the next sample and the body - # is getting set to None since the next bar is flat but the shm - # array index update wasn't read by the time this code runs. Iow - # we're doing this removal of the body for a bar index that is - # now out of date / from some previous sample. It's weird - # though because i've seen it do this to bars i - 3 back? - def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect From a7ff47158bb3730f1728d82ad46d5168122a310b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 31 May 2022 18:07:22 -0400 Subject: [PATCH 097/113] Pass tsdb flag when db is up XD --- piker/data/feed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index 561d063b..c49ab0fe 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -338,7 +338,7 @@ async def start_backfill( log.debug(f'New datetime index:\n{pformat(dtrange)}') for end_dt in dtrange: - log.warning(f'Yielding next frame start {end_dt}') + log.info(f'Yielding next frame start {end_dt}') start = yield end_dt # if caller sends a new start date, reset to that @@ -722,6 +722,7 @@ async def manage_history( bfqsn, shm, last_tsdb_dt=last_tsdb_dt, + tsdb_is_up=True, storage=storage, ) ) From fc24f5efd1826100275a3c4d5208e4250674a0b2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 31 May 2022 18:07:51 -0400 Subject: [PATCH 098/113] Iterate 1s and 1m from tsdb series --- piker/data/marketstore.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/piker/data/marketstore.py b/piker/data/marketstore.py index 43b15671..4d1c91ad 100644 --- a/piker/data/marketstore.py +++ b/piker/data/marketstore.py @@ -639,12 +639,13 @@ async def tsdb_history_update( tsdb_arrays = await storage.read_ohlcv(fqsn) # hist diffing if tsdb_arrays: - onesec = tsdb_arrays[1] - - # these aren't currently used but can be referenced from - # within the embedded ipython shell below. - to_append = ohlcv[ohlcv['time'] > onesec['Epoch'][-1]] - to_prepend = ohlcv[ohlcv['time'] < onesec['Epoch'][0]] + for secs in (1, 60): + ts = tsdb_arrays.get(secs) + if ts is not None and len(ts): + # these aren't currently used but can be referenced from + # within the embedded ipython shell below. + to_append = ohlcv[ohlcv['time'] > ts['Epoch'][-1]] + to_prepend = ohlcv[ohlcv['time'] < ts['Epoch'][0]] profiler('Finished db arrays diffs') From 363ba8f9ae527649a3bcf9d27ce10f96baaa47b5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 1 Jun 2022 12:13:08 -0400 Subject: [PATCH 099/113] Only drop throttle feeds if channel disconnects? --- piker/data/_sampling.py | 45 +++++++++++++++++++++++++++++++---------- piker/data/feed.py | 13 +++++++++--- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/piker/data/_sampling.py b/piker/data/_sampling.py index 10dc43f6..fda93e21 100644 --- a/piker/data/_sampling.py +++ b/piker/data/_sampling.py @@ -22,7 +22,7 @@ financial data flows. from __future__ import annotations from collections import Counter import time -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Optional, Union import tractor import trio @@ -32,6 +32,7 @@ from ..log import get_logger if TYPE_CHECKING: from ._sharedmem import ShmArray + from .feed import _FeedsBus log = get_logger(__name__) @@ -219,7 +220,7 @@ async def iter_ohlc_periods( async def sample_and_broadcast( - bus: '_FeedsBus', # noqa + bus: _FeedsBus, # noqa shm: ShmArray, quote_stream: trio.abc.ReceiveChannel, brokername: str, @@ -298,7 +299,13 @@ async def sample_and_broadcast( # end up triggering backpressure which which will # eventually block this producer end of the feed and # thus other consumers still attached. - subs = bus._subscribers[broker_symbol.lower()] + subs: list[ + tuple[ + Union[tractor.MsgStream, trio.MemorySendChannel], + tractor.Context, + Optional[float], # tick throttle in Hz + ] + ] = bus._subscribers[broker_symbol.lower()] # NOTE: by default the broker backend doesn't append # it's own "name" into the fqsn schema (but maybe it @@ -307,7 +314,7 @@ async def sample_and_broadcast( bsym = f'{broker_symbol}.{brokername}' lags: int = 0 - for (stream, tick_throttle) in subs: + for (stream, ctx, tick_throttle) in subs: try: with trio.move_on_after(0.2) as cs: @@ -319,11 +326,11 @@ async def sample_and_broadcast( (bsym, quote) ) except trio.WouldBlock: - ctx = getattr(stream, '_ctx', None) + chan = ctx.chan if ctx: log.warning( f'Feed overrun {bus.brokername} ->' - f'{ctx.channel.uid} !!!' + f'{chan.uid} !!!' ) else: key = id(stream) @@ -333,11 +340,26 @@ async def sample_and_broadcast( f'feed @ {tick_throttle} Hz' ) if overruns[key] > 6: - log.warning( - f'Dropping consumer {stream}' - ) - await stream.aclose() - raise trio.BrokenResourceError + # TODO: should we check for the + # context being cancelled? this + # could happen but the + # channel-ipc-pipe is still up. + if not chan.connected(): + log.warning( + 'Dropping broken consumer:\n' + f'{broker_symbol}:' + f'{ctx.cid}@{chan.uid}' + ) + await stream.aclose() + raise trio.BrokenResourceError + else: + log.warning( + 'Feed getting overrun bro!\n' + f'{broker_symbol}:' + f'{ctx.cid}@{chan.uid}' + ) + continue + else: await stream.send( {bsym: quote} @@ -482,6 +504,7 @@ async def uniform_rate_send( # if the feed consumer goes down then drop # out of this rate limiter log.warning(f'{stream} closed') + await stream.aclose() return # reset send cycle state diff --git a/piker/data/feed.py b/piker/data/feed.py index c49ab0fe..94c2f81d 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -33,6 +33,7 @@ from typing import ( Generator, Awaitable, TYPE_CHECKING, + Union, ) import trio @@ -117,7 +118,13 @@ class _FeedsBus(BaseModel): # https://github.com/samuelcolvin/pydantic/issues/2816 _subscribers: dict[ str, - list[tuple[tractor.MsgStream, Optional[float]]] + list[ + tuple[ + Union[tractor.MsgStream, trio.MemorySendChannel], + tractor.Context, + Optional[float], # tick throttle in Hz + ] + ] ] = {} async def start_task( @@ -1118,10 +1125,10 @@ async def open_feed_bus( recv, stream, ) - sub = (send, tick_throttle) + sub = (send, ctx, tick_throttle) else: - sub = (stream, tick_throttle) + sub = (stream, ctx, tick_throttle) subs = bus._subscribers[bfqsn] subs.append(sub) From 064d18539521f6f9f304cbcbdd72bee18b275ce0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 1 Jun 2022 12:13:31 -0400 Subject: [PATCH 100/113] Drop pointless geo call from `.pain()` --- piker/ui/_curve.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 89180200..64922356 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -282,7 +282,6 @@ class FastAppendCurve(pg.GraphicsObject): disabled=not pg_profile_enabled(), ms_threshold=ms_slower_then, ) - self.prepareGeometryChange() if ( self._step_mode From b71e8c5e6d02a5cc7bee607677c6fddbc176b24a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 1 Jun 2022 12:14:15 -0400 Subject: [PATCH 101/113] Guard against empty source history slice output --- piker/ui/_flows.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 907446a2..e0cc21f8 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -256,7 +256,8 @@ def render_baritems( reset: bool, ) -> None: lasts = src_data[-2:] - x, y = lasts['index'], lasts['close'] + x = lasts['index'] + y = lasts['close'] # draw the "current" step graphic segment so it # lines up with the "middle" of the current @@ -717,7 +718,7 @@ class Flow(msgspec.Struct): # , frozen=True): # - determine downsampling ops if needed # - (incrementally) update ``QPainterPath`` - path, data, reset = r.render( + out = r.render( read, array_key, profiler, @@ -738,6 +739,12 @@ class Flow(msgspec.Struct): # , frozen=True): **rkwargs, ) + if not out: + log.warning(f'{self.name} failed to render!?') + return graphics + + path, data, reset = out + # XXX: SUPER UGGGHHH... without this we get stale cache # graphics that don't update until you downsampler again.. if reset: @@ -974,7 +981,7 @@ class Renderer(msgspec.Struct): # the target display(s) on the sys. # if no_path_yet: # graphics.path.reserve(int(500e3)) - path=path, # path re-use / reserving + # path=path, # path re-use / reserving ) # avoid mem allocs if possible @@ -1113,6 +1120,9 @@ class Renderer(msgspec.Struct): # xy-path data transform: convert source data to a format # able to be passed to a `QPainterPath` rendering routine. + if not len(hist): + return + x_out, y_out, connect = self.format_xy( self, # TODO: hist here should be the pre-sliced From e6d03ba97fe8e9914ba2bc2202a5ba82d101db40 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 1 Jun 2022 14:42:39 -0400 Subject: [PATCH 102/113] Add missing f-str prefix --- piker/clearing/_ems.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index e00676f2..17f9be1a 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -80,7 +80,9 @@ def mk_check( return check_lt - raise ValueError('trigger: {trigger_price}, last: {known_last}') + raise ValueError( + f'trigger: {trigger_price}, last: {known_last}' + ) @dataclass From 80835d4e04fe86bda09c6dc458a4fce4080b5c8b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 1 Jun 2022 15:01:30 -0400 Subject: [PATCH 103/113] More detailed rt feed drop logging --- piker/data/_sampling.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/piker/data/_sampling.py b/piker/data/_sampling.py index fda93e21..77b15d7f 100644 --- a/piker/data/_sampling.py +++ b/piker/data/_sampling.py @@ -336,7 +336,8 @@ async def sample_and_broadcast( key = id(stream) overruns[key] += 1 log.warning( - f'Feed overrun {bus.brokername} -> ' + f'Feed overrun {broker_symbol}' + '@{bus.brokername} -> ' f'feed @ {tick_throttle} Hz' ) if overruns[key] > 6: @@ -375,11 +376,12 @@ async def sample_and_broadcast( trio.ClosedResourceError, trio.EndOfChannel, ): - ctx = getattr(stream, '_ctx', None) + chan = ctx.chan if ctx: log.warning( - f'{ctx.chan.uid} dropped ' - '`brokerd`-quotes-feed connection' + 'Dropped `brokerd`-quotes-feed connection:\n' + f'{broker_symbol}:' + f'{ctx.cid}@{chan.uid}' ) if tick_throttle: assert stream._closed @@ -392,7 +394,11 @@ async def sample_and_broadcast( try: subs.remove((stream, tick_throttle)) except ValueError: - log.error(f'{stream} was already removed from subs!?') + log.error( + f'Stream was already removed from subs!?\n' + f'{broker_symbol}:' + f'{ctx.cid}@{chan.uid}' + ) # TODO: a less naive throttler, here's some snippets: From 0f4bfcdf22218bc611627b52e05ac30e428016e5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Jun 2022 13:34:36 -0400 Subject: [PATCH 104/113] Drop global pg settings --- piker/ui/_exec.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index 7b69acef..1d1a9c3d 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -49,10 +49,6 @@ from . import _style log = get_logger(__name__) # pyqtgraph global config -# might as well enable this for now? -pg.useOpenGL = True -pg.enableExperimental = True - # engage core tweaks that give us better response # latency then the average pg user _do_overrides() From 4138cef512de08e9957bc8c0b5638d5499ad505b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Jun 2022 13:35:01 -0400 Subject: [PATCH 105/113] Drop old state from `BarsItems` --- piker/ui/_ohlc.py | 41 +++-------------------------------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index ad449597..d4a93065 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -31,7 +31,6 @@ from PyQt5.QtCore import QLineF, QPointF from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor from ..log import get_logger -from ._curve import FastAppendCurve if TYPE_CHECKING: from ._chart import LinkedSplits @@ -42,6 +41,7 @@ log = get_logger(__name__) def bar_from_ohlc_row( row: np.ndarray, + # 0.5 is no overlap between arms, 1.0 is full overlap w: float = 0.43 ) -> tuple[QLineF]: @@ -87,9 +87,6 @@ class BarItems(pg.GraphicsObject): ''' sigPlotChanged = QtCore.pyqtSignal(object) - # 0.5 is no overlap between arms, 1.0 is full overlap - w: float = 0.43 - def __init__( self, linked: LinkedSplits, @@ -109,42 +106,13 @@ class BarItems(pg.GraphicsObject): self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2) self._name = name - self._ds_line_xy: Optional[ - tuple[np.ndarray, np.ndarray] - ] = None - - # NOTE: this prevents redraws on mouse interaction which is - # a huge boon for avg interaction latency. - - # TODO: one question still remaining is if this makes trasform - # interactions slower (such as zooming) and if so maybe if/when - # we implement a "history" mode for the view we disable this in - # that mode? self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - self._pi = plotitem self.path = QtGui.QPainterPath() - self.fast_path = QtGui.QPainterPath() - - self._xrange: tuple[int, int] - self._vrange = None self._last_bar_lines: Optional[tuple[QLineF, ...]] = None - # track the current length of drawable lines within the larger array - self.start_index: int = 0 - self.stop_index: int = 0 - - # downsampler-line state - self._in_ds: bool = False - self._ds_line: Optional[FastAppendCurve] = None - self._dsi: tuple[int, int] = 0, 0 - self._xs_in_px: float = 0 - def x_uppx(self) -> int: - if self._ds_line: - return self._ds_line.x_uppx() - else: - return 0 + # we expect the downsample curve report this. + return 0 def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect @@ -203,9 +171,6 @@ class BarItems(pg.GraphicsObject): ) -> None: - if self._in_ds: - return - profiler = pg.debug.Profiler( disabled=not pg_profile_enabled(), ms_threshold=ms_slower_then, From c518553aa9196abd09876b7e9f7f3d84c93985ec Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Jun 2022 13:36:55 -0400 Subject: [PATCH 106/113] Add new curve doc string --- piker/ui/_curve.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 64922356..0d32749e 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -55,15 +55,28 @@ _line_styles: dict[str, int] = { class FastAppendCurve(pg.GraphicsObject): ''' - A faster, append friendly version of ``pyqtgraph.PlotCurveItem`` - built for real-time data updates. + A faster, simpler, append friendly version of + ``pyqtgraph.PlotCurveItem`` built for highly customizable real-time + updates. - The main difference is avoiding regeneration of the entire - historical path where possible and instead only updating the "new" - segment(s) via a ``numpy`` array diff calc. Further the "last" - graphic segment is drawn independently such that near-term (high - frequency) discrete-time-sampled style updates don't trigger a full - path redraw. + This type is a much stripped down version of a ``pyqtgraph`` style "graphics object" in + the sense that the internal lower level graphics which are drawn in the ``.paint()`` method + are actually rendered outside of this class entirely and instead are assigned as state + (instance vars) here and then drawn during a Qt graphics cycle. + + The main motivation for this more modular, composed design is that + lower level graphics data can be rendered in different threads and + then read and drawn in this main thread without having to worry + about dealing with Qt's concurrency primitives. See + ``piker.ui._flows.Renderer`` for details and logic related to lower + level path generation and incremental update. The main differences in + the path generation code include: + + - avoiding regeneration of the entire historical path where possible and instead + only updating the "new" segment(s) via a ``numpy`` array diff calc. + - here, the "last" graphics datum-segment is drawn independently + such that near-term (high frequency) discrete-time-sampled style + updates don't trigger a full path redraw. ''' def __init__( @@ -89,6 +102,9 @@ class FastAppendCurve(pg.GraphicsObject): self._name = name self.path: Optional[QtGui.QPainterPath] = None + # additional path used for appends which tries to avoid + # triggering an update/redraw of the presumably larger + # historical ``.path`` above. self.use_fpath = use_fpath self.fast_path: Optional[QtGui.QPainterPath] = None @@ -119,11 +135,13 @@ class FastAppendCurve(pg.GraphicsObject): # self._fill = True self._brush = pg.functions.mkBrush(hcolor(fill_color or color)) + # NOTE: this setting seems to mostly prevent redraws on mouse + # interaction which is a huge boon for avg interaction latency. + # TODO: one question still remaining is if this makes trasform # interactions slower (such as zooming) and if so maybe if/when # we implement a "history" mode for the view we disable this in # that mode? - # if step_mode: # don't enable caching by default for the case where the # only thing drawn is the "last" line segment which can # have a weird artifact where it won't be fully drawn to its From d770867163b1e240ce24472d0545cc9ac25b3dec Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Jun 2022 13:38:14 -0400 Subject: [PATCH 107/113] Drop width arg to bar lines factory --- piker/ui/_flows.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index e0cc21f8..c5e813ea 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -254,6 +254,7 @@ def render_baritems( src_data: np.ndarray, render_data: np.ndarray, reset: bool, + ) -> None: lasts = src_data[-2:] x = lasts['index'] @@ -276,10 +277,12 @@ def render_baritems( src_data: np.ndarray, render_data: np.ndarray, reset: bool, + ) -> None: last = src_data[-1] + # generate new lines objects for updatable "current bar" - graphics._last_bar_lines = bar_from_ohlc_row(last, graphics.w) + graphics._last_bar_lines = bar_from_ohlc_row(last) # last bar update i, o, h, l, last, v = last[ From 736178adfd78f8bc5fe5ed1bfea0583e27c7324f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Jun 2022 18:11:59 -0400 Subject: [PATCH 108/113] Rename `FastAppendCurve` -> `Curve` --- piker/ui/_chart.py | 4 ++-- piker/ui/_curve.py | 4 ++-- piker/ui/_flows.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 329f15f5..bbab5a41 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -50,7 +50,7 @@ from ._cursor import ( from ..data._sharedmem import ShmArray from ._l1 import L1Labels from ._ohlc import BarItems -from ._curve import FastAppendCurve +from ._curve import Curve from ._style import ( hcolor, CHART_MARGINS, @@ -1069,7 +1069,7 @@ class ChartPlotWidget(pg.PlotWidget): # yah, we wrote our own B) data = shm.array - curve = FastAppendCurve( + curve = Curve( # antialias=True, name=name, diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 0d32749e..2cf9f0b7 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -53,7 +53,7 @@ _line_styles: dict[str, int] = { } -class FastAppendCurve(pg.GraphicsObject): +class Curve(pg.GraphicsObject): ''' A faster, simpler, append friendly version of ``pyqtgraph.PlotCurveItem`` built for highly customizable real-time @@ -296,7 +296,7 @@ class FastAppendCurve(pg.GraphicsObject): ) -> None: profiler = pg.debug.Profiler( - msg=f'FastAppendCurve.paint(): `{self._name}`', + msg=f'Curve.paint(): `{self._name}`', disabled=not pg_profile_enabled(), ms_threshold=ms_slower_then, ) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index c5e813ea..c2e6ec09 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -60,7 +60,7 @@ from ._ohlc import ( bar_from_ohlc_row, ) from ._curve import ( - FastAppendCurve, + Curve, ) from ..log import get_logger @@ -175,7 +175,7 @@ def render_baritems( format_xy=ohlc_flat_to_xy, ) - curve = FastAppendCurve( + curve = Curve( name=f'{flow.name}_ds_ohlc', color=bars._color, ) @@ -661,7 +661,7 @@ class Flow(msgspec.Struct): # , frozen=True): last_read=read, ) - # ``FastAppendCurve`` case: + # ``Curve`` case: array_key = array_key or self.name # print(array_key) From 55772efb34b01ab633edd8e9d9771ae010ad5748 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Jun 2022 10:18:32 -0400 Subject: [PATCH 109/113] Bleh, try avoiding the too many files bug-thing.. --- piker/data/_sharedmem.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index 47d58d3e..1172fc7b 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -20,6 +20,7 @@ NumPy compatible shared memory buffers for real-time IPC streaming. """ from __future__ import annotations from sys import byteorder +import time from typing import Optional from multiprocessing.shared_memory import SharedMemory, _USE_POSIX @@ -546,7 +547,21 @@ def attach_shm_array( # https://stackoverflow.com/a/11103289 # attach to array buffer and view as per dtype - shm = SharedMemory(name=key) + _err: Optional[Exception] = None + for _ in range(3): + try: + shm = SharedMemory( + name=key, + create=False, + ) + break + except OSError as oserr: + _err = oserr + time.sleep(0.1) + else: + if _err: + raise _err + shmarr = np.ndarray( (size,), dtype=token.dtype, From a66934a49d7eea6d368eb175ceee38a05f8cb6f6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Jun 2022 13:55:34 -0400 Subject: [PATCH 110/113] Add `Curve` sub-types with new custom graphics API Instead of using a bunch of internal logic to modify low level paint-able elements create a `Curve` lineage that allows for graphics "style" customization via a small set of public methods: - `Curve.declare_paintables()` to allow setup of state/elements to be drawn in later methods. - `.sub_paint()` to allow painting additional elements along with the defaults. - `.sub_br()` to customize the `.boundingRect()` dimensions. - `.draw_last_datum()` which is expected to produce the paintable elements which will show the last datum in view. Introduce the new sub-types and load as necessary in `ChartPlotWidget.draw_curve()`: - `FlattenedOHLC` - `StepCurve` Reimplement all `.draw_last()` routines as a `Curve` method and call it the same way from `Flow.update_graphics()` --- piker/ui/_chart.py | 33 +++---- piker/ui/_curve.py | 212 +++++++++++++++++++++++++++++++++++++-------- piker/ui/_flows.py | 198 ++++++------------------------------------ piker/ui/_ohlc.py | 50 ++++++++++- 4 files changed, 264 insertions(+), 229 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index bbab5a41..7b40f0d7 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -50,7 +50,10 @@ from ._cursor import ( from ..data._sharedmem import ShmArray from ._l1 import L1Labels from ._ohlc import BarItems -from ._curve import Curve +from ._curve import ( + Curve, + StepCurve, +) from ._style import ( hcolor, CHART_MARGINS, @@ -1051,6 +1054,7 @@ class ChartPlotWidget(pg.PlotWidget): color: Optional[str] = None, add_label: bool = True, pi: Optional[pg.PlotItem] = None, + step_mode: bool = False, **pdi_kwargs, @@ -1067,29 +1071,18 @@ class ChartPlotWidget(pg.PlotWidget): data_key = array_key or name - # yah, we wrote our own B) - data = shm.array - curve = Curve( - # antialias=True, + curve_type = { + None: Curve, + 'step': StepCurve, + # TODO: + # 'bars': BarsItems + }['step' if step_mode else None] + + curve = curve_type( name=name, - - # XXX: pretty sure this is just more overhead - # on data reads and makes graphics rendering no faster - # clipToView=True, - **pdi_kwargs, ) - # XXX: see explanation for different caching modes: - # https://stackoverflow.com/a/39410081 - # seems to only be useful if we don't re-generate the entire - # QPainterPath every time - # curve.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - # don't ever use this - it's a colossal nightmare of artefacts - # and is disastrous for performance. - # curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache) - pi = pi or self.plotItem self._flows[data_key] = Flow( diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 2cf9f0b7..8feb24b9 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -19,20 +19,24 @@ Fast, smooth, sexy curves. """ from contextlib import contextmanager as cm -from typing import Optional +from typing import Optional, Callable import numpy as np import pyqtgraph as pg -from PyQt5 import QtGui, QtWidgets +from PyQt5 import QtWidgets from PyQt5.QtWidgets import QGraphicsItem from PyQt5.QtCore import ( Qt, QLineF, QSizeF, QRectF, + # QRect, QPointF, ) - +from PyQt5.QtGui import ( + QPainter, + QPainterPath, +) from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor # from ._compression import ( @@ -59,10 +63,12 @@ class Curve(pg.GraphicsObject): ``pyqtgraph.PlotCurveItem`` built for highly customizable real-time updates. - This type is a much stripped down version of a ``pyqtgraph`` style "graphics object" in - the sense that the internal lower level graphics which are drawn in the ``.paint()`` method - are actually rendered outside of this class entirely and instead are assigned as state - (instance vars) here and then drawn during a Qt graphics cycle. + This type is a much stripped down version of a ``pyqtgraph`` style + "graphics object" in the sense that the internal lower level + graphics which are drawn in the ``.paint()`` method are actually + rendered outside of this class entirely and instead are assigned as + state (instance vars) here and then drawn during a Qt graphics + cycle. The main motivation for this more modular, composed design is that lower level graphics data can be rendered in different threads and @@ -72,13 +78,20 @@ class Curve(pg.GraphicsObject): level path generation and incremental update. The main differences in the path generation code include: - - avoiding regeneration of the entire historical path where possible and instead - only updating the "new" segment(s) via a ``numpy`` array diff calc. + - avoiding regeneration of the entire historical path where possible + and instead only updating the "new" segment(s) via a ``numpy`` + array diff calc. - here, the "last" graphics datum-segment is drawn independently such that near-term (high frequency) discrete-time-sampled style updates don't trigger a full path redraw. ''' + + # sub-type customization methods + sub_br: Optional[Callable] = None + sub_paint: Optional[Callable] = None + declare_paintables: Optional[Callable] = None + def __init__( self, *args, @@ -94,19 +107,20 @@ class Curve(pg.GraphicsObject): ) -> None: + self._name = name + # brutaaalll, see comments within.. self.yData = None self.xData = None - self._last_cap: int = 0 - self._name = name - self.path: Optional[QtGui.QPainterPath] = None + # self._last_cap: int = 0 + self.path: Optional[QPainterPath] = None # additional path used for appends which tries to avoid # triggering an update/redraw of the presumably larger # historical ``.path`` above. self.use_fpath = use_fpath - self.fast_path: Optional[QtGui.QPainterPath] = None + self.fast_path: Optional[QPainterPath] = None # TODO: we can probably just dispense with the parent since # we're basically only using the pen setting now... @@ -125,12 +139,12 @@ class Curve(pg.GraphicsObject): # self.last_step_pen = pg.mkPen(hcolor(color), width=2) self.last_step_pen = pg.mkPen(pen, width=2) - self._last_line: Optional[QLineF] = None - self._last_step_rect: Optional[QRectF] = None + # self._last_line: Optional[QLineF] = None + self._last_line = QLineF() self._last_w: float = 1 # flat-top style histogram-like discrete curve - self._step_mode: bool = step_mode + # self._step_mode: bool = step_mode # self._fill = True self._brush = pg.functions.mkBrush(hcolor(fill_color or color)) @@ -148,6 +162,21 @@ class Curve(pg.GraphicsObject): # endpoint (something we saw on trade rate curves) self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + # XXX: see explanation for different caching modes: + # https://stackoverflow.com/a/39410081 + # seems to only be useful if we don't re-generate the entire + # QPainterPath every time + # curve.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + + # don't ever use this - it's a colossal nightmare of artefacts + # and is disastrous for performance. + # curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache) + + # allow sub-type customization + declare = self.declare_paintables + if declare: + declare() + # TODO: probably stick this in a new parent # type which will contain our own version of # what ``PlotCurveItem`` had in terms of base @@ -215,7 +244,7 @@ class Curve(pg.GraphicsObject): Compute and then cache our rect. ''' if self.path is None: - return QtGui.QPainterPath().boundingRect() + return QPainterPath().boundingRect() else: # dynamically override this method after initial # path is created to avoid requiring the above None check @@ -227,14 +256,15 @@ class Curve(pg.GraphicsObject): Post init ``.boundingRect()```. ''' - hb = self.path.controlPointRect() # hb = self.path.boundingRect() + hb = self.path.controlPointRect() hb_size = hb.size() fp = self.fast_path if fp: fhb = fp.controlPointRect() hb_size = fhb.size() + hb_size + # print(f'hb_size: {hb_size}') # if self._last_step_rect: @@ -255,7 +285,13 @@ class Curve(pg.GraphicsObject): w = hb_size.width() h = hb_size.height() - if not self._last_step_rect: + sbr = self.sub_br + if sbr: + w, h = self.sub_br(w, h) + else: + # assume plain line graphic and use + # default unit step in each direction. + # only on a plane line do we include # and extra index step's worth of width # since in the step case the end of the curve @@ -289,7 +325,7 @@ class Curve(pg.GraphicsObject): def paint( self, - p: QtGui.QPainter, + p: QPainter, opt: QtWidgets.QStyleOptionGraphicsItem, w: QtWidgets.QWidget @@ -301,25 +337,16 @@ class Curve(pg.GraphicsObject): ms_threshold=ms_slower_then, ) - if ( - self._step_mode - and self._last_step_rect - ): - brush = self._brush + sub_paint = self.sub_paint + if sub_paint: + sub_paint(p, profiler) - # p.drawLines(*tuple(filter(bool, self._last_step_lines))) - # p.drawRect(self._last_step_rect) - p.fillRect(self._last_step_rect, brush) - profiler('.fillRect()') - - if self._last_line: - p.setPen(self.last_step_pen) - p.drawLine(self._last_line) - profiler('.drawLine()') - p.setPen(self._pen) + p.setPen(self.last_step_pen) + p.drawLine(self._last_line) + profiler('.drawLine()') + p.setPen(self._pen) path = self.path - # cap = path.capacity() # if cap != self._last_cap: # print(f'NEW CAPACITY: {self._last_cap} -> {cap}') @@ -341,3 +368,116 @@ class Curve(pg.GraphicsObject): # if self._fill: # brush = self.opts['brush'] # p.fillPath(self.path, brush) + + def draw_last_datum( + self, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + array_key: str, + + ) -> None: + # default line draw last call + with self.reset_cache(): + x = render_data['index'] + y = render_data[array_key] + + x_last = x[-1] + y_last = y[-1] + + # draw the "current" step graphic segment so it + # lines up with the "middle" of the current + # (OHLC) sample. + self._last_line = QLineF( + x[-2], y[-2], + x_last, y_last + ) + + +# TODO: this should probably be a "downsampled" curve type +# that draws a bar-style (but for the px column) last graphics +# element such that the current datum in view can be shown +# (via it's max / min) even when highly zoomed out. +class FlattenedOHLC(Curve): + + def draw_last_datum( + self, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + array_key: str, + + ) -> None: + lasts = src_data[-2:] + x = lasts['index'] + y = lasts['close'] + + # draw the "current" step graphic segment so it + # lines up with the "middle" of the current + # (OHLC) sample. + self._last_line = QLineF( + x[-2], y[-2], + x[-1], y[-1] + ) + + +class StepCurve(Curve): + + def declare_paintables( + self, + ) -> None: + self._last_step_rect = QRectF() + + def draw_last_datum( + self, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + array_key: str, + + w: float = 0.5, + + ) -> None: + + # TODO: remove this and instead place all step curve + # updating into pre-path data render callbacks. + # full input data + x = src_data['index'] + y = src_data[array_key] + + x_last = x[-1] + y_last = y[-1] + + # lol, commenting this makes step curves + # all "black" for me :eyeroll:.. + self._last_line = QLineF( + x_last - w, 0, + x_last + w, 0, + ) + self._last_step_rect = QRectF( + x_last - w, 0, + x_last + w, y_last, + ) + + def sub_paint( + self, + p: QPainter, + profiler: pg.debug.Profiler, + + ) -> None: + # p.drawLines(*tuple(filter(bool, self._last_step_lines))) + # p.drawRect(self._last_step_rect) + p.fillRect(self._last_step_rect, self._brush) + profiler('.fillRect()') + + def sub_br( + self, + path_w: float, + path_h: float, + + ) -> (float, float): + # passthrough + return path_w, path_h diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index c2e6ec09..7960d649 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -34,13 +34,6 @@ import numpy as np from numpy.lib import recfunctions as rfn import pyqtgraph as pg from PyQt5.QtGui import QPainterPath -from PyQt5.QtCore import ( - # Qt, - QLineF, - # QSizeF, - QRectF, - # QPointF, -) from ..data._sharedmem import ( ShmArray, @@ -57,10 +50,12 @@ from ._pathops import ( ) from ._ohlc import ( BarItems, - bar_from_ohlc_row, + # bar_from_ohlc_row, ) from ._curve import ( Curve, + StepCurve, + FlattenedOHLC, ) from ..log import get_logger @@ -175,7 +170,7 @@ def render_baritems( format_xy=ohlc_flat_to_xy, ) - curve = Curve( + curve = FlattenedOHLC( name=f'{flow.name}_ds_ohlc', color=bars._color, ) @@ -244,84 +239,10 @@ def render_baritems( bars.show() bars.update() - draw_last = False - - if should_line: - - def draw_last_flattened_ohlc_line( - graphics: pg.GraphicsObject, - path: QPainterPath, - src_data: np.ndarray, - render_data: np.ndarray, - reset: bool, - - ) -> None: - lasts = src_data[-2:] - x = lasts['index'] - y = lasts['close'] - - # draw the "current" step graphic segment so it - # lines up with the "middle" of the current - # (OHLC) sample. - graphics._last_line = QLineF( - x[-2], y[-2], - x[-1], y[-1] - ) - - draw_last = draw_last_flattened_ohlc_line - - else: - def draw_last_ohlc_bar( - graphics: pg.GraphicsObject, - path: QPainterPath, - src_data: np.ndarray, - render_data: np.ndarray, - reset: bool, - - ) -> None: - last = src_data[-1] - - # generate new lines objects for updatable "current bar" - graphics._last_bar_lines = bar_from_ohlc_row(last) - - # last bar update - i, o, h, l, last, v = last[ - ['index', 'open', 'high', 'low', 'close', 'volume'] - ] - # assert i == graphics.start_index - 1 - # assert i == last_index - body, larm, rarm = graphics._last_bar_lines - - # XXX: is there a faster way to modify this? - rarm.setLine(rarm.x1(), last, rarm.x2(), last) - - # writer is responsible for changing open on "first" volume of bar - larm.setLine(larm.x1(), o, larm.x2(), o) - - if l != h: # noqa - - if body is None: - body = graphics._last_bar_lines[0] = QLineF(i, l, i, h) - else: - # update body - body.setLine(i, l, i, h) - - # XXX: pretty sure this is causing an issue where the - # bar has a large upward move right before the next - # sample and the body is getting set to None since the - # next bar is flat but the shm array index update wasn't - # read by the time this code runs. Iow we're doing this - # removal of the body for a bar index that is now out of - # date / from some previous sample. It's weird though - # because i've seen it do this to bars i - 3 back? - - draw_last = draw_last_ohlc_bar - return ( graphics, r, {'read_from_key': False}, - draw_last, should_line, changed_to_line, ) @@ -411,10 +332,10 @@ class Flow(msgspec.Struct): # , frozen=True): ''' name: str plot: pg.PlotItem - graphics: pg.GraphicsObject + graphics: Curve _shm: ShmArray - draw_last_datum: Optional[ + draw_last: Optional[ Callable[ [np.ndarray, str], tuple[np.ndarray] @@ -597,12 +518,9 @@ class Flow(msgspec.Struct): # , frozen=True): render to graphics. ''' - - # profiler = profiler or pg.debug.Profiler( profiler = pg.debug.Profiler( msg=f'Flow.update_graphics() for {self.name}', disabled=not pg_profile_enabled(), - # disabled=False, ms_threshold=4, # ms_threshold=ms_slower_then, ) @@ -623,13 +541,9 @@ class Flow(msgspec.Struct): # , frozen=True): # print('exiting early') return graphics - draw_last: bool = True slice_to_head: int = -1 - should_redraw: bool = False - rkwargs = {} - bars = False if isinstance(graphics, BarItems): # XXX: special case where we change out graphics @@ -638,7 +552,6 @@ class Flow(msgspec.Struct): # , frozen=True): graphics, r, rkwargs, - draw_last, should_line, changed_to_line, ) = render_baritems( @@ -648,7 +561,7 @@ class Flow(msgspec.Struct): # , frozen=True): profiler, **kwargs, ) - bars = True + # bars = True should_redraw = changed_to_line or not should_line else: @@ -661,7 +574,7 @@ class Flow(msgspec.Struct): # , frozen=True): last_read=read, ) - # ``Curve`` case: + # ``Curve`` derivative case(s): array_key = array_key or self.name # print(array_key) @@ -670,20 +583,19 @@ class Flow(msgspec.Struct): # , frozen=True): should_ds: bool = r._in_ds showing_src_data: bool = not r._in_ds - step_mode = getattr(graphics, '_step_mode', False) + # step_mode = getattr(graphics, '_step_mode', False) + step_mode = isinstance(graphics, StepCurve) if step_mode: r.allocate_xy = to_step_format r.update_xy = update_step_xy r.format_xy = step_to_xy - slice_to_head = -2 - # TODO: append logic inside ``.render()`` isn't - # corrent yet for step curves.. remove this to see it. + # correct yet for step curves.. remove this to see it. should_redraw = True - - draw_last = True + # draw_last = True + slice_to_head = -2 # downsampling incremental state checking # check for and set std m4 downsample conditions @@ -760,77 +672,23 @@ class Flow(msgspec.Struct): # , frozen=True): graphics.path = r.path graphics.fast_path = r.fast_path - if draw_last and not bars: + graphics.draw_last_datum( + path, + src_array, + data, + reset, + array_key, + ) - if not step_mode: + # TODO: is this ever better? + # graphics.prepareGeometryChange() + # profiler('.prepareGeometryChange()') - def draw_last_line( - graphics: pg.GraphicsObject, - path: QPainterPath, - src_data: np.ndarray, - render_data: np.ndarray, - reset: bool, - - ) -> None: - # default line draw last call - with graphics.reset_cache(): - x = render_data['index'] - y = render_data[array_key] - x_last = x[-1] - y_last = y[-1] - - # draw the "current" step graphic segment so it - # lines up with the "middle" of the current - # (OHLC) sample. - graphics._last_line = QLineF( - x[-2], y[-2], - x_last, y_last - ) - - draw_last_line(graphics, path, src_array, data, reset) - - else: - - def draw_last_step( - graphics: pg.GraphicsObject, - path: QPainterPath, - src_data: np.ndarray, - render_data: np.ndarray, - reset: bool, - - ) -> None: - w = 0.5 - # TODO: remove this and instead place all step curve - # updating into pre-path data render callbacks. - # full input data - x = src_array['index'] - y = src_array[array_key] - x_last = x[-1] - y_last = y[-1] - - # lol, commenting this makes step curves - # all "black" for me :eyeroll:.. - graphics._last_line = QLineF( - x_last - w, 0, - x_last + w, 0, - ) - graphics._last_step_rect = QRectF( - x_last - w, 0, - x_last + w, y_last, - ) - - draw_last_step(graphics, path, src_array, data, reset) - - # TODO: does this actuallly help us in any way (prolly should - # look at the source / ask ogi). I think it avoid artifacts on - # wheel-scroll downsampling curve updates? - graphics.update() - profiler('.prepareGeometryChange()') - - elif bars and draw_last: - draw_last(graphics, path, src_array, data, reset) - graphics.update() - profiler('.update()') + # TODO: does this actuallly help us in any way (prolly should + # look at the source / ask ogi). I think it avoid artifacts on + # wheel-scroll downsampling curve updates? + graphics.update() + profiler('.update()') return graphics diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index d4a93065..0f7ce6f7 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -27,6 +27,7 @@ import numpy as np import pyqtgraph as pg from PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtCore import QLineF, QPointF +from PyQt5.QtGui import QPainterPath from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor @@ -85,8 +86,6 @@ class BarItems(pg.GraphicsObject): "Price range" bars graphics rendered from a OHLC sampled sequence. ''' - sigPlotChanged = QtCore.pyqtSignal(object) - def __init__( self, linked: LinkedSplits, @@ -107,7 +106,7 @@ class BarItems(pg.GraphicsObject): self._name = name self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - self.path = QtGui.QPainterPath() + self.path = QPainterPath() self._last_bar_lines: Optional[tuple[QLineF, ...]] = None def x_uppx(self) -> int: @@ -192,3 +191,48 @@ class BarItems(pg.GraphicsObject): p.setPen(self.bars_pen) p.drawPath(self.path) profiler(f'draw history path: {self.path.capacity()}') + + def draw_last_datum( + self, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + array_key: str, + + ) -> None: + last = src_data[-1] + + # generate new lines objects for updatable "current bar" + self._last_bar_lines = bar_from_ohlc_row(last) + + # last bar update + i, o, h, l, last, v = last[ + ['index', 'open', 'high', 'low', 'close', 'volume'] + ] + # assert i == graphics.start_index - 1 + # assert i == last_index + body, larm, rarm = self._last_bar_lines + + # XXX: is there a faster way to modify this? + rarm.setLine(rarm.x1(), last, rarm.x2(), last) + + # writer is responsible for changing open on "first" volume of bar + larm.setLine(larm.x1(), o, larm.x2(), o) + + if l != h: # noqa + + if body is None: + body = self._last_bar_lines[0] = QLineF(i, l, i, h) + else: + # update body + body.setLine(i, l, i, h) + + # XXX: pretty sure this is causing an issue where the + # bar has a large upward move right before the next + # sample and the body is getting set to None since the + # next bar is flat but the shm array index update wasn't + # read by the time this code runs. Iow we're doing this + # removal of the body for a bar index that is now out of + # date / from some previous sample. It's weird though + # because i've seen it do this to bars i - 3 back? From e5f96391e30a20835bcbd23402cf5fdd2833c46a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Jun 2022 16:44:04 -0400 Subject: [PATCH 111/113] Return xy data from `Curve.draw_last_datum()` methods --- piker/ui/_curve.py | 9 +++++---- piker/ui/_ohlc.py | 24 ++++++++++++++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 8feb24b9..ac967bf7 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -383,17 +383,16 @@ class Curve(pg.GraphicsObject): x = render_data['index'] y = render_data[array_key] - x_last = x[-1] - y_last = y[-1] - # draw the "current" step graphic segment so it # lines up with the "middle" of the current # (OHLC) sample. self._last_line = QLineF( x[-2], y[-2], - x_last, y_last + x[-1], y[-1], ) + return x, y + # TODO: this should probably be a "downsampled" curve type # that draws a bar-style (but for the px column) last graphics @@ -421,6 +420,7 @@ class FlattenedOHLC(Curve): x[-2], y[-2], x[-1], y[-1] ) + return x, y class StepCurve(Curve): @@ -461,6 +461,7 @@ class StepCurve(Curve): x_last - w, 0, x_last + w, y_last, ) + return x, y def sub_paint( self, diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 0f7ce6f7..dbe4c18e 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -200,16 +200,26 @@ class BarItems(pg.GraphicsObject): reset: bool, array_key: str, + fields: list[str] = [ + 'index', + 'open', + 'high', + 'low', + 'close', + ], + ) -> None: - last = src_data[-1] + + # relevant fields + ohlc = src_data[fields] + last_row = ohlc[-1:] + + # individual values + last_row = i, o, h, l, last = ohlc[-1] # generate new lines objects for updatable "current bar" - self._last_bar_lines = bar_from_ohlc_row(last) + self._last_bar_lines = bar_from_ohlc_row(last_row) - # last bar update - i, o, h, l, last, v = last[ - ['index', 'open', 'high', 'low', 'close', 'volume'] - ] # assert i == graphics.start_index - 1 # assert i == last_index body, larm, rarm = self._last_bar_lines @@ -236,3 +246,5 @@ class BarItems(pg.GraphicsObject): # removal of the body for a bar index that is now out of # date / from some previous sample. It's weird though # because i've seen it do this to bars i - 3 back? + + return ohlc['index'], ohlc['close'] From 99965e7601a2e5a7522e7a7086128ed29d1c122f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Jun 2022 16:45:53 -0400 Subject: [PATCH 112/113] Only draw mx/mn line for last uppx's worth of datums When using m4, we downsample to the max and min of each pixel-column's-worth of data thus preserving range / dispersion details whilst not drawing more graphics then can be displayed by the available amount of horizontal pixels. Take and apply this exact same concept to the "last datum" graphics elements for any `Flow` that is reported as being in a downsampled state: - take the xy output from the `Curve.draw_last_datum()`, - slice out all data that fits in the last pixel's worth of x-range by using the uppx, - compute the highest and lowest value from that data, - draw a singe line segment which spans this yrange thus creating a simple vertical set of pixels which are "filled in" and show the entire y-range for the most recent data "contained with that pixel". --- piker/ui/_display.py | 13 ++++++++++++ piker/ui/_flows.py | 47 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index e33c2c74..415827fb 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -654,6 +654,19 @@ def graphics_update_cycle( # run synchronous update on all linked flows for curve_name, flow in chart._flows.items(): + + if ( + not (do_rt_update or do_append) + and liv + # even if we're downsampled bigly + # draw the last datum in the final + # px column to give the user the mx/mn + # range of that set. + ): + # always update the last datum-element + # graphic for all flows + flow.draw_last(array_key=curve_name) + # TODO: should the "main" (aka source) flow be special? if curve_name == chart.data_key: continue diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 7960d649..01bbbece 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -34,6 +34,7 @@ import numpy as np from numpy.lib import recfunctions as rfn import pyqtgraph as pg from PyQt5.QtGui import QPainterPath +from PyQt5.QtCore import QLineF from ..data._sharedmem import ( ShmArray, @@ -335,12 +336,6 @@ class Flow(msgspec.Struct): # , frozen=True): graphics: Curve _shm: ShmArray - draw_last: Optional[ - Callable[ - [np.ndarray, str], - tuple[np.ndarray] - ] - ] = None is_ohlc: bool = False render: bool = True # toggle for display loop @@ -594,7 +589,6 @@ class Flow(msgspec.Struct): # , frozen=True): # TODO: append logic inside ``.render()`` isn't # correct yet for step curves.. remove this to see it. should_redraw = True - # draw_last = True slice_to_head = -2 # downsampling incremental state checking @@ -690,8 +684,47 @@ class Flow(msgspec.Struct): # , frozen=True): graphics.update() profiler('.update()') + # track downsampled state + self._in_ds = r._in_ds + return graphics + def draw_last( + self, + array_key: Optional[str] = None, + + ) -> None: + + # shm read and slice to view + ( + xfirst, xlast, src_array, + ivl, ivr, in_view, + ) = self.read() + + g = self.graphics + array_key = array_key or self.name + x, y = g.draw_last_datum( + g.path, + src_array, + src_array, + False, # never reset path + array_key, + ) + + if self._in_ds: + # we only care about the last pixel's + # worth of data since that's all the screen + # can represent on the last column where + # the most recent datum is being drawn. + uppx = self._last_uppx + y = y[-uppx:] + ymn, ymx = y.min(), y.max() + # print(f'drawing uppx={uppx} mxmn line: {ymn}, {ymx}') + g._last_line = QLineF( + x[-2], ymn, + x[-1], ymx, + ) + def by_index_and_key( renderer: Renderer, From 44c242a7943cd818722710a9a9bc53ab56871510 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 5 Jun 2022 22:01:37 -0400 Subject: [PATCH 113/113] Fill in label with pairs from `status` value of backend init msg --- piker/data/feed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index 94c2f81d..1165fddc 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -47,7 +47,6 @@ import pendulum import numpy as np from ..brokers import get_brokermod -from .._cacheables import maybe_open_context from ..calc import humanize from ..log import get_logger, get_console_log from .._daemon import (