pikers
/
piker
3
0
Fork 0
Code Issues 5 Pull Requests 7 Packages Projects Releases Wiki Activity

Factor `.update_from_array()` into `Flow.update_graphics()` 

A bit hacky to get all graphics types working but this is hopefully the
first step toward moving all the generic update logic into `Renderer`
types which can be themselves managed more compactly and cached per
uppx-m4 level.
incremental_update_paths
Tyler Goodlet 2022-05-18 08:24:12 -04:00
parent e258654c86
commit 4c7661fc23
2 changed files with 883 additions and 464 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

606
piker/ui/_curve.py
View File

@ -172,332 +172,332 @@ class FastAppendCurve(pg.GraphicsObject):
QLineF(lbar, 0, rbar, 0) QLineF(lbar, 0, rbar, 0)
).length() ).length()
def update_from_array( # def update_from_array(
self, # self,
# full array input history # # full array input history
x: np.ndarray, # x: np.ndarray,
y: np.ndarray, # y: np.ndarray,
# pre-sliced array data that's "in view" # # pre-sliced array data that's "in view"
x_iv: np.ndarray, # x_iv: np.ndarray,
y_iv: np.ndarray, # y_iv: np.ndarray,
view_range: Optional[tuple[int, int]] = None, # view_range: Optional[tuple[int, int]] = None,
profiler: Optional[pg.debug.Profiler] = None, # profiler: Optional[pg.debug.Profiler] = None,
draw_last: bool = True, # draw_last: bool = True,
slice_to_head: int = -1, # slice_to_head: int = -1,
do_append: bool = True, # do_append: bool = True,
should_redraw: bool = False, # should_redraw: bool = False,
) -> QtGui.QPainterPath: # ) -> QtGui.QPainterPath:
''' # '''
Update curve from input 2-d data. # Update curve from input 2-d data.
Compare with a cached "x-range" state and (pre/a)ppend based on # Compare with a cached "x-range" state and (pre/a)ppend based on
a length diff. # a length diff.
''' # '''
profiler = profiler or pg.debug.Profiler( # profiler = profiler or pg.debug.Profiler(
msg=f'FastAppendCurve.update_from_array(): `{self._name}`', # msg=f'FastAppendCurve.update_from_array(): `{self._name}`',
disabled=not pg_profile_enabled(), # disabled=not pg_profile_enabled(),
ms_threshold=ms_slower_then, # ms_threshold=ms_slower_then,
)
# flip_cache = False
if self._xrange:
istart, istop = self._xrange
else:
self._xrange = istart, istop = x[0], x[-1]
# compute the length diffs between the first/last index entry in
# the input data and the last indexes we have on record from the
# last time we updated the curve index.
prepend_length = int(istart - x[0])
append_length = int(x[-1] - istop)
# this is the diff-mode, "data"-rendered index
# tracking var..
self._xrange = x[0], x[-1]
# print(f"xrange: {self._xrange}")
# XXX: lol brutal, the internals of `CurvePoint` (inherited by
# our `LineDot`) required ``.getData()`` to work..
self.xData = x
self.yData = y
# downsampling incremental state checking
uppx = self.x_uppx()
px_width = self.px_width()
uppx_diff = (uppx - self._last_uppx)
new_sample_rate = False
should_ds = self._in_ds
showing_src_data = self._in_ds
# should_redraw = False
# by default we only pull data up to the last (current) index
x_out = x[:slice_to_head]
y_out = y[:slice_to_head]
# if a view range is passed, plan to draw the
# source ouput that's "in view" of the chart.
if (
view_range
# and not self._in_ds
# and not prepend_length > 0
):
# print(f'{self._name} vr: {view_range}')
# by default we only pull data up to the last (current) index
x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head]
profiler(f'view range slice {view_range}')
vl, vr = view_range
# last_ivr = self._x_iv_range
# ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1])
zoom_or_append = False
last_vr = self._vr
last_ivr = self._avr
if last_vr:
# relative slice indices
lvl, lvr = last_vr
# abs slice indices
al, ar = last_ivr
# append_length = int(x[-1] - istop)
# append_length = int(x_iv[-1] - ar)
# left_change = abs(x_iv[0] - al) >= 1
# right_change = abs(x_iv[-1] - ar) >= 1
if (
# likely a zoom view change
(vr - lvr) > 2 or vl < lvl
# append / prepend update
# we had an append update where the view range
# didn't change but the data-viewed (shifted)
# underneath, so we need to redraw.
# or left_change and right_change and last_vr == view_range
# not (left_change and right_change) and ivr
# (
# or abs(x_iv[ivr] - livr) > 1
):
zoom_or_append = True
# if last_ivr:
# liivl, liivr = last_ivr
if (
view_range != last_vr
and (
append_length > 1
or zoom_or_append
)
):
should_redraw = True
# print("REDRAWING BRUH")
self._vr = view_range
self._avr = x_iv[0], x_iv[slice_to_head]
# x_last = x_iv[-1]
# y_last = y_iv[-1]
# self._last_vr = view_range
# self.disable_cache()
# flip_cache = True
if prepend_length > 0:
should_redraw = True
# check for downsampling conditions
if (
# std m4 downsample conditions
abs(uppx_diff) >= 1
):
log.info(
f'{self._name} sampler change: {self._last_uppx} -> {uppx}'
)
self._last_uppx = uppx
new_sample_rate = True
showing_src_data = False
should_redraw = True
should_ds = True
elif (
uppx <= 2
and self._in_ds
):
# we should de-downsample back to our original
# source data so we clear our path data in prep
# to generate a new one from original source data.
should_redraw = True
new_sample_rate = True
should_ds = False
showing_src_data = True
# no_path_yet = self.path is None
if (
self.path is None
or should_redraw
or new_sample_rate
or prepend_length > 0
):
if should_redraw:
if self.path:
self.path.clear()
profiler('cleared paths due to `should_redraw=True`')
if self.fast_path:
self.fast_path.clear()
profiler('cleared paths due to `should_redraw` set')
if new_sample_rate and showing_src_data:
# if self._in_ds:
log.info(f'DEDOWN -> {self._name}')
self._in_ds = False
elif should_ds and uppx > 1:
x_out, y_out = xy_downsample(
x_out,
y_out,
uppx,
)
profiler(f'FULL PATH downsample redraw={should_ds}')
self._in_ds = True
self.path = pg.functions.arrayToQPath(
x_out,
y_out,
connect='all',
finiteCheck=False,
path=self.path,
)
self.prepareGeometryChange()
profiler(
'generated fresh path. '
f'(should_redraw: {should_redraw} '
f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})'
)
# profiler(f'DRAW PATH IN VIEW -> {self._name}')
# reserve mem allocs see:
# - https://doc.qt.io/qt-5/qpainterpath.html#reserve
# - https://doc.qt.io/qt-5/qpainterpath.html#capacity
# - https://doc.qt.io/qt-5/qpainterpath.html#clear
# XXX: right now this is based on had hoc checks on a
# hidpi 3840x2160 4k monitor but we should optimize for
# the target display(s) on the sys.
# if no_path_yet:
# self.path.reserve(int(500e3))
# TODO: get this piecewise prepend working - right now it's
# giving heck on vwap...
# elif prepend_length:
# breakpoint()
# prepend_path = pg.functions.arrayToQPath(
# x[0:prepend_length],
# y[0:prepend_length],
# connect='all'
# ) # )
# # flip_cache = False
# # swap prepend path in "front" # if self._xrange:
# old_path = self.path # istart, istop = self._xrange
# self.path = prepend_path # else:
# # self.path.moveTo(new_x[0], new_y[0]) # self._xrange = istart, istop = x[0], x[-1]
# self.path.connectPath(old_path)
elif ( # # compute the length diffs between the first/last index entry in
append_length > 0 # # the input data and the last indexes we have on record from the
and do_append # # last time we updated the curve index.
and not should_redraw # prepend_length = int(istart - x[0])
# and not view_range # append_length = int(x[-1] - istop)
):
print(f'{self._name} append len: {append_length}')
new_x = x[-append_length - 2:slice_to_head]
new_y = y[-append_length - 2:slice_to_head]
profiler('sliced append path')
profiler( # # this is the diff-mode, "data"-rendered index
f'diffed array input, append_length={append_length}' # # tracking var..
) # self._xrange = x[0], x[-1]
# if should_ds: # # print(f"xrange: {self._xrange}")
# new_x, new_y = xy_downsample(
# new_x, # # XXX: lol brutal, the internals of `CurvePoint` (inherited by
# new_y, # # our `LineDot`) required ``.getData()`` to work..
# self.xData = x
# self.yData = y
# # downsampling incremental state checking
# uppx = self.x_uppx()
# px_width = self.px_width()
# uppx_diff = (uppx - self._last_uppx)
# new_sample_rate = False
# should_ds = self._in_ds
# showing_src_data = self._in_ds
# # should_redraw = False
# # by default we only pull data up to the last (current) index
# x_out = x[:slice_to_head]
# y_out = y[:slice_to_head]
# # if a view range is passed, plan to draw the
# # source ouput that's "in view" of the chart.
# if (
# view_range
# # and not self._in_ds
# # and not prepend_length > 0
# ):
# # print(f'{self._name} vr: {view_range}')
# # by default we only pull data up to the last (current) index
# x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head]
# profiler(f'view range slice {view_range}')
# vl, vr = view_range
# # last_ivr = self._x_iv_range
# # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1])
# zoom_or_append = False
# last_vr = self._vr
# last_ivr = self._avr
# if last_vr:
# # relative slice indices
# lvl, lvr = last_vr
# # abs slice indices
# al, ar = last_ivr
# # append_length = int(x[-1] - istop)
# # append_length = int(x_iv[-1] - ar)
# # left_change = abs(x_iv[0] - al) >= 1
# # right_change = abs(x_iv[-1] - ar) >= 1
# if (
# # likely a zoom view change
# (vr - lvr) > 2 or vl < lvl
# # append / prepend update
# # we had an append update where the view range
# # didn't change but the data-viewed (shifted)
# # underneath, so we need to redraw.
# # or left_change and right_change and last_vr == view_range
# # not (left_change and right_change) and ivr
# # (
# # or abs(x_iv[ivr] - livr) > 1
# ):
# zoom_or_append = True
# # if last_ivr:
# # liivl, liivr = last_ivr
# if (
# view_range != last_vr
# and (
# append_length > 1
# or zoom_or_append
# )
# ):
# should_redraw = True
# # print("REDRAWING BRUH")
# self._vr = view_range
# self._avr = x_iv[0], x_iv[slice_to_head]
# # x_last = x_iv[-1]
# # y_last = y_iv[-1]
# # self._last_vr = view_range
# # self.disable_cache()
# # flip_cache = True
# if prepend_length > 0:
# should_redraw = True
# # check for downsampling conditions
# if (
# # std m4 downsample conditions
# abs(uppx_diff) >= 1
# ):
# log.info(
# f'{self._name} sampler change: {self._last_uppx} -> {uppx}'
# )
# self._last_uppx = uppx
# new_sample_rate = True
# showing_src_data = False
# should_redraw = True
# should_ds = True
# elif (
# uppx <= 2
# and self._in_ds
# ):
# # we should de-downsample back to our original
# # source data so we clear our path data in prep
# # to generate a new one from original source data.
# should_redraw = True
# new_sample_rate = True
# should_ds = False
# showing_src_data = True
# # no_path_yet = self.path is None
# if (
# self.path is None
# or should_redraw
# or new_sample_rate
# or prepend_length > 0
# ):
# if should_redraw:
# if self.path:
# self.path.clear()
# profiler('cleared paths due to `should_redraw=True`')
# if self.fast_path:
# self.fast_path.clear()
# profiler('cleared paths due to `should_redraw` set')
# if new_sample_rate and showing_src_data:
# # if self._in_ds:
# log.info(f'DEDOWN -> {self._name}')
# self._in_ds = False
# elif should_ds and uppx > 1:
# x_out, y_out = xy_downsample(
# x_out,
# y_out,
# uppx, # uppx,
# ) # )
# profiler(f'fast path downsample redraw={should_ds}') # profiler(f'FULL PATH downsample redraw={should_ds}')
# self._in_ds = True
append_path = pg.functions.arrayToQPath( # self.path = pg.functions.arrayToQPath(
new_x, # x_out,
new_y, # y_out,
connect='all', # connect='all',
finiteCheck=False, # finiteCheck=False,
path=self.fast_path, # path=self.path,
) # )
profiler('generated append qpath') # self.prepareGeometryChange()
# profiler(
# 'generated fresh path. '
# f'(should_redraw: {should_redraw} '
# f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})'
# )
# # profiler(f'DRAW PATH IN VIEW -> {self._name}')
if self.use_fpath: # # reserve mem allocs see:
# an attempt at trying to make append-updates faster.. # # - https://doc.qt.io/qt-5/qpainterpath.html#reserve
if self.fast_path is None: # # - https://doc.qt.io/qt-5/qpainterpath.html#capacity
self.fast_path = append_path # # - https://doc.qt.io/qt-5/qpainterpath.html#clear
# self.fast_path.reserve(int(6e3)) # # XXX: right now this is based on had hoc checks on a
else: # # hidpi 3840x2160 4k monitor but we should optimize for
self.fast_path.connectPath(append_path) # # the target display(s) on the sys.
size = self.fast_path.capacity() # # if no_path_yet:
profiler(f'connected fast path w size: {size}') # # self.path.reserve(int(500e3))
# print(f"append_path br: {append_path.boundingRect()}") # # TODO: get this piecewise prepend working - right now it's
# self.path.moveTo(new_x[0], new_y[0]) # # giving heck on vwap...
# path.connectPath(append_path) # # elif prepend_length:
# # breakpoint()
# XXX: lol this causes a hang.. # # prepend_path = pg.functions.arrayToQPath(
# self.path = self.path.simplified() # # x[0:prepend_length],
else: # # y[0:prepend_length],
size = self.path.capacity() # # connect='all'
profiler(f'connected history path w size: {size}') # # )
self.path.connectPath(append_path)
# other merging ideas: # # # swap prepend path in "front"
# https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths # # old_path = self.path
# path.addPath(append_path) # # self.path = prepend_path
# path.closeSubpath() # # # self.path.moveTo(new_x[0], new_y[0])
# # self.path.connectPath(old_path)
# TODO: try out new work from `pyqtgraph` main which # elif (
# should repair horrid perf: # append_length > 0
# https://github.com/pyqtgraph/pyqtgraph/pull/2032 # and do_append
# ok, nope still horrible XD # and not should_redraw
# if self._fill: # # and not view_range
# # XXX: super slow set "union" op # ):
# self.path = self.path.united(append_path).simplified() # print(f'{self._name} append len: {append_length}')
# new_x = x[-append_length - 2:slice_to_head]
# new_y = y[-append_length - 2:slice_to_head]
# profiler('sliced append path')
# self.disable_cache() # profiler(
# flip_cache = True # f'diffed array input, append_length={append_length}'
# )
# if draw_last: # # if should_ds:
# self.draw_last(x, y) # # new_x, new_y = xy_downsample(
# profiler('draw last segment') # # new_x,
# # new_y,
# # uppx,
# # )
# # profiler(f'fast path downsample redraw={should_ds}')
# if flip_cache: # append_path = pg.functions.arrayToQPath(
# # # XXX: seems to be needed to avoid artifacts (see above). # new_x,
# self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) # new_y,
# connect='all',
# finiteCheck=False,
# path=self.fast_path,
# )
# profiler('generated append qpath')
# trigger redraw of path # if self.use_fpath:
# do update before reverting to cache mode # # an attempt at trying to make append-updates faster..
self.update() # if self.fast_path is None:
profiler('.update()') # self.fast_path = append_path
# # self.fast_path.reserve(int(6e3))
# else:
# self.fast_path.connectPath(append_path)
# size = self.fast_path.capacity()
# profiler(f'connected fast path w size: {size}')
# # print(f"append_path br: {append_path.boundingRect()}")
# # self.path.moveTo(new_x[0], new_y[0])
# # path.connectPath(append_path)
# # XXX: lol this causes a hang..
# # self.path = self.path.simplified()
# else:
# size = self.path.capacity()
# profiler(f'connected history path w size: {size}')
# self.path.connectPath(append_path)
# # other merging ideas:
# # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths
# # path.addPath(append_path)
# # path.closeSubpath()
# # TODO: try out new work from `pyqtgraph` main which
# # should repair horrid perf:
# # https://github.com/pyqtgraph/pyqtgraph/pull/2032
# # ok, nope still horrible XD
# # if self._fill:
# # # XXX: super slow set "union" op
# # self.path = self.path.united(append_path).simplified()
# # self.disable_cache()
# # flip_cache = True
# # if draw_last:
# # self.draw_last(x, y)
# # profiler('draw last segment')
# # if flip_cache:
# # # # XXX: seems to be needed to avoid artifacts (see above).
# # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
# # trigger redraw of path
# # do update before reverting to cache mode
# self.update()
# profiler('.update()')
def draw_last( def draw_last(
self, self,

797
piker/ui/_flows.py
View File

@ -23,7 +23,7 @@ incremental update.
''' '''
from __future__ import annotations from __future__ import annotations
from functools import partial # from functools import partial
from typing import ( from typing import (
Optional, Optional,
Callable, Callable,
@ -54,6 +54,7 @@ from ._pathops import (
gen_ohlc_qpath, gen_ohlc_qpath,
ohlc_to_line, ohlc_to_line,
to_step_format, to_step_format,
xy_downsample,
) )
from ._ohlc import ( from ._ohlc import (
BarItems, BarItems,
@ -152,18 +153,18 @@ def render_baritems(
last_read=read, last_read=read,
) )
ds_curve_r = Renderer( # ds_curve_r = Renderer(
flow=self, # flow=self,
# just swap in the flat view # # just swap in the flat view
# data_t=lambda array: self.gy.array, # # data_t=lambda array: self.gy.array,
last_read=read, # last_read=read,
draw_path=partial( # draw_path=partial(
rowarr_to_path, # rowarr_to_path,
x_basis=None, # x_basis=None,
), # ),
) # )
curve = FastAppendCurve( curve = FastAppendCurve(
name='OHLC', name='OHLC',
color=graphics._color, color=graphics._color,
@ -173,12 +174,14 @@ def render_baritems(
# baseline "line" downsampled OHLC curve that should # baseline "line" downsampled OHLC curve that should
# kick on only when we reach a certain uppx threshold. # kick on only when we reach a certain uppx threshold.
self._render_table[0] = ( self._render_table[0] = curve
ds_curve_r, # (
curve, # # ds_curve_r,
) # curve,
# )
dsc_r, curve = self._render_table[0] curve = self._render_table[0]
# dsc_r, curve = self._render_table[0]
# do checks for whether or not we require downsampling: # do checks for whether or not we require downsampling:
# - if we're **not** downsampling then we simply want to # - if we're **not** downsampling then we simply want to
@ -276,19 +279,20 @@ def render_baritems(
profiler('flattened ustruct in-view OHLC data') profiler('flattened ustruct in-view OHLC data')
# pass into curve graphics processing # pass into curve graphics processing
curve.update_from_array( # curve.update_from_array(
x, # x,
y, # y,
x_iv=x_iv, # x_iv=x_iv,
y_iv=y_iv, # y_iv=y_iv,
view_range=(ivl, ivr), # hack # view_range=(ivl, ivr), # hack
profiler=profiler, # profiler=profiler,
# should_redraw=False, # # should_redraw=False,
# NOTE: already passed through by display loop? # # NOTE: already passed through by display loop?
# do_append=uppx < 16, # # do_append=uppx < 16,
**kwargs, # **kwargs,
) # )
# curve.draw_last(x, y)
curve.show() curve.show()
profiler('updated ds curve') profiler('updated ds curve')
@ -349,6 +353,130 @@ def render_baritems(
# ) # )
# graphics.draw_last(last) # graphics.draw_last(last)
if should_line:
return (
curve,
x,
y,
x_iv,
y_iv,
)
def update_step_data(
flow: Flow,
shm: ShmArray,
ivl: int,
ivr: int,
array_key: str,
iflat_first: int,
iflat: int,
profiler: pg.debug.Profiler,
) -> tuple:
self = flow
(
# iflat_first,
# iflat,
ishm_last,
ishm_first,
) = (
# self._iflat_first,
# self._iflat_last,
shm._last.value,
shm._first.value
)
il = max(iflat - 1, 0)
profiler('read step mode incr update indices')
# check for shm prepend updates since last read.
if iflat_first != ishm_first:
print(f'prepend {array_key}')
# i_prepend = self.shm._array['index'][
# ishm_first:iflat_first]
y_prepend = self.shm._array[array_key][
ishm_first:iflat_first
]
y2_prepend = np.broadcast_to(
y_prepend[:, None], (y_prepend.size, 2),
)
# write newly prepended data to flattened copy
self.gy[ishm_first:iflat_first] = y2_prepend
self._iflat_first = ishm_first
profiler('prepended step mode history')
append_diff = ishm_last - iflat
if append_diff:
# slice up to the last datum since last index/append update
# new_x = self.shm._array[il:ishm_last]['index']
new_y = self.shm._array[il:ishm_last][array_key]
new_y2 = np.broadcast_to(
new_y[:, None], (new_y.size, 2),
)
self.gy[il:ishm_last] = new_y2
profiler('updated step curve data')
# print(
# f'append size: {append_diff}\n'
# f'new_x: {new_x}\n'
# f'new_y: {new_y}\n'
# f'new_y2: {new_y2}\n'
# f'new gy: {gy}\n'
# )
# update local last-index tracking
self._iflat_last = ishm_last
# slice out up-to-last step contents
x_step = self.gx[ishm_first:ishm_last+2]
# shape to 1d
x = x_step.reshape(-1)
profiler('sliced step x')
y_step = self.gy[ishm_first:ishm_last+2]
lasts = self.shm.array[['index', array_key]]
last = lasts[array_key][-1]
y_step[-1] = last
# shape to 1d
y = y_step.reshape(-1)
# s = 6
# print(f'lasts: {x[-2*s:]}, {y[-2*s:]}')
profiler('sliced step y')
# do all the same for only in-view data
ys_iv = y_step[ivl:ivr+1]
xs_iv = x_step[ivl:ivr+1]
y_iv = ys_iv.reshape(ys_iv.size)
x_iv = xs_iv.reshape(xs_iv.size)
# print(
# f'ys_iv : {ys_iv[-s:]}\n'
# f'y_iv: {y_iv[-s:]}\n'
# f'xs_iv: {xs_iv[-s:]}\n'
# f'x_iv: {x_iv[-s:]}\n'
# )
profiler('sliced in view step data')
# legacy full-recompute-everytime method
# x, y = ohlc_flatten(array)
# x_iv, y_iv = ohlc_flatten(in_view)
# profiler('flattened OHLC data')
return (
x,
y,
x_iv,
y_iv,
append_diff,
)
class Flow(msgspec.Struct): # , frozen=True): class Flow(msgspec.Struct): # , frozen=True):
''' '''
@ -368,11 +496,19 @@ class Flow(msgspec.Struct): # , frozen=True):
is_ohlc: bool = False is_ohlc: bool = False
render: bool = True # toggle for display loop render: bool = True # toggle for display loop
# pre-graphics formatted data
gy: Optional[ShmArray] = None gy: Optional[ShmArray] = None
gx: Optional[np.ndarray] = None gx: Optional[np.ndarray] = None
# pre-graphics update indices
_iflat_last: int = 0 _iflat_last: int = 0
_iflat_first: int = 0 _iflat_first: int = 0
# view-range incremental state
_vr: Optional[tuple] = None
_avr: Optional[tuple] = None
# downsampling state
_last_uppx: float = 0 _last_uppx: float = 0
_in_ds: bool = False _in_ds: bool = False
@ -495,7 +631,11 @@ class Flow(msgspec.Struct): # , frozen=True):
start, l, lbar, rbar, r, end, start, l, lbar, rbar, r, end,
) )
def read(self) -> tuple[ def read(
self,
array_field: Optional[str] = None,
) -> tuple[
int, int, np.ndarray, int, int, np.ndarray,
int, int, np.ndarray, int, int, np.ndarray,
]: ]:
@ -513,6 +653,9 @@ class Flow(msgspec.Struct): # , frozen=True):
lbar_i = max(l, ifirst) - ifirst lbar_i = max(l, ifirst) - ifirst
rbar_i = min(r, ilast) - ifirst rbar_i = min(r, ilast) - ifirst
if array_field:
array = array[array_field]
# TODO: we could do it this way as well no? # TODO: we could do it this way as well no?
# to_draw = array[lbar - ifirst:(rbar - ifirst) + 1] # to_draw = array[lbar - ifirst:(rbar - ifirst) + 1]
in_view = array[lbar_i: rbar_i + 1] in_view = array[lbar_i: rbar_i + 1]
@ -532,6 +675,7 @@ class Flow(msgspec.Struct): # , frozen=True):
array_key: Optional[str] = None, array_key: Optional[str] = None,
profiler: Optional[pg.debug.Profiler] = None, profiler: Optional[pg.debug.Profiler] = None,
do_append: bool = True,
**kwargs, **kwargs,
@ -557,15 +701,20 @@ class Flow(msgspec.Struct): # , frozen=True):
) = self.read() ) = self.read()
profiler('read src shm data') profiler('read src shm data')
graphics = self.graphics
if ( if (
not in_view.size not in_view.size
or not render or not render
): ):
return self.graphics return graphics
graphics = self.graphics out: Optional[tuple] = None
if isinstance(graphics, BarItems): if isinstance(graphics, BarItems):
render_baritems( # XXX: special case where we change out graphics
# to a line after a certain uppx threshold.
# render_baritems(
out = render_baritems(
self, self,
graphics, graphics,
read, read,
@ -573,14 +722,74 @@ class Flow(msgspec.Struct): # , frozen=True):
**kwargs, **kwargs,
) )
else: if out is None:
return graphics
# return graphics
r = self._src_r
if not r:
# just using for ``.diff()`` atm..
r = self._src_r = Renderer(
flow=self,
# TODO: rename this to something with ohlc
# draw_path=gen_ohlc_qpath,
last_read=read,
)
# ``FastAppendCurve`` case: # ``FastAppendCurve`` case:
array_key = array_key or self.name array_key = array_key or self.name
uppx = graphics.x_uppx()
profiler(f'read uppx {uppx}')
if graphics._step_mode and self.gy is None: new_sample_rate = False
should_ds = self._in_ds
showing_src_data = self._in_ds
# draw_last: bool = True
slice_to_head: int = -1
should_redraw: bool = False
shm = self.shm shm = self.shm
# if a view range is passed, plan to draw the
# source ouput that's "in view" of the chart.
view_range = (ivl, ivr) if use_vr else None
if out is not None:
# hack to handle ds curve from bars above
(
graphics, # curve
x,
y,
x_iv,
y_iv,
) = out
else:
# full input data
x = array['index']
y = array[array_key]
# inview data
x_iv = in_view['index']
y_iv = in_view[array_key]
# downsampling incremental state checking
uppx = graphics.x_uppx()
# px_width = graphics.px_width()
uppx_diff = (uppx - self._last_uppx)
profiler(f'diffed uppx {uppx}')
x_last = x[-1]
y_last = y[-1]
slice_to_head = -1
profiler('sliced input arrays')
if graphics._step_mode:
slice_to_head = -2
if self.gy is None:
( (
self._iflat_first, self._iflat_first,
self.gx, self.gx,
@ -591,104 +800,24 @@ class Flow(msgspec.Struct): # , frozen=True):
) )
profiler('generated step mode data') profiler('generated step mode data')
if graphics._step_mode:
( (
iflat_first, x,
iflat, y,
ishm_last, x_iv,
ishm_first, y_iv,
) = ( append_diff,
) = update_step_data(
self,
shm,
ivl,
ivr,
array_key,
self._iflat_first, self._iflat_first,
self._iflat_last, self._iflat_last,
self.shm._last.value, profiler,
self.shm._first.value
) )
il = max(iflat - 1, 0)
profiler('read step mode incr update indices')
# check for shm prepend updates since last read.
if iflat_first != ishm_first:
print(f'prepend {array_key}')
# i_prepend = self.shm._array['index'][
# ishm_first:iflat_first]
y_prepend = self.shm._array[array_key][
ishm_first:iflat_first
]
y2_prepend = np.broadcast_to(
y_prepend[:, None], (y_prepend.size, 2),
)
# write newly prepended data to flattened copy
self.gy[ishm_first:iflat_first] = y2_prepend
self._iflat_first = ishm_first
profiler('prepended step mode history')
append_diff = ishm_last - iflat
if append_diff:
# slice up to the last datum since last index/append update
# new_x = self.shm._array[il:ishm_last]['index']
new_y = self.shm._array[il:ishm_last][array_key]
new_y2 = np.broadcast_to(
new_y[:, None], (new_y.size, 2),
)
self.gy[il:ishm_last] = new_y2
profiler('updated step curve data')
# print(
# f'append size: {append_diff}\n'
# f'new_x: {new_x}\n'
# f'new_y: {new_y}\n'
# f'new_y2: {new_y2}\n'
# f'new gy: {gy}\n'
# )
# update local last-index tracking
self._iflat_last = ishm_last
# slice out up-to-last step contents
x_step = self.gx[ishm_first:ishm_last+2]
# shape to 1d
x = x_step.reshape(-1)
profiler('sliced step x')
y_step = self.gy[ishm_first:ishm_last+2]
lasts = self.shm.array[['index', array_key]]
last = lasts[array_key][-1]
y_step[-1] = last
# shape to 1d
y = y_step.reshape(-1)
# s = 6
# print(f'lasts: {x[-2*s:]}, {y[-2*s:]}')
profiler('sliced step y')
# do all the same for only in-view data
ys_iv = y_step[ivl:ivr+1]
xs_iv = x_step[ivl:ivr+1]
y_iv = ys_iv.reshape(ys_iv.size)
x_iv = xs_iv.reshape(xs_iv.size)
# print(
# f'ys_iv : {ys_iv[-s:]}\n'
# f'y_iv: {y_iv[-s:]}\n'
# f'xs_iv: {xs_iv[-s:]}\n'
# f'x_iv: {x_iv[-s:]}\n'
# )
profiler('sliced in view step data')
# legacy full-recompute-everytime method
# x, y = ohlc_flatten(array)
# x_iv, y_iv = ohlc_flatten(in_view)
# profiler('flattened OHLC data')
x_last = array['index'][-1]
y_last = array[array_key][-1]
graphics._last_line = QLineF( graphics._last_line = QLineF(
x_last - 0.5, 0, x_last - 0.5, 0,
x_last + 0.5, 0, x_last + 0.5, 0,
@ -697,29 +826,9 @@ class Flow(msgspec.Struct): # , frozen=True):
x_last - 0.5, 0, x_last - 0.5, 0,
x_last + 0.5, y_last, x_last + 0.5, y_last,
) )
# graphics.update()
graphics.update_from_array( should_redraw = bool(append_diff)
x=x,
y=y,
x_iv=x_iv,
y_iv=y_iv,
view_range=(ivl, ivr) if use_vr else None,
draw_last=False,
slice_to_head=-2,
should_redraw=bool(append_diff),
# NOTE: already passed through by display loop?
# do_append=uppx < 16,
profiler=profiler,
**kwargs
)
profiler('updated step mode curve')
# graphics.reset_cache() # graphics.reset_cache()
# print( # print(
# f"path br: {graphics.path.boundingRect()}\n", # f"path br: {graphics.path.boundingRect()}\n",
@ -728,40 +837,287 @@ class Flow(msgspec.Struct): # , frozen=True):
# f"full br: {graphics._br}\n", # f"full br: {graphics._br}\n",
# ) # )
else: # compute the length diffs between the first/last index entry in
x = array['index'] # the input data and the last indexes we have on record from the
y = array[array_key] # last time we updated the curve index.
x_iv = in_view['index'] prepend_length, append_length = r.diff(read)
y_iv = in_view[array_key] # print((prepend_length, append_length))
profiler('sliced input arrays')
# graphics.draw_last(x, y) # old_prepend_length = int(istart - x[0])
# old_append_length = int(x[-1] - istop)
graphics.update_from_array( # MAIN RENDER LOGIC:
x=x, # - determine in view data and redraw on range change
y=y, # - determine downsampling ops if needed
# - (incrementally) update ``QPainterPath``
x_iv=x_iv, if (
y_iv=y_iv, view_range
# and not self._in_ds
# and not prepend_length > 0
):
# print(f'{self._name} vr: {view_range}')
view_range=(ivl, ivr) if use_vr else None, # by default we only pull data up to the last (current) index
x_out = x_iv[:slice_to_head]
y_out = y_iv[:slice_to_head]
profiler(f'view range slice {view_range}')
# NOTE: already passed through by display loop? vl, vr = view_range
# do_append=uppx < 16,
profiler=profiler, zoom_or_append = False
**kwargs last_vr = self._vr
last_ivr = self._avr
# incremental in-view data update.
if last_vr:
# relative slice indices
lvl, lvr = last_vr
# abs slice indices
al, ar = last_ivr
# append_length = int(x[-1] - istop)
# append_length = int(x_iv[-1] - ar)
# left_change = abs(x_iv[0] - al) >= 1
# right_change = abs(x_iv[-1] - ar) >= 1
if (
# likely a zoom view change
(vr - lvr) > 2 or vl < lvl
# append / prepend update
# we had an append update where the view range
# didn't change but the data-viewed (shifted)
# underneath, so we need to redraw.
# or left_change and right_change and last_vr == view_range
# not (left_change and right_change) and ivr
# (
# or abs(x_iv[ivr] - livr) > 1
):
zoom_or_append = True
# if last_ivr:
# liivl, liivr = last_ivr
if (
view_range != last_vr
and (
append_length > 1
or zoom_or_append
) )
profiler('`graphics.update_from_array()` complete') ):
should_redraw = True
# print("REDRAWING BRUH")
self._vr = view_range
self._avr = x_iv[0], x_iv[slice_to_head]
if prepend_length > 0:
should_redraw = True
# check for downsampling conditions
if (
# std m4 downsample conditions
# px_width
# and abs(uppx_diff) >= 1
abs(uppx_diff) >= 1
):
log.info(
f'{array_key} sampler change: {self._last_uppx} -> {uppx}'
)
self._last_uppx = uppx
new_sample_rate = True
showing_src_data = False
should_redraw = True
should_ds = True
elif (
uppx <= 2
and self._in_ds
):
# we should de-downsample back to our original
# source data so we clear our path data in prep
# to generate a new one from original source data.
should_redraw = True
new_sample_rate = True
should_ds = False
showing_src_data = True
# no_path_yet = self.path is None
fast_path = graphics.fast_path
if (
graphics.path is None
or should_redraw
or new_sample_rate
or prepend_length > 0
):
if should_redraw:
if graphics.path:
graphics.path.clear()
profiler('cleared paths due to `should_redraw=True`')
if graphics.fast_path:
graphics.fast_path.clear()
profiler('cleared paths due to `should_redraw` set')
if new_sample_rate and showing_src_data:
# if self._in_ds:
log.info(f'DEDOWN -> {self.name}')
self._in_ds = False
# elif should_ds and uppx and px_width > 1:
elif should_ds and uppx > 1:
x_out, y_out = xy_downsample(
x_out,
y_out,
uppx,
# px_width,
)
profiler(f'FULL PATH downsample redraw={should_ds}')
self._in_ds = True
graphics.path = pg.functions.arrayToQPath(
x_out,
y_out,
connect='all',
finiteCheck=False,
path=graphics.path,
)
graphics.prepareGeometryChange()
profiler(
'generated fresh path. '
f'(should_redraw: {should_redraw} '
f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})'
)
# profiler(f'DRAW PATH IN VIEW -> {self.name}')
# reserve mem allocs see:
# - https://doc.qt.io/qt-5/qpainterpath.html#reserve
# - https://doc.qt.io/qt-5/qpainterpath.html#capacity
# - https://doc.qt.io/qt-5/qpainterpath.html#clear
# XXX: right now this is based on had hoc checks on a
# hidpi 3840x2160 4k monitor but we should optimize for
# the target display(s) on the sys.
# if no_path_yet:
# graphics.path.reserve(int(500e3))
# TODO: get this piecewise prepend working - right now it's
# giving heck on vwap...
# elif prepend_length:
# breakpoint()
# prepend_path = pg.functions.arrayToQPath(
# x[0:prepend_length],
# y[0:prepend_length],
# connect='all'
# )
# # swap prepend path in "front"
# old_path = graphics.path
# graphics.path = prepend_path
# # graphics.path.moveTo(new_x[0], new_y[0])
# graphics.path.connectPath(old_path)
elif (
append_length > 0
and do_append
and not should_redraw
# and not view_range
):
print(f'{self.name} append len: {append_length}')
new_x = x[-append_length - 2:slice_to_head]
new_y = y[-append_length - 2:slice_to_head]
profiler('sliced append path')
profiler(
f'diffed array input, append_length={append_length}'
)
# if should_ds:
# new_x, new_y = xy_downsample(
# new_x,
# new_y,
# px_width,
# uppx,
# )
# profiler(f'fast path downsample redraw={should_ds}')
append_path = pg.functions.arrayToQPath(
new_x,
new_y,
connect='all',
finiteCheck=False,
path=graphics.fast_path,
)
profiler('generated append qpath')
if graphics.use_fpath:
print("USING FPATH")
# an attempt at trying to make append-updates faster..
if fast_path is None:
graphics.fast_path = append_path
# self.fast_path.reserve(int(6e3))
else:
fast_path.connectPath(append_path)
size = fast_path.capacity()
profiler(f'connected fast path w size: {size}')
# print(f"append_path br: {append_path.boundingRect()}")
# graphics.path.moveTo(new_x[0], new_y[0])
# path.connectPath(append_path)
# XXX: lol this causes a hang..
# graphics.path = graphics.path.simplified()
else:
size = graphics.path.capacity()
profiler(f'connected history path w size: {size}')
graphics.path.connectPath(append_path)
# graphics.update_from_array(
# x=x,
# y=y,
# x_iv=x_iv,
# y_iv=y_iv,
# view_range=(ivl, ivr) if use_vr else None,
# # NOTE: already passed through by display loop.
# # do_append=uppx < 16,
# do_append=do_append,
# slice_to_head=slice_to_head,
# should_redraw=should_redraw,
# profiler=profiler,
# **kwargs
# )
graphics.draw_last(x, y)
profiler('draw last segment')
graphics.update()
profiler('.update()')
profiler('`graphics.update_from_array()` complete')
return graphics return graphics
class Renderer(msgspec.Struct): class Renderer(msgspec.Struct):
flow: Flow flow: Flow
# last array view read
last_read: Optional[tuple] = None
# called to render path graphics # called to render path graphics
draw_path: Callable[np.ndarray, QPainterPath] draw_path: Optional[Callable[np.ndarray, QPainterPath]] = None
# output graphics rendering, the main object
# processed in ``QGraphicsObject.paint()``
path: Optional[QPainterPath] = None
# called on input data but before any graphics format # called on input data but before any graphics format
# conversions or processing. # conversions or processing.
@ -778,25 +1134,66 @@ class Renderer(msgspec.Struct):
prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None
append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None
# last array view read # incremental update state(s)
last_read: Optional[np.ndarray] = None # _in_ds: bool = False
# _last_uppx: float = 0
_last_vr: Optional[tuple[float, float]] = None
_last_ivr: Optional[tuple[float, float]] = None
# output graphics rendering, the main object def diff(
# processed in ``QGraphicsObject.paint()`` self,
path: Optional[QPainterPath] = None new_read: tuple[np.ndarray],
# def diff( ) -> tuple[np.ndarray]:
# self,
# latest_read: tuple[np.ndarray],
# ) -> tuple[np.ndarray]: (
# # blah blah blah last_xfirst,
# # do diffing for prepend, append and last entry last_xlast,
# return ( last_array,
# to_prepend last_ivl, last_ivr,
# to_append last_in_view,
) = self.last_read
# TODO: can the renderer just call ``Flow.read()`` directly?
# unpack latest source data read
(
xfirst,
xlast,
array,
ivl,
ivr,
in_view,
) = new_read
# compute the length diffs between the first/last index entry in
# the input data and the last indexes we have on record from the
# last time we updated the curve index.
prepend_length = int(last_xfirst - xfirst)
append_length = int(xlast - last_xlast)
# TODO: eventually maybe we can implement some kind of
# transform on the ``QPainterPath`` that will more or less
# detect the diff in "elements" terms?
# update state
self.last_read = new_read
# blah blah blah
# do diffing for prepend, append and last entry
return (
prepend_length,
append_length,
# last, # last,
# ) )
def draw_path(
self,
should_redraw: bool = False,
) -> QPainterPath:
if should_redraw:
if self.path:
self.path.clear()
# profiler('cleared paths due to `should_redraw=True`')
def render( def render(
self, self,
@ -819,11 +1216,30 @@ class Renderer(msgspec.Struct):
- blah blah blah (from notes) - blah blah blah (from notes)
''' '''
# do full source data render to path # get graphics info
# TODO: can the renderer just call ``Flow.read()`` directly?
# unpack latest source data read
( (
xfirst, xlast, array, xfirst,
ivl, ivr, in_view, xlast,
) = self.last_read array,
ivl,
ivr,
in_view,
) = new_read
(
prepend_length,
append_length,
) = self.diff(new_read)
# do full source data render to path
# x = array['index']
# y = array#[array_key]
# x_iv = in_view['index']
# y_iv = in_view#[array_key]
if only_in_view: if only_in_view:
array = in_view array = in_view
@ -832,7 +1248,10 @@ class Renderer(msgspec.Struct):
# xfirst, xlast, array, ivl, ivr, in_view # xfirst, xlast, array, ivl, ivr, in_view
# ) = new_read # ) = new_read
if self.path is None or only_in_view: if (
self.path is None
or only_in_view
):
# redraw the entire source data if we have either of: # redraw the entire source data if we have either of:
# - no prior path graphic rendered or, # - no prior path graphic rendered or,
# - we always intend to re-render the data only in view # - we always intend to re-render the data only in view