pikers
/
piker
3
0
Fork 0
Code Issues 5 Pull Requests 7 Packages Projects Releases Wiki Activity

Factor `.update_from_array()` into `Flow.update_graphics()` 

A bit hacky to get all graphics types working but this is hopefully the
first step toward moving all the generic update logic into `Renderer`
types which can be themselves managed more compactly and cached per
uppx-m4 level.
incremental_update_paths
Tyler Goodlet 2022-05-18 08:24:12 -04:00
parent e258654c86
commit 4c7661fc23
2 changed files with 883 additions and 464 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

538
piker/ui/_curve.py
View File

@ -172,332 +172,332 @@ class FastAppendCurve(pg.GraphicsObject):
QLineF(lbar, 0, rbar, 0) QLineF(lbar, 0, rbar, 0)
).length() ).length()
def update_from_array( # def update_from_array(
self, # self,
# full array input history # # full array input history
x: np.ndarray, # x: np.ndarray,
y: np.ndarray, # y: np.ndarray,
# pre-sliced array data that's "in view" # # pre-sliced array data that's "in view"
x_iv: np.ndarray, # x_iv: np.ndarray,
y_iv: np.ndarray, # y_iv: np.ndarray,
view_range: Optional[tuple[int, int]] = None, # view_range: Optional[tuple[int, int]] = None,
profiler: Optional[pg.debug.Profiler] = None, # profiler: Optional[pg.debug.Profiler] = None,
draw_last: bool = True, # draw_last: bool = True,
slice_to_head: int = -1, # slice_to_head: int = -1,
do_append: bool = True, # do_append: bool = True,
should_redraw: bool = False, # should_redraw: bool = False,
) -> QtGui.QPainterPath: # ) -> QtGui.QPainterPath:
''' # '''
Update curve from input 2-d data. # Update curve from input 2-d data.
Compare with a cached "x-range" state and (pre/a)ppend based on # Compare with a cached "x-range" state and (pre/a)ppend based on
a length diff. # a length diff.
''' # '''
profiler = profiler or pg.debug.Profiler( # profiler = profiler or pg.debug.Profiler(
msg=f'FastAppendCurve.update_from_array(): `{self._name}`', # msg=f'FastAppendCurve.update_from_array(): `{self._name}`',
disabled=not pg_profile_enabled(), # disabled=not pg_profile_enabled(),
ms_threshold=ms_slower_then, # ms_threshold=ms_slower_then,
) # )
# flip_cache = False # # flip_cache = False
if self._xrange: # if self._xrange:
istart, istop = self._xrange # istart, istop = self._xrange
else: # else:
self._xrange = istart, istop = x[0], x[-1] # self._xrange = istart, istop = x[0], x[-1]
# compute the length diffs between the first/last index entry in # # compute the length diffs between the first/last index entry in
# the input data and the last indexes we have on record from the # # the input data and the last indexes we have on record from the
# last time we updated the curve index. # # last time we updated the curve index.
prepend_length = int(istart - x[0]) # prepend_length = int(istart - x[0])
append_length = int(x[-1] - istop) # append_length = int(x[-1] - istop)
# this is the diff-mode, "data"-rendered index # # this is the diff-mode, "data"-rendered index
# tracking var.. # # tracking var..
self._xrange = x[0], x[-1] # self._xrange = x[0], x[-1]
# print(f"xrange: {self._xrange}") # # print(f"xrange: {self._xrange}")
# XXX: lol brutal, the internals of `CurvePoint` (inherited by # # XXX: lol brutal, the internals of `CurvePoint` (inherited by
# our `LineDot`) required ``.getData()`` to work.. # # our `LineDot`) required ``.getData()`` to work..
self.xData = x # self.xData = x
self.yData = y # self.yData = y
# downsampling incremental state checking # # downsampling incremental state checking
uppx = self.x_uppx() # uppx = self.x_uppx()
px_width = self.px_width() # px_width = self.px_width()
uppx_diff = (uppx - self._last_uppx) # uppx_diff = (uppx - self._last_uppx)
new_sample_rate = False # new_sample_rate = False
should_ds = self._in_ds # should_ds = self._in_ds
showing_src_data = self._in_ds # showing_src_data = self._in_ds
# should_redraw = False # # should_redraw = False
# by default we only pull data up to the last (current) index # # by default we only pull data up to the last (current) index
x_out = x[:slice_to_head] # x_out = x[:slice_to_head]
y_out = y[:slice_to_head] # y_out = y[:slice_to_head]
# if a view range is passed, plan to draw the # # if a view range is passed, plan to draw the
# source ouput that's "in view" of the chart. # # source ouput that's "in view" of the chart.
if ( # if (
view_range # view_range
# and not self._in_ds # # and not self._in_ds
# and not prepend_length > 0 # # and not prepend_length > 0
): # ):
# print(f'{self._name} vr: {view_range}') # # print(f'{self._name} vr: {view_range}')
# by default we only pull data up to the last (current) index # # by default we only pull data up to the last (current) index
x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head] # x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head]
profiler(f'view range slice {view_range}') # profiler(f'view range slice {view_range}')
vl, vr = view_range # vl, vr = view_range
# last_ivr = self._x_iv_range # # last_ivr = self._x_iv_range
# ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) # # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1])
zoom_or_append = False # zoom_or_append = False
last_vr = self._vr # last_vr = self._vr
last_ivr = self._avr # last_ivr = self._avr
if last_vr: # if last_vr:
# relative slice indices # # relative slice indices
lvl, lvr = last_vr # lvl, lvr = last_vr
# abs slice indices # # abs slice indices
al, ar = last_ivr # al, ar = last_ivr
# append_length = int(x[-1] - istop) # # append_length = int(x[-1] - istop)
# append_length = int(x_iv[-1] - ar) # # append_length = int(x_iv[-1] - ar)
# left_change = abs(x_iv[0] - al) >= 1 # # left_change = abs(x_iv[0] - al) >= 1
# right_change = abs(x_iv[-1] - ar) >= 1 # # right_change = abs(x_iv[-1] - ar) >= 1
if ( # if (
# likely a zoom view change # # likely a zoom view change
(vr - lvr) > 2 or vl < lvl # (vr - lvr) > 2 or vl < lvl
# append / prepend update # # append / prepend update
# we had an append update where the view range # # we had an append update where the view range
# didn't change but the data-viewed (shifted) # # didn't change but the data-viewed (shifted)
# underneath, so we need to redraw. # # underneath, so we need to redraw.
# or left_change and right_change and last_vr == view_range # # or left_change and right_change and last_vr == view_range
# not (left_change and right_change) and ivr # # not (left_change and right_change) and ivr
# ( # # (
# or abs(x_iv[ivr] - livr) > 1 # # or abs(x_iv[ivr] - livr) > 1
): # ):
zoom_or_append = True # zoom_or_append = True
# if last_ivr: # # if last_ivr:
# liivl, liivr = last_ivr # # liivl, liivr = last_ivr
if ( # if (
view_range != last_vr # view_range != last_vr
and ( # and (
append_length > 1 # append_length > 1
or zoom_or_append # or zoom_or_append
) # )
): # ):
should_redraw = True # should_redraw = True
# print("REDRAWING BRUH") # # print("REDRAWING BRUH")
self._vr = view_range # self._vr = view_range
self._avr = x_iv[0], x_iv[slice_to_head] # self._avr = x_iv[0], x_iv[slice_to_head]
# x_last = x_iv[-1] # # x_last = x_iv[-1]
# y_last = y_iv[-1] # # y_last = y_iv[-1]
# self._last_vr = view_range # # self._last_vr = view_range
# self.disable_cache() # # self.disable_cache()
# flip_cache = True # # flip_cache = True
if prepend_length > 0: # if prepend_length > 0:
should_redraw = True # should_redraw = True
# check for downsampling conditions # # check for downsampling conditions
if ( # if (
# std m4 downsample conditions # # std m4 downsample conditions
abs(uppx_diff) >= 1 # abs(uppx_diff) >= 1
): # ):
log.info( # log.info(
f'{self._name} sampler change: {self._last_uppx} -> {uppx}' # f'{self._name} sampler change: {self._last_uppx} -> {uppx}'
) # )
self._last_uppx = uppx # self._last_uppx = uppx
new_sample_rate = True # new_sample_rate = True
showing_src_data = False # showing_src_data = False
should_redraw = True # should_redraw = True
should_ds = True # should_ds = True
elif ( # elif (
uppx <= 2 # uppx <= 2
and self._in_ds # and self._in_ds
): # ):
# we should de-downsample back to our original # # we should de-downsample back to our original
# source data so we clear our path data in prep # # source data so we clear our path data in prep
# to generate a new one from original source data. # # to generate a new one from original source data.
should_redraw = True # should_redraw = True
new_sample_rate = True # new_sample_rate = True
should_ds = False # should_ds = False
showing_src_data = True # showing_src_data = True
# no_path_yet = self.path is None # # no_path_yet = self.path is None
if ( # if (
self.path is None # self.path is None
or should_redraw # or should_redraw
or new_sample_rate # or new_sample_rate
or prepend_length > 0 # or prepend_length > 0
): # ):
if should_redraw: # if should_redraw:
if self.path: # if self.path:
self.path.clear() # self.path.clear()
profiler('cleared paths due to `should_redraw=True`') # profiler('cleared paths due to `should_redraw=True`')
if self.fast_path: # if self.fast_path:
self.fast_path.clear() # self.fast_path.clear()
profiler('cleared paths due to `should_redraw` set') # profiler('cleared paths due to `should_redraw` set')
if new_sample_rate and showing_src_data: # if new_sample_rate and showing_src_data:
# if self._in_ds: # # if self._in_ds:
log.info(f'DEDOWN -> {self._name}') # log.info(f'DEDOWN -> {self._name}')
self._in_ds = False # self._in_ds = False
elif should_ds and uppx > 1: # elif should_ds and uppx > 1:
x_out, y_out = xy_downsample( # x_out, y_out = xy_downsample(
x_out, # x_out,
y_out, # y_out,
uppx, # uppx,
) # )
profiler(f'FULL PATH downsample redraw={should_ds}') # profiler(f'FULL PATH downsample redraw={should_ds}')
self._in_ds = True # self._in_ds = True
self.path = pg.functions.arrayToQPath( # self.path = pg.functions.arrayToQPath(
x_out, # x_out,
y_out, # y_out,
connect='all', # connect='all',
finiteCheck=False, # finiteCheck=False,
path=self.path, # path=self.path,
) # )
self.prepareGeometryChange() # self.prepareGeometryChange()
profiler( # profiler(
'generated fresh path. ' # 'generated fresh path. '
f'(should_redraw: {should_redraw} ' # f'(should_redraw: {should_redraw} '
f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' # f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})'
) # )
# profiler(f'DRAW PATH IN VIEW -> {self._name}') # # profiler(f'DRAW PATH IN VIEW -> {self._name}')
# reserve mem allocs see: # # reserve mem allocs see:
# - https://doc.qt.io/qt-5/qpainterpath.html#reserve # # - https://doc.qt.io/qt-5/qpainterpath.html#reserve
# - https://doc.qt.io/qt-5/qpainterpath.html#capacity # # - https://doc.qt.io/qt-5/qpainterpath.html#capacity
# - https://doc.qt.io/qt-5/qpainterpath.html#clear # # - https://doc.qt.io/qt-5/qpainterpath.html#clear
# XXX: right now this is based on had hoc checks on a # # XXX: right now this is based on had hoc checks on a
# hidpi 3840x2160 4k monitor but we should optimize for # # hidpi 3840x2160 4k monitor but we should optimize for
# the target display(s) on the sys. # # the target display(s) on the sys.
# if no_path_yet: # # if no_path_yet:
# self.path.reserve(int(500e3)) # # self.path.reserve(int(500e3))
# TODO: get this piecewise prepend working - right now it's # # TODO: get this piecewise prepend working - right now it's
# giving heck on vwap... # # giving heck on vwap...
# elif prepend_length: # # elif prepend_length:
# breakpoint() # # breakpoint()
# prepend_path = pg.functions.arrayToQPath( # # prepend_path = pg.functions.arrayToQPath(
# x[0:prepend_length], # # x[0:prepend_length],
# y[0:prepend_length], # # y[0:prepend_length],
# connect='all' # # connect='all'
# ) # # )
# # swap prepend path in "front" # # # swap prepend path in "front"
# old_path = self.path # # old_path = self.path
# self.path = prepend_path # # self.path = prepend_path
# # self.path.moveTo(new_x[0], new_y[0]) # # # self.path.moveTo(new_x[0], new_y[0])
# self.path.connectPath(old_path) # # self.path.connectPath(old_path)
elif ( # elif (
append_length > 0 # append_length > 0
and do_append # and do_append
and not should_redraw # and not should_redraw
# and not view_range # # and not view_range
): # ):
print(f'{self._name} append len: {append_length}') # print(f'{self._name} append len: {append_length}')
new_x = x[-append_length - 2:slice_to_head] # new_x = x[-append_length - 2:slice_to_head]
new_y = y[-append_length - 2:slice_to_head] # new_y = y[-append_length - 2:slice_to_head]
profiler('sliced append path') # profiler('sliced append path')
profiler( # profiler(
f'diffed array input, append_length={append_length}' # f'diffed array input, append_length={append_length}'
) # )
# if should_ds: # # if should_ds:
# new_x, new_y = xy_downsample( # # new_x, new_y = xy_downsample(
# new_x, # # new_x,
# new_y, # # new_y,
# uppx, # # uppx,
# ) # # )
# profiler(f'fast path downsample redraw={should_ds}') # # profiler(f'fast path downsample redraw={should_ds}')
append_path = pg.functions.arrayToQPath( # append_path = pg.functions.arrayToQPath(
new_x, # new_x,
new_y, # new_y,
connect='all', # connect='all',
finiteCheck=False, # finiteCheck=False,
path=self.fast_path, # path=self.fast_path,
) # )
profiler('generated append qpath') # profiler('generated append qpath')
if self.use_fpath: # if self.use_fpath:
# an attempt at trying to make append-updates faster.. # # an attempt at trying to make append-updates faster..
if self.fast_path is None: # if self.fast_path is None:
self.fast_path = append_path # self.fast_path = append_path
# self.fast_path.reserve(int(6e3)) # # self.fast_path.reserve(int(6e3))
else: # else:
self.fast_path.connectPath(append_path) # self.fast_path.connectPath(append_path)
size = self.fast_path.capacity() # size = self.fast_path.capacity()
profiler(f'connected fast path w size: {size}') # profiler(f'connected fast path w size: {size}')
# print(f"append_path br: {append_path.boundingRect()}") # # print(f"append_path br: {append_path.boundingRect()}")
# self.path.moveTo(new_x[0], new_y[0]) # # self.path.moveTo(new_x[0], new_y[0])
# path.connectPath(append_path) # # path.connectPath(append_path)
# XXX: lol this causes a hang.. # # XXX: lol this causes a hang..
# self.path = self.path.simplified() # # self.path = self.path.simplified()
else: # else:
size = self.path.capacity() # size = self.path.capacity()
profiler(f'connected history path w size: {size}') # profiler(f'connected history path w size: {size}')
self.path.connectPath(append_path) # self.path.connectPath(append_path)
# other merging ideas: # # other merging ideas:
# https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths # # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths
# path.addPath(append_path) # # path.addPath(append_path)
# path.closeSubpath() # # path.closeSubpath()
# TODO: try out new work from `pyqtgraph` main which # # TODO: try out new work from `pyqtgraph` main which
# should repair horrid perf: # # should repair horrid perf:
# https://github.com/pyqtgraph/pyqtgraph/pull/2032 # # https://github.com/pyqtgraph/pyqtgraph/pull/2032
# ok, nope still horrible XD # # ok, nope still horrible XD
# if self._fill: # # if self._fill:
# # XXX: super slow set "union" op # # # XXX: super slow set "union" op
# self.path = self.path.united(append_path).simplified() # # self.path = self.path.united(append_path).simplified()
# self.disable_cache() # # self.disable_cache()
# flip_cache = True # # flip_cache = True
# if draw_last: # # if draw_last:
# self.draw_last(x, y) # # self.draw_last(x, y)
# profiler('draw last segment') # # profiler('draw last segment')
# if flip_cache: # # if flip_cache:
# # # XXX: seems to be needed to avoid artifacts (see above). # # # # XXX: seems to be needed to avoid artifacts (see above).
# self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) # # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
# trigger redraw of path # # trigger redraw of path
# do update before reverting to cache mode # # do update before reverting to cache mode
self.update() # self.update()
profiler('.update()') # profiler('.update()')
def draw_last( def draw_last(
self, self,

809
piker/ui/_flows.py
View File

@ -23,7 +23,7 @@ incremental update.
''' '''
from __future__ import annotations from __future__ import annotations
from functools import partial # from functools import partial
from typing import ( from typing import (
Optional, Optional,
Callable, Callable,
@ -54,6 +54,7 @@ from ._pathops import (
gen_ohlc_qpath, gen_ohlc_qpath,
ohlc_to_line, ohlc_to_line,
to_step_format, to_step_format,
xy_downsample,
) )
from ._ohlc import ( from ._ohlc import (
BarItems, BarItems,
@ -152,18 +153,18 @@ def render_baritems(
last_read=read, last_read=read,
) )
ds_curve_r = Renderer( # ds_curve_r = Renderer(
flow=self, # flow=self,
# just swap in the flat view # # just swap in the flat view
# data_t=lambda array: self.gy.array, # # data_t=lambda array: self.gy.array,
last_read=read, # last_read=read,
draw_path=partial( # draw_path=partial(
rowarr_to_path, # rowarr_to_path,
x_basis=None, # x_basis=None,
), # ),
) # )
curve = FastAppendCurve( curve = FastAppendCurve(
name='OHLC', name='OHLC',
color=graphics._color, color=graphics._color,
@ -173,12 +174,14 @@ def render_baritems(
# baseline "line" downsampled OHLC curve that should # baseline "line" downsampled OHLC curve that should
# kick on only when we reach a certain uppx threshold. # kick on only when we reach a certain uppx threshold.
self._render_table[0] = ( self._render_table[0] = curve
ds_curve_r, # (
curve, # # ds_curve_r,
) # curve,
# )
dsc_r, curve = self._render_table[0] curve = self._render_table[0]
# dsc_r, curve = self._render_table[0]
# do checks for whether or not we require downsampling: # do checks for whether or not we require downsampling:
# - if we're **not** downsampling then we simply want to # - if we're **not** downsampling then we simply want to
@ -276,19 +279,20 @@ def render_baritems(
profiler('flattened ustruct in-view OHLC data') profiler('flattened ustruct in-view OHLC data')
# pass into curve graphics processing # pass into curve graphics processing
curve.update_from_array( # curve.update_from_array(
x, # x,
y, # y,
x_iv=x_iv, # x_iv=x_iv,
y_iv=y_iv, # y_iv=y_iv,
view_range=(ivl, ivr), # hack # view_range=(ivl, ivr), # hack
profiler=profiler, # profiler=profiler,
# should_redraw=False, # # should_redraw=False,
# NOTE: already passed through by display loop? # # NOTE: already passed through by display loop?
# do_append=uppx < 16, # # do_append=uppx < 16,
**kwargs, # **kwargs,
) # )
# curve.draw_last(x, y)
curve.show() curve.show()
profiler('updated ds curve') profiler('updated ds curve')
@ -349,6 +353,130 @@ def render_baritems(
# ) # )
# graphics.draw_last(last) # graphics.draw_last(last)
if should_line:
return (
curve,
x,
y,
x_iv,
y_iv,
)
def update_step_data(
flow: Flow,
shm: ShmArray,
ivl: int,
ivr: int,
array_key: str,
iflat_first: int,
iflat: int,
profiler: pg.debug.Profiler,
) -> tuple:
self = flow
(
# iflat_first,
# iflat,
ishm_last,
ishm_first,
) = (
# self._iflat_first,
# self._iflat_last,
shm._last.value,
shm._first.value
)
il = max(iflat - 1, 0)
profiler('read step mode incr update indices')
# check for shm prepend updates since last read.
if iflat_first != ishm_first:
print(f'prepend {array_key}')
# i_prepend = self.shm._array['index'][
# ishm_first:iflat_first]
y_prepend = self.shm._array[array_key][
ishm_first:iflat_first
]
y2_prepend = np.broadcast_to(
y_prepend[:, None], (y_prepend.size, 2),
)
# write newly prepended data to flattened copy
self.gy[ishm_first:iflat_first] = y2_prepend
self._iflat_first = ishm_first
profiler('prepended step mode history')
append_diff = ishm_last - iflat
if append_diff:
# slice up to the last datum since last index/append update
# new_x = self.shm._array[il:ishm_last]['index']
new_y = self.shm._array[il:ishm_last][array_key]
new_y2 = np.broadcast_to(
new_y[:, None], (new_y.size, 2),
)
self.gy[il:ishm_last] = new_y2
profiler('updated step curve data')
# print(
# f'append size: {append_diff}\n'
# f'new_x: {new_x}\n'
# f'new_y: {new_y}\n'
# f'new_y2: {new_y2}\n'
# f'new gy: {gy}\n'
# )
# update local last-index tracking
self._iflat_last = ishm_last
# slice out up-to-last step contents
x_step = self.gx[ishm_first:ishm_last+2]
# shape to 1d
x = x_step.reshape(-1)
profiler('sliced step x')
y_step = self.gy[ishm_first:ishm_last+2]
lasts = self.shm.array[['index', array_key]]
last = lasts[array_key][-1]
y_step[-1] = last
# shape to 1d
y = y_step.reshape(-1)
# s = 6
# print(f'lasts: {x[-2*s:]}, {y[-2*s:]}')
profiler('sliced step y')
# do all the same for only in-view data
ys_iv = y_step[ivl:ivr+1]
xs_iv = x_step[ivl:ivr+1]
y_iv = ys_iv.reshape(ys_iv.size)
x_iv = xs_iv.reshape(xs_iv.size)
# print(
# f'ys_iv : {ys_iv[-s:]}\n'
# f'y_iv: {y_iv[-s:]}\n'
# f'xs_iv: {xs_iv[-s:]}\n'
# f'x_iv: {x_iv[-s:]}\n'
# )
profiler('sliced in view step data')
# legacy full-recompute-everytime method
# x, y = ohlc_flatten(array)
# x_iv, y_iv = ohlc_flatten(in_view)
# profiler('flattened OHLC data')
return (
x,
y,
x_iv,
y_iv,
append_diff,
)
class Flow(msgspec.Struct): # , frozen=True): class Flow(msgspec.Struct): # , frozen=True):
''' '''
@ -368,11 +496,19 @@ class Flow(msgspec.Struct): # , frozen=True):
is_ohlc: bool = False is_ohlc: bool = False
render: bool = True # toggle for display loop render: bool = True # toggle for display loop
# pre-graphics formatted data
gy: Optional[ShmArray] = None gy: Optional[ShmArray] = None
gx: Optional[np.ndarray] = None gx: Optional[np.ndarray] = None
# pre-graphics update indices
_iflat_last: int = 0 _iflat_last: int = 0
_iflat_first: int = 0 _iflat_first: int = 0
# view-range incremental state
_vr: Optional[tuple] = None
_avr: Optional[tuple] = None
# downsampling state
_last_uppx: float = 0 _last_uppx: float = 0
_in_ds: bool = False _in_ds: bool = False
@ -495,7 +631,11 @@ class Flow(msgspec.Struct): # , frozen=True):
start, l, lbar, rbar, r, end, start, l, lbar, rbar, r, end,
) )
def read(self) -> tuple[ def read(
self,
array_field: Optional[str] = None,
) -> tuple[
int, int, np.ndarray, int, int, np.ndarray,
int, int, np.ndarray, int, int, np.ndarray,
]: ]:
@ -513,6 +653,9 @@ class Flow(msgspec.Struct): # , frozen=True):
lbar_i = max(l, ifirst) - ifirst lbar_i = max(l, ifirst) - ifirst
rbar_i = min(r, ilast) - ifirst rbar_i = min(r, ilast) - ifirst
if array_field:
array = array[array_field]
# TODO: we could do it this way as well no? # TODO: we could do it this way as well no?
# to_draw = array[lbar - ifirst:(rbar - ifirst) + 1] # to_draw = array[lbar - ifirst:(rbar - ifirst) + 1]
in_view = array[lbar_i: rbar_i + 1] in_view = array[lbar_i: rbar_i + 1]
@ -532,6 +675,7 @@ class Flow(msgspec.Struct): # , frozen=True):
array_key: Optional[str] = None, array_key: Optional[str] = None,
profiler: Optional[pg.debug.Profiler] = None, profiler: Optional[pg.debug.Profiler] = None,
do_append: bool = True,
**kwargs, **kwargs,
@ -557,15 +701,20 @@ class Flow(msgspec.Struct): # , frozen=True):
) = self.read() ) = self.read()
profiler('read src shm data') profiler('read src shm data')
graphics = self.graphics
if ( if (
not in_view.size not in_view.size
or not render or not render
): ):
return self.graphics return graphics
graphics = self.graphics out: Optional[tuple] = None
if isinstance(graphics, BarItems): if isinstance(graphics, BarItems):
render_baritems( # XXX: special case where we change out graphics
# to a line after a certain uppx threshold.
# render_baritems(
out = render_baritems(
self, self,
graphics, graphics,
read, read,
@ -573,14 +722,74 @@ class Flow(msgspec.Struct): # , frozen=True):
**kwargs, **kwargs,
) )
else: if out is None:
# ``FastAppendCurve`` case: return graphics
array_key = array_key or self.name
uppx = graphics.x_uppx()
profiler(f'read uppx {uppx}')
if graphics._step_mode and self.gy is None: # return graphics
shm = self.shm
r = self._src_r
if not r:
# just using for ``.diff()`` atm..
r = self._src_r = Renderer(
flow=self,
# TODO: rename this to something with ohlc
# draw_path=gen_ohlc_qpath,
last_read=read,
)
# ``FastAppendCurve`` case:
array_key = array_key or self.name
new_sample_rate = False
should_ds = self._in_ds
showing_src_data = self._in_ds
# draw_last: bool = True
slice_to_head: int = -1
should_redraw: bool = False
shm = self.shm
# if a view range is passed, plan to draw the
# source ouput that's "in view" of the chart.
view_range = (ivl, ivr) if use_vr else None
if out is not None:
# hack to handle ds curve from bars above
(
graphics, # curve
x,
y,
x_iv,
y_iv,
) = out
else:
# full input data
x = array['index']
y = array[array_key]
# inview data
x_iv = in_view['index']
y_iv = in_view[array_key]
# downsampling incremental state checking
uppx = graphics.x_uppx()
# px_width = graphics.px_width()
uppx_diff = (uppx - self._last_uppx)
profiler(f'diffed uppx {uppx}')
x_last = x[-1]
y_last = y[-1]
slice_to_head = -1
profiler('sliced input arrays')
if graphics._step_mode:
slice_to_head = -2
if self.gy is None:
( (
self._iflat_first, self._iflat_first,
self.gx, self.gx,
@ -591,177 +800,324 @@ class Flow(msgspec.Struct): # , frozen=True):
) )
profiler('generated step mode data') profiler('generated step mode data')
if graphics._step_mode: (
( x,
iflat_first, y,
iflat, x_iv,
ishm_last, y_iv,
ishm_first, append_diff,
) = (
self._iflat_first, ) = update_step_data(
self._iflat_last, self,
self.shm._last.value, shm,
self.shm._first.value ivl,
ivr,
array_key,
self._iflat_first,
self._iflat_last,
profiler,
)
graphics._last_line = QLineF(
x_last - 0.5, 0,
x_last + 0.5, 0,
)
graphics._last_step_rect = QRectF(
x_last - 0.5, 0,
x_last + 0.5, y_last,
)
should_redraw = bool(append_diff)
# graphics.reset_cache()
# print(
# f"path br: {graphics.path.boundingRect()}\n",
# # f"fast path br: {graphics.fast_path.boundingRect()}",
# f"last rect br: {graphics._last_step_rect}\n",
# f"full br: {graphics._br}\n",
# )
# compute the length diffs between the first/last index entry in
# the input data and the last indexes we have on record from the
# last time we updated the curve index.
prepend_length, append_length = r.diff(read)
# print((prepend_length, append_length))
# old_prepend_length = int(istart - x[0])
# old_append_length = int(x[-1] - istop)
# MAIN RENDER LOGIC:
# - determine in view data and redraw on range change
# - determine downsampling ops if needed
# - (incrementally) update ``QPainterPath``
if (
view_range
# and not self._in_ds
# and not prepend_length > 0
):
# print(f'{self._name} vr: {view_range}')
# by default we only pull data up to the last (current) index
x_out = x_iv[:slice_to_head]
y_out = y_iv[:slice_to_head]
profiler(f'view range slice {view_range}')
vl, vr = view_range
zoom_or_append = False
last_vr = self._vr
last_ivr = self._avr
# incremental in-view data update.
if last_vr:
# relative slice indices
lvl, lvr = last_vr
# abs slice indices
al, ar = last_ivr
# append_length = int(x[-1] - istop)
# append_length = int(x_iv[-1] - ar)
# left_change = abs(x_iv[0] - al) >= 1
# right_change = abs(x_iv[-1] - ar) >= 1
if (
# likely a zoom view change
(vr - lvr) > 2 or vl < lvl
# append / prepend update
# we had an append update where the view range
# didn't change but the data-viewed (shifted)
# underneath, so we need to redraw.
# or left_change and right_change and last_vr == view_range
# not (left_change and right_change) and ivr
# (
# or abs(x_iv[ivr] - livr) > 1
):
zoom_or_append = True
# if last_ivr:
# liivl, liivr = last_ivr
if (
view_range != last_vr
and (
append_length > 1
or zoom_or_append
) )
):
should_redraw = True
# print("REDRAWING BRUH")
il = max(iflat - 1, 0) self._vr = view_range
profiler('read step mode incr update indices') self._avr = x_iv[0], x_iv[slice_to_head]
# check for shm prepend updates since last read. if prepend_length > 0:
if iflat_first != ishm_first: should_redraw = True
print(f'prepend {array_key}') # check for downsampling conditions
if (
# std m4 downsample conditions
# px_width
# and abs(uppx_diff) >= 1
abs(uppx_diff) >= 1
):
log.info(
f'{array_key} sampler change: {self._last_uppx} -> {uppx}'
)
self._last_uppx = uppx
new_sample_rate = True
showing_src_data = False
should_redraw = True
should_ds = True
# i_prepend = self.shm._array['index'][ elif (
# ishm_first:iflat_first] uppx <= 2
y_prepend = self.shm._array[array_key][ and self._in_ds
ishm_first:iflat_first ):
] # we should de-downsample back to our original
# source data so we clear our path data in prep
# to generate a new one from original source data.
should_redraw = True
new_sample_rate = True
should_ds = False
showing_src_data = True
y2_prepend = np.broadcast_to( # no_path_yet = self.path is None
y_prepend[:, None], (y_prepend.size, 2), fast_path = graphics.fast_path
) if (
graphics.path is None
or should_redraw
or new_sample_rate
or prepend_length > 0
):
if should_redraw:
if graphics.path:
graphics.path.clear()
profiler('cleared paths due to `should_redraw=True`')
# write newly prepended data to flattened copy if graphics.fast_path:
self.gy[ishm_first:iflat_first] = y2_prepend graphics.fast_path.clear()
self._iflat_first = ishm_first
profiler('prepended step mode history')
append_diff = ishm_last - iflat profiler('cleared paths due to `should_redraw` set')
if append_diff:
# slice up to the last datum since last index/append update if new_sample_rate and showing_src_data:
# new_x = self.shm._array[il:ishm_last]['index'] # if self._in_ds:
new_y = self.shm._array[il:ishm_last][array_key] log.info(f'DEDOWN -> {self.name}')
new_y2 = np.broadcast_to( self._in_ds = False
new_y[:, None], (new_y.size, 2),
)
self.gy[il:ishm_last] = new_y2
profiler('updated step curve data')
# print( # elif should_ds and uppx and px_width > 1:
# f'append size: {append_diff}\n' elif should_ds and uppx > 1:
# f'new_x: {new_x}\n'
# f'new_y: {new_y}\n'
# f'new_y2: {new_y2}\n'
# f'new gy: {gy}\n'
# )
# update local last-index tracking x_out, y_out = xy_downsample(
self._iflat_last = ishm_last x_out,
y_out,
# slice out up-to-last step contents uppx,
x_step = self.gx[ishm_first:ishm_last+2] # px_width,
# shape to 1d
x = x_step.reshape(-1)
profiler('sliced step x')
y_step = self.gy[ishm_first:ishm_last+2]
lasts = self.shm.array[['index', array_key]]
last = lasts[array_key][-1]
y_step[-1] = last
# shape to 1d
y = y_step.reshape(-1)
# s = 6
# print(f'lasts: {x[-2*s:]}, {y[-2*s:]}')
profiler('sliced step y')
# do all the same for only in-view data
ys_iv = y_step[ivl:ivr+1]
xs_iv = x_step[ivl:ivr+1]
y_iv = ys_iv.reshape(ys_iv.size)
x_iv = xs_iv.reshape(xs_iv.size)
# print(
# f'ys_iv : {ys_iv[-s:]}\n'
# f'y_iv: {y_iv[-s:]}\n'
# f'xs_iv: {xs_iv[-s:]}\n'
# f'x_iv: {x_iv[-s:]}\n'
# )
profiler('sliced in view step data')
# legacy full-recompute-everytime method
# x, y = ohlc_flatten(array)
# x_iv, y_iv = ohlc_flatten(in_view)
# profiler('flattened OHLC data')
x_last = array['index'][-1]
y_last = array[array_key][-1]
graphics._last_line = QLineF(
x_last - 0.5, 0,
x_last + 0.5, 0,
) )
graphics._last_step_rect = QRectF( profiler(f'FULL PATH downsample redraw={should_ds}')
x_last - 0.5, 0, self._in_ds = True
x_last + 0.5, y_last,
)
# graphics.update()
graphics.update_from_array( graphics.path = pg.functions.arrayToQPath(
x=x, x_out,
y=y, y_out,
connect='all',
finiteCheck=False,
path=graphics.path,
)
graphics.prepareGeometryChange()
profiler(
'generated fresh path. '
f'(should_redraw: {should_redraw} '
f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})'
)
# profiler(f'DRAW PATH IN VIEW -> {self.name}')
x_iv=x_iv, # reserve mem allocs see:
y_iv=y_iv, # - https://doc.qt.io/qt-5/qpainterpath.html#reserve
# - https://doc.qt.io/qt-5/qpainterpath.html#capacity
# - https://doc.qt.io/qt-5/qpainterpath.html#clear
# XXX: right now this is based on had hoc checks on a
# hidpi 3840x2160 4k monitor but we should optimize for
# the target display(s) on the sys.
# if no_path_yet:
# graphics.path.reserve(int(500e3))
view_range=(ivl, ivr) if use_vr else None, # TODO: get this piecewise prepend working - right now it's
# giving heck on vwap...
# elif prepend_length:
# breakpoint()
draw_last=False, # prepend_path = pg.functions.arrayToQPath(
slice_to_head=-2, # x[0:prepend_length],
# y[0:prepend_length],
# connect='all'
# )
should_redraw=bool(append_diff), # # swap prepend path in "front"
# old_path = graphics.path
# graphics.path = prepend_path
# # graphics.path.moveTo(new_x[0], new_y[0])
# graphics.path.connectPath(old_path)
# NOTE: already passed through by display loop? elif (
# do_append=uppx < 16, append_length > 0
profiler=profiler, and do_append
and not should_redraw
# and not view_range
):
print(f'{self.name} append len: {append_length}')
new_x = x[-append_length - 2:slice_to_head]
new_y = y[-append_length - 2:slice_to_head]
profiler('sliced append path')
**kwargs profiler(
) f'diffed array input, append_length={append_length}'
profiler('updated step mode curve') )
# graphics.reset_cache()
# print(
# f"path br: {graphics.path.boundingRect()}\n",
# # f"fast path br: {graphics.fast_path.boundingRect()}",
# f"last rect br: {graphics._last_step_rect}\n",
# f"full br: {graphics._br}\n",
# )
# if should_ds:
# new_x, new_y = xy_downsample(
# new_x,
# new_y,
# px_width,
# uppx,
# )
# profiler(f'fast path downsample redraw={should_ds}')
append_path = pg.functions.arrayToQPath(
new_x,
new_y,
connect='all',
finiteCheck=False,
path=graphics.fast_path,
)
profiler('generated append qpath')
if graphics.use_fpath:
print("USING FPATH")
# an attempt at trying to make append-updates faster..
if fast_path is None:
graphics.fast_path = append_path
# self.fast_path.reserve(int(6e3))
else:
fast_path.connectPath(append_path)
size = fast_path.capacity()
profiler(f'connected fast path w size: {size}')
# print(f"append_path br: {append_path.boundingRect()}")
# graphics.path.moveTo(new_x[0], new_y[0])
# path.connectPath(append_path)
# XXX: lol this causes a hang..
# graphics.path = graphics.path.simplified()
else: else:
x = array['index'] size = graphics.path.capacity()
y = array[array_key] profiler(f'connected history path w size: {size}')
x_iv = in_view['index'] graphics.path.connectPath(append_path)
y_iv = in_view[array_key]
profiler('sliced input arrays')
# graphics.draw_last(x, y) # graphics.update_from_array(
# x=x,
# y=y,
graphics.update_from_array( # x_iv=x_iv,
x=x, # y_iv=y_iv,
y=y,
x_iv=x_iv, # view_range=(ivl, ivr) if use_vr else None,
y_iv=y_iv,
view_range=(ivl, ivr) if use_vr else None, # # NOTE: already passed through by display loop.
# # do_append=uppx < 16,
# do_append=do_append,
# NOTE: already passed through by display loop? # slice_to_head=slice_to_head,
# do_append=uppx < 16, # should_redraw=should_redraw,
profiler=profiler, # profiler=profiler,
**kwargs # **kwargs
) # )
profiler('`graphics.update_from_array()` complete')
graphics.draw_last(x, y)
profiler('draw last segment')
graphics.update()
profiler('.update()')
profiler('`graphics.update_from_array()` complete')
return graphics return graphics
class Renderer(msgspec.Struct): class Renderer(msgspec.Struct):
flow: Flow flow: Flow
# last array view read
last_read: Optional[tuple] = None
# called to render path graphics # called to render path graphics
draw_path: Callable[np.ndarray, QPainterPath] draw_path: Optional[Callable[np.ndarray, QPainterPath]] = None
# output graphics rendering, the main object
# processed in ``QGraphicsObject.paint()``
path: Optional[QPainterPath] = None
# called on input data but before any graphics format # called on input data but before any graphics format
# conversions or processing. # conversions or processing.
@ -778,25 +1134,66 @@ class Renderer(msgspec.Struct):
prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None
append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None
# last array view read # incremental update state(s)
last_read: Optional[np.ndarray] = None # _in_ds: bool = False
# _last_uppx: float = 0
_last_vr: Optional[tuple[float, float]] = None
_last_ivr: Optional[tuple[float, float]] = None
# output graphics rendering, the main object def diff(
# processed in ``QGraphicsObject.paint()`` self,
path: Optional[QPainterPath] = None new_read: tuple[np.ndarray],
# def diff( ) -> tuple[np.ndarray]:
# self,
# latest_read: tuple[np.ndarray],
# ) -> tuple[np.ndarray]: (
# # blah blah blah last_xfirst,
# # do diffing for prepend, append and last entry last_xlast,
# return ( last_array,
# to_prepend last_ivl, last_ivr,
# to_append last_in_view,
# last, ) = self.last_read
# )
# TODO: can the renderer just call ``Flow.read()`` directly?
# unpack latest source data read
(
xfirst,
xlast,
array,
ivl,
ivr,
in_view,
) = new_read
# compute the length diffs between the first/last index entry in
# the input data and the last indexes we have on record from the
# last time we updated the curve index.
prepend_length = int(last_xfirst - xfirst)
append_length = int(xlast - last_xlast)
# TODO: eventually maybe we can implement some kind of
# transform on the ``QPainterPath`` that will more or less
# detect the diff in "elements" terms?
# update state
self.last_read = new_read
# blah blah blah
# do diffing for prepend, append and last entry
return (
prepend_length,
append_length,
# last,
)
def draw_path(
self,
should_redraw: bool = False,
) -> QPainterPath:
if should_redraw:
if self.path:
self.path.clear()
# profiler('cleared paths due to `should_redraw=True`')
def render( def render(
self, self,
@ -819,11 +1216,30 @@ class Renderer(msgspec.Struct):
- blah blah blah (from notes) - blah blah blah (from notes)
''' '''
# do full source data render to path # get graphics info
# TODO: can the renderer just call ``Flow.read()`` directly?
# unpack latest source data read
( (
xfirst, xlast, array, xfirst,
ivl, ivr, in_view, xlast,
) = self.last_read array,
ivl,
ivr,
in_view,
) = new_read
(
prepend_length,
append_length,
) = self.diff(new_read)
# do full source data render to path
# x = array['index']
# y = array#[array_key]
# x_iv = in_view['index']
# y_iv = in_view#[array_key]
if only_in_view: if only_in_view:
array = in_view array = in_view
@ -832,7 +1248,10 @@ class Renderer(msgspec.Struct):
# xfirst, xlast, array, ivl, ivr, in_view # xfirst, xlast, array, ivl, ivr, in_view
# ) = new_read # ) = new_read
if self.path is None or only_in_view: if (
self.path is None
or only_in_view
):
# redraw the entire source data if we have either of: # redraw the entire source data if we have either of:
# - no prior path graphic rendered or, # - no prior path graphic rendered or,
# - we always intend to re-render the data only in view # - we always intend to re-render the data only in view