WIP get incremental step curve updates working

This took longer then i care to admit XD but it definitely adds a huge
speedup and with only a few outstanding correctness bugs:

- panning from left to right causes strange trailing artifacts in the
  flows fsp (vlm) sub-plot but only when some data is off-screen on the
  left but doesn't appear to be an issue if we keep the `._set_yrange()`
  handler hooked up to the `.sigXRangeChanged` signal (but we aren't
  going to because this makes panning way slower). i've got a feeling
  this is a bug todo with the device coordinate cache stuff and we may
  need to report to Qt core?
- factoring out the step curve logic from
  `FastAppendCurve.update_from_array()` (un)fortunately required some
  logic branch uncoupling but also meant we needed special input controls
  to avoid things like redraws and curve appends for special cases,
  this will hopefully all be better rectified in code when the core of
  this method is moved into a renderer type/implementation.
- the `tina_vwap` fsp curve now somehow causes hangs when doing erratic
  scrolling on downsampled graphics data. i have no idea why or how but
  disabling it makes the issue go away (ui will literally just freeze
  and gobble CPU on a `.paint()` call until you ctrl-c the hell out of
  it). my guess is that something in the logic for standard line curves
  and appends on large data sets is the issue?

Code related changes/hacks:
- drop use of `step_path_arrays_from_1d()`, it was always a bit hacky
  (being based on `pyqtgraph` internals) and was generally hard to
  understand since it returns 1d data instead of the more expected (N,2)
  array of "step levels"; instead this is now implemented (uglily) in
  the `Flow.update_graphics()` block for step curves (which will
  obviously get cleaned up and factored elsewhere).
- add a bunch of new flags to the update method on the fast append
  curve:  `draw_last: bool`, `slice_to_head: int`, `do_append: bool`,
  `should_redraw: bool` which are all controls to aid with previously
  mentioned issues specific to getting step curve updates working
  correctly.
- add a ton of commented tinkering related code (that we may end up
  using) to both the flow and append curve methods that was written as
  part of the effort to get this all working.
- implement all step curve updating inline in `Flow.update_graphics()`
  including prepend and append logic for pre-graphics incremental step
  data maintenance and in-view slicing as well as "last step" graphics
  updating.

Obviously clean up commits coming stat B)
incremental_update_paths
Tyler Goodlet 2022-04-26 08:34:53 -04:00
parent c5beecf8a1
commit 12d60e6d9c
2 changed files with 369 additions and 138 deletions

View File

@ -45,74 +45,77 @@ log = get_logger(__name__)
# TODO: numba this instead..
def step_path_arrays_from_1d(
x: np.ndarray,
y: np.ndarray,
include_endpoints: bool = False,
# def step_path_arrays_from_1d(
# x: np.ndarray,
# y: np.ndarray,
# include_endpoints: bool = True,
) -> (np.ndarray, np.ndarray):
'''
Generate a "step mode" curve aligned with OHLC style bars
such that each segment spans each bar (aka "centered" style).
# ) -> (np.ndarray, np.ndarray):
# '''
# Generate a "step mode" curve aligned with OHLC style bars
# such that each segment spans each bar (aka "centered" style).
'''
y_out = y.copy()
x_out = x.copy()
# '''
# # y_out = y.copy()
# # x_out = x.copy()
# x2 = np.empty(
# # the data + 2 endpoints on either end for
# # "termination of the path".
# (len(x) + 1, 2),
# # we want to align with OHLC or other sampling style
# # bars likely so we need fractinal values
# dtype=float,
# # x2 = np.empty(
# # # the data + 2 endpoints on either end for
# # # "termination of the path".
# # (len(x) + 1, 2),
# # # we want to align with OHLC or other sampling style
# # # bars likely so we need fractinal values
# # dtype=float,
# # )
# x2 = np.broadcast_to(
# x[:, None],
# (
# x.size + 1,
# # 4, # only ohlc
# 2,
# ),
# ) + np.array([-0.5, 0.5])
# # x2[0] = x[0] - 0.5
# # x2[1] = x[0] + 0.5
# # x2[0, 0] = x[0] - 0.5
# # x2[0, 1] = x[0] + 0.5
# # x2[1:] = x[:, np.newaxis] + 0.5
# # import pdbpp
# # pdbpp.set_trace()
# # flatten to 1-d
# # x_out = x2.reshape(x2.size)
# # x_out = x2
# # we create a 1d with 2 extra indexes to
# # hold the start and (current) end value for the steps
# # on either end
# y2 = np.empty(
# (len(y) + 1, 2),
# dtype=y.dtype,
# )
# y2[:] = y[:, np.newaxis]
# # y2[-1] = 0
x2 = np.broadcast_to(
x[:, None],
(
x_out.size,
# 4, # only ohlc
2,
),
) + np.array([-0.5, 0.5])
# # y_out = y2
# x2[0] = x[0] - 0.5
# x2[1] = x[0] + 0.5
# x2[0, 0] = x[0] - 0.5
# x2[0, 1] = x[0] + 0.5
# x2[1:] = x[:, np.newaxis] + 0.5
# import pdbpp
# pdbpp.set_trace()
# # y_out = np.empty(
# # 2*len(y) + 2,
# # dtype=y.dtype
# # )
# flatten to 1-d
# x_out = x2.reshape(x2.size)
x_out = x2
# # flatten and set 0 endpoints
# # y_out[1:-1] = y2.reshape(y2.size)
# # y_out[0] = 0
# # y_out[-1] = 0
# we create a 1d with 2 extra indexes to
# hold the start and (current) end value for the steps
# on either end
y2 = np.empty((len(y), 2), dtype=y.dtype)
y2[:] = y[:, np.newaxis]
y2[-1] = 0
# if not include_endpoints:
# return x2[:-1], y2[:-1]
y_out = y2
# y_out = np.empty(
# 2*len(y) + 2,
# dtype=y.dtype
# )
# flatten and set 0 endpoints
# y_out[1:-1] = y2.reshape(y2.size)
# y_out[0] = 0
# y_out[-1] = 0
if not include_endpoints:
return x_out[:-1], y_out[:-1]
else:
return x_out, y_out
# else:
# return x2, y2
_line_styles: dict[str, int] = {
@ -158,6 +161,8 @@ class FastAppendCurve(pg.GraphicsObject):
self._y = self.yData = y
self._x = self.xData = x
self._vr: Optional[tuple] = None
self._avr: Optional[tuple] = None
self._br = None
self._name = name
self.path: Optional[QtGui.QPainterPath] = None
@ -171,6 +176,7 @@ class FastAppendCurve(pg.GraphicsObject):
# self._xrange: tuple[int, int] = self.dataBounds(ax=0)
self._xrange: Optional[tuple[int, int]] = None
# self._x_iv_range = None
# self._last_draw = time.time()
self._in_ds: bool = False
@ -283,6 +289,10 @@ class FastAppendCurve(pg.GraphicsObject):
view_range: Optional[tuple[int, int]] = None,
profiler: Optional[pg.debug.Profiler] = None,
draw_last: bool = True,
slice_to_head: int = -1,
do_append: bool = True,
should_redraw: bool = False,
) -> QtGui.QPainterPath:
'''
@ -297,7 +307,7 @@ class FastAppendCurve(pg.GraphicsObject):
disabled=not pg_profile_enabled(),
gt=ms_slower_then,
)
# flip_cache = False
flip_cache = False
if self._xrange:
istart, istop = self._xrange
@ -330,7 +340,7 @@ class FastAppendCurve(pg.GraphicsObject):
new_sample_rate = False
should_ds = self._in_ds
showing_src_data = self._in_ds
should_redraw = False
# should_redraw = False
# if a view range is passed, plan to draw the
# source ouput that's "in view" of the chart.
@ -342,32 +352,60 @@ class FastAppendCurve(pg.GraphicsObject):
# print(f'{self._name} vr: {view_range}')
# by default we only pull data up to the last (current) index
x_out, y_out = x_iv[:-1], y_iv[:-1]
x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head]
profiler(f'view range slice {view_range}')
ivl, ivr = view_range
vl, vr = view_range
probably_zoom_change = False
# last_ivr = self._x_iv_range
# ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1])
zoom_or_append = False
last_vr = self._vr
last_ivr = self._avr
if last_vr:
livl, livr = last_vr
# relative slice indices
lvl, lvr = last_vr
# abs slice indices
al, ar = last_ivr
# append_length = int(x[-1] - istop)
# append_length = int(x_iv[-1] - ar)
# left_change = abs(x_iv[0] - al) >= 1
# right_change = abs(x_iv[-1] - ar) >= 1
if (
ivl < livl
or (ivr - livr) > 2
# likely a zoom view change
(vr - lvr) > 2 or vl < lvl
# append / prepend update
# we had an append update where the view range
# didn't change but the data-viewed (shifted)
# underneath, so we need to redraw.
# or left_change and right_change and last_vr == view_range
# not (left_change and right_change) and ivr
# (
# or abs(x_iv[ivr] - livr) > 1
):
probably_zoom_change = True
zoom_or_append = True
# if last_ivr:
# liivl, liivr = last_ivr
if (
view_range != last_vr
and (
append_length > 1
or probably_zoom_change
or zoom_or_append
)
):
should_redraw = True
# print("REDRAWING BRUH")
self._vr = view_range
self._avr = x_iv[0], x_iv[slice_to_head]
# x_last = x_iv[-1]
# y_last = y_iv[-1]
@ -382,7 +420,7 @@ class FastAppendCurve(pg.GraphicsObject):
# or self._in_ds
# ):
# by default we only pull data up to the last (current) index
x_out, y_out = x[:-1], y[:-1]
x_out, y_out = x[:slice_to_head], y[:slice_to_head]
if prepend_length > 0:
should_redraw = True
@ -434,12 +472,12 @@ class FastAppendCurve(pg.GraphicsObject):
# step mode: draw flat top discrete "step"
# over the index space for each datum.
# if self._step_mode:
# self.disable_cache()
# flip_cache = True
# x_out, y_out = step_path_arrays_from_1d(
# x_out,
# y_out,
# )
# # self.disable_cache()
# # flip_cache = True
# # TODO: numba this bish
# profiler('generated step arrays')
@ -460,7 +498,7 @@ class FastAppendCurve(pg.GraphicsObject):
self._in_ds = False
elif should_ds and px_width and uppx:
elif should_ds and uppx and px_width > 1:
x_out, y_out = self.downsample(
x_out,
y_out,
@ -477,11 +515,9 @@ class FastAppendCurve(pg.GraphicsObject):
finiteCheck=False,
path=self.path,
)
self.prepareGeometryChange()
profiler(
'generated fresh path\n'
f'should_redraw: {should_redraw}\n'
f'should_ds: {should_ds}\n'
f'new_sample_rate: {new_sample_rate}\n'
f'generated fresh path. (should_redraw: {should_redraw} should_ds: {should_ds} new_sample_rate: {new_sample_rate})'
)
# profiler(f'DRAW PATH IN VIEW -> {self._name}')
@ -514,26 +550,29 @@ class FastAppendCurve(pg.GraphicsObject):
elif (
append_length > 0
and do_append
and not should_redraw
# and not view_range
):
new_x = x[-append_length - 2:-1]
new_y = y[-append_length - 2:-1]
print(f'{self._name} append len: {append_length}')
new_x = x[-append_length - 2:slice_to_head]
new_y = y[-append_length - 2:slice_to_head]
profiler('sliced append path')
# if self._step_mode:
# new_x, new_y = step_path_arrays_from_1d(
# new_x,
# new_y,
# )
# # [1:] since we don't need the vertical line normally at
# # the beginning of the step curve taking the first (x,
# # y) poing down to the x-axis **because** this is an
# # appended path graphic.
# new_x = new_x[1:]
# new_y = new_y[1:]
# # new_x, new_y = step_path_arrays_from_1d(
# # new_x,
# # new_y,
# # )
# # # [1:] since we don't need the vertical line normally at
# # # the beginning of the step curve taking the first (x,
# # # y) poing down to the x-axis **because** this is an
# # # appended path graphic.
# # new_x = new_x[1:]
# # new_y = new_y[1:]
# # self.disable_cache()
# # flip_cache = True
# self.disable_cache()
# flip_cache = True
# profiler('generated step data')
@ -563,7 +602,7 @@ class FastAppendCurve(pg.GraphicsObject):
# an attempt at trying to make append-updates faster..
if self.fast_path is None:
self.fast_path = append_path
self.fast_path.reserve(int(6e3))
# self.fast_path.reserve(int(6e3))
else:
self.fast_path.connectPath(append_path)
size = self.fast_path.capacity()
@ -596,19 +635,20 @@ class FastAppendCurve(pg.GraphicsObject):
# self.disable_cache()
# flip_cache = True
if draw_last:
self.draw_last(x, y)
profiler('draw last segment')
# trigger redraw of path
# do update before reverting to cache mode
# self.prepareGeometryChange()
self.update()
profiler('.update()')
# if flip_cache:
# # XXX: seems to be needed to avoid artifacts (see above).
# # # XXX: seems to be needed to avoid artifacts (see above).
# self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
# trigger redraw of path
# do update before reverting to cache mode
self.update()
profiler('.update()')
def draw_last(
self,
x: np.ndarray,
@ -624,10 +664,14 @@ class FastAppendCurve(pg.GraphicsObject):
self._last_line = QLineF(
x_last - 0.5, 0,
x_last + 0.5, 0,
# x_last, 0,
# x_last, 0,
)
self._last_step_rect = QRectF(
x_last - 0.5, 0,
x_last + 0.5, y_last
# x_last, 0,
# x_last, y_last
)
# print(
# f"path br: {self.path.boundingRect()}",
@ -640,6 +684,8 @@ class FastAppendCurve(pg.GraphicsObject):
x_last, y_last
)
self.update()
# XXX: lol brutal, the internals of `CurvePoint` (inherited by
# our `LineDot`) required ``.getData()`` to work..
def getData(self):
@ -685,7 +731,7 @@ class FastAppendCurve(pg.GraphicsObject):
# XXX: pretty annoying but, without this there's little
# artefacts on the append updates to the curve...
self.setCacheMode(QtWidgets.QGraphicsItem.NoCache)
self.prepareGeometryChange()
# self.prepareGeometryChange()
def boundingRect(self):
'''
@ -705,6 +751,7 @@ class FastAppendCurve(pg.GraphicsObject):
'''
hb = self.path.controlPointRect()
# hb = self.path.boundingRect()
hb_size = hb.size()
fp = self.fast_path
@ -713,17 +760,47 @@ class FastAppendCurve(pg.GraphicsObject):
hb_size = fhb.size() + hb_size
# print(f'hb_size: {hb_size}')
# if self._last_step_rect:
# hb_size += self._last_step_rect.size()
# if self._line:
# br = self._last_step_rect.bottomRight()
# tl = QPointF(
# # self._vr[0],
# # hb.topLeft().y(),
# # 0,
# # hb_size.height() + 1
# )
# if self._last_step_rect:
# br = self._last_step_rect.bottomRight()
# else:
# hb_size += QSizeF(1, 1)
w = hb_size.width() + 1
h = hb_size.height() + 1
# br = QPointF(
# self._vr[-1],
# # tl.x() + w,
# tl.y() + h,
# )
br = QRectF(
# top left
# hb.topLeft()
# tl,
QPointF(hb.topLeft()),
# br,
# total size
# QSizeF(hb_size)
# hb_size,
QSizeF(w, h)
)
self._br = br
# print(f'bounding rect: {br}')
return br
@ -740,6 +817,7 @@ class FastAppendCurve(pg.GraphicsObject):
disabled=not pg_profile_enabled(),
gt=ms_slower_then,
)
self.prepareGeometryChange()
if (
self._step_mode

View File

@ -34,6 +34,13 @@ import numpy as np
from numpy.lib import recfunctions as rfn
import pyqtgraph as pg
from PyQt5.QtGui import QPainterPath
from PyQt5.QtCore import (
# Qt,
QLineF,
# QSizeF,
QRectF,
# QPointF,
)
from ..data._sharedmem import (
ShmArray,
@ -465,7 +472,7 @@ class Flow(msgspec.Struct): # , frozen=True):
self.gy[
ishm_first:iflat_first
] = rfn.structured_to_unstructured(
self.shm.array[fields][:iflat_first]
self.shm._array[fields][ishm_first:iflat_first]
)
self._iflat_first = ishm_first
@ -516,6 +523,8 @@ class Flow(msgspec.Struct): # , frozen=True):
y_iv=y_iv,
view_range=(ivl, ivr), # hack
profiler=profiler,
# should_redraw=False,
# do_append=False,
)
curve.show()
profiler('updated ds curve')
@ -578,21 +587,36 @@ class Flow(msgspec.Struct): # , frozen=True):
# graphics.draw_last(last)
else:
# ``FastAppendCurve`` case:
array_key = array_key or self.name
# ``FastAppendCurve`` case:
if graphics._step_mode and self.gy is None:
self._iflat_first = self.shm._first.value
# create a flattened view onto the OHLC array
# which can be read as a line-style format
shm = self.shm
# fields = ['index', array_key]
i = shm._array['index']
out = shm._array[array_key]
i = shm._array['index'].copy()
out = shm._array[array_key].copy()
self.gx, self.gy = step_path_arrays_from_1d(i, out)
self.gx = np.broadcast_to(
i[:, None],
(i.size, 2),
) + np.array([-0.5, 0.5])
# self.gy = np.broadcast_to(
# out[:, None], (out.size, 2),
# )
self.gy = np.empty((len(out), 2), dtype=out.dtype)
self.gy[:] = out[:, np.newaxis]
# start y at origin level
self.gy[0, 0] = 0
# self.gx, self.gy = step_path_arrays_from_1d(i, out)
# flat = self.gy = self.shm.unstruct_view(fields)
# self.gy = self.shm.ustruct(fields)
@ -635,17 +659,29 @@ class Flow(msgspec.Struct): # , frozen=True):
self.shm._first.value
)
il = max(iflat - 1, 0)
# check for shm prepend updates since last read.
if iflat_first != ishm_first:
# write newly prepended data to flattened copy
_gx, self.gy[
ishm_first:iflat_first
] = step_path_arrays_from_1d(
self.shm.array['index'][:iflat_first],
self.shm.array[array_key][:iflat_first],
print(f'prepend {array_key}')
i_prepend = self.shm._array['index'][ishm_first:iflat_first]
y_prepend = self.shm._array[array_key][ishm_first:iflat_first]
y2_prepend = np.broadcast_to(
y_prepend[:, None], (y_prepend.size, 2),
)
# write newly prepended data to flattened copy
self.gy[ishm_first:iflat_first] = y2_prepend
# ] = step_path_arrays_from_1d(
# ] = step_path_arrays_from_1d(
# i_prepend,
# y_prepend,
# )
self._iflat_first = ishm_first
# # flat = self.gy = self.shm.unstruct_view(fields)
# self.gy = self.shm.ustruct(fields)
# # self._iflat_last = self.shm._last.value
@ -654,40 +690,112 @@ class Flow(msgspec.Struct): # , frozen=True):
# # do an update for the most recent prepend
# # index
# iflat = ishm_first
if iflat != ishm_last:
_x, to_update = step_path_arrays_from_1d(
self.shm._array[iflat:ishm_last]['index'],
self.shm._array[iflat:ishm_last][array_key],
append_diff = ishm_last - iflat
# if iflat != ishm_last:
if append_diff:
# slice up to the last datum since last index/append update
new_x = self.shm._array[il:ishm_last]['index']#.copy()
new_y = self.shm._array[il:ishm_last][array_key]#.copy()
# _x, to_update = step_path_arrays_from_1d(new_x, new_y)
# new_x2 = = np.broadcast_to(
# new_x2[:, None],
# (new_x2.size, 2),
# ) + np.array([-0.5, 0.5])
new_y2 = np.broadcast_to(
new_y[:, None], (new_y.size, 2),
)
# new_y2 = np.empty((len(new_y), 2), dtype=new_y.dtype)
# new_y2[:] = new_y[:, np.newaxis]
# import pdbpp
# pdbpp.set_trace()
# print(
# f'updating step curve {to_update}\n'
# f'last array val: {new_x}, {new_y}'
# )
# to_update = rfn.structured_to_unstructured(
# self.shm._array[iflat:ishm_last][fields]
# )
# if not to_update.any():
# if new_y.any() and not to_update.any():
# import pdbpp
# pdbpp.set_trace()
self.gy[iflat:ishm_last-1] = to_update
self.gy[-1] = 0
print(f'updating step curve {to_update}')
# print(f'{array_key} new values new_x:{new_x}, new_y:{new_y}')
# head, last = to_update[:-1], to_update[-1]
self.gy[il:ishm_last] = new_y2
gy = self.gy[il:ishm_last]
# self.gy[-1] = to_update[-1]
profiler('updated step curve data')
# slice out up-to-last step contents
x_step = self.gx[ishm_first:ishm_last]
x = x_step.reshape(-1)
y_step = self.gy[ishm_first:ishm_last]
y = y_step.reshape(-1)
profiler('sliced step data')
# print(
# f'append size: {append_diff}\n'
# f'new_x: {new_x}\n'
# f'new_y: {new_y}\n'
# f'new_y2: {new_y2}\n'
# f'new gy: {gy}\n'
# )
# update local last-index tracking
self._iflat_last = ishm_last
# (
# iflat_first,
# iflat,
# ishm_last,
# ishm_first,
# ) = (
# self._iflat_first,
# self._iflat_last,
# self.shm._last.value,
# self.shm._first.value
# )
# graphics.draw_last(last['index'], last[array_key])
# slice out up-to-last step contents
x_step = self.gx[ishm_first:ishm_last+2]
# x_step[-1] = last['index']
# x_step[-1] = last['index']
# to 1d
x = x_step.reshape(-1)
y_step = self.gy[ishm_first:ishm_last+2]
lasts = self.shm.array[['index', array_key]]
last = lasts[array_key][-1]
y_step[-1] = last
# to 1d
y = y_step.reshape(-1)
# y[-1] = 0
# s = 6
# print(f'lasts: {x[-2*s:]}, {y[-2*s:]}')
profiler('sliced step data')
# reshape to 1d for graphics rendering
# y = y_flat.reshape(-1)
# x = x_flat.reshape(-1)
# do all the same for only in-view data
y_iv = y_step[ivl:ivr].reshape(-1)
x_iv = x_step[ivl:ivr].reshape(-1)
ys_iv = y_step[ivl:ivr+1]
xs_iv = x_step[ivl:ivr+1]
y_iv = ys_iv.reshape(ys_iv.size)
x_iv = xs_iv.reshape(xs_iv.size)
# print(
# f'ys_iv : {ys_iv[-s:]}\n'
# f'y_iv: {y_iv[-s:]}\n'
# f'xs_iv: {xs_iv[-s:]}\n'
# f'x_iv: {x_iv[-s:]}\n'
# )
# y_iv = y_iv_flat.reshape(-1)
# x_iv = x_iv_flat.reshape(-1)
profiler('flattened ustruct in-view OHLC data')
@ -696,7 +804,49 @@ class Flow(msgspec.Struct): # , frozen=True):
# x, y = ohlc_flatten(array)
# x_iv, y_iv = ohlc_flatten(in_view)
# profiler('flattened OHLC data')
graphics.reset_cache()
x_last = array['index'][-1]
y_last = array[array_key][-1]
graphics._last_line = QLineF(
x_last - 0.5, 0,
x_last + 0.5, 0,
# x_last, 0,
# x_last, 0,
)
graphics._last_step_rect = QRectF(
x_last - 0.5, 0,
x_last + 0.5, y_last,
# x_last, 0,
# x_last, y_last
)
# graphics.update()
graphics.update_from_array(
x=x,
y=y,
x_iv=x_iv,
y_iv=y_iv,
view_range=(ivl, ivr) if use_vr else None,
draw_last=False,
slice_to_head=-2,
should_redraw=bool(append_diff),
# do_append=False,
**kwargs
)
# graphics.reset_cache()
# print(
# f"path br: {graphics.path.boundingRect()}\n",
# # f"fast path br: {graphics.fast_path.boundingRect()}",
# f"last rect br: {graphics._last_step_rect}\n",
# f"full br: {graphics._br}\n",
# )
# graphics.boundingRect()
else:
x = array['index']
@ -704,6 +854,9 @@ class Flow(msgspec.Struct): # , frozen=True):
x_iv = in_view['index']
y_iv = in_view[array_key]
# graphics.draw_last(x, y)
profiler('draw last segment {array_key}')
graphics.update_from_array(
x=x,
y=y,