Make `BarItems` use our line curve for downsampling

Drop all the logic originally in `.update_ds_line()` which is now done
internal to our `FastAppendCurve`. Add incremental update of the
flattened OHLC -> line curve (unfortunately using `np.concatenate()` for
the moment) and maintain a new `._ds_line_xy` arrays tuple which keeps
the internal state. Add `.maybe_downsample()` as per the new interaction
update method requirement. Draft out some fast path curve stuff like in
our line graphic. Short-circuit bars path updates when we downsample to
line. Oh, and add a ton more profiling in prep for getting
all this stuff faf.
big_data_lines
Tyler Goodlet 2022-04-01 13:28:50 -04:00
parent 6410c68e2e
commit 5128e4c304
1 changed files with 152 additions and 197 deletions

View File

@ -31,13 +31,11 @@ from PyQt5.QtCore import QLineF, QPointF
# from numba import types as ntypes # from numba import types as ntypes
# from ..data._source import numba_ohlc_dtype # from ..data._source import numba_ohlc_dtype
from .._profile import pg_profile_enabled from .._profile import pg_profile_enabled, ms_slower_then
from ._style import hcolor from ._style import hcolor
from ..log import get_logger from ..log import get_logger
from ._curve import FastAppendCurve from ._curve import FastAppendCurve
from ._compression import ( from ._compression import ohlc_flatten
ohlc_to_m4_line,
)
if TYPE_CHECKING: if TYPE_CHECKING:
from ._chart import LinkedSplits from ._chart import LinkedSplits
@ -46,29 +44,16 @@ if TYPE_CHECKING:
log = get_logger(__name__) log = get_logger(__name__)
def _mk_lines_array( def bar_from_ohlc_row(
data: list,
size: int,
elements_step: int = 6,
) -> np.ndarray:
'''
Create an ndarray to hold lines graphics info.
'''
return np.zeros_like(
data,
shape=(int(size), elements_step),
dtype=object,
)
def lines_from_ohlc(
row: np.ndarray, row: np.ndarray,
w: float w: float
) -> tuple[QLineF]: ) -> tuple[QLineF]:
'''
Generate the minimal ``QLineF`` lines to construct a single
OHLC "bar" for use in the "last datum" of a series.
'''
open, high, low, close, index = row[ open, high, low, close, index = row[
['open', 'high', 'low', 'close', 'index']] ['open', 'high', 'low', 'close', 'index']]
@ -178,7 +163,11 @@ def gen_qpath(
) -> QtGui.QPainterPath: ) -> QtGui.QPainterPath:
profiler = pg.debug.Profiler(disabled=not pg_profile_enabled()) profiler = pg.debug.Profiler(
msg=f'gen_qpath ohlc',
disabled=not pg_profile_enabled(),
gt=ms_slower_then,
)
x, y, c = path_arrays_from_ohlc(data, start, bar_gap=w) x, y, c = path_arrays_from_ohlc(data, start, bar_gap=w)
profiler("generate stream with numba") profiler("generate stream with numba")
@ -202,7 +191,6 @@ class BarItems(pg.GraphicsObject):
def __init__( def __init__(
self, self,
# scene: 'QGraphicsScene', # noqa
linked: LinkedSplits, linked: LinkedSplits,
plotitem: 'pg.PlotItem', # noqa plotitem: 'pg.PlotItem', # noqa
pen_color: str = 'bracket', pen_color: str = 'bracket',
@ -211,15 +199,17 @@ class BarItems(pg.GraphicsObject):
name: Optional[str] = None, name: Optional[str] = None,
) -> None: ) -> None:
self.linked = linked
super().__init__() super().__init__()
self.linked = linked
# XXX: for the mega-lulz increasing width here increases draw # XXX: for the mega-lulz increasing width here increases draw
# latency... so probably don't do it until we figure that out. # latency... so probably don't do it until we figure that out.
self._color = pen_color self._color = pen_color
self.bars_pen = pg.mkPen(hcolor(pen_color), width=1) self.bars_pen = pg.mkPen(hcolor(pen_color), width=1)
self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2) self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2)
self._array = None
self._ds_line_xy: Optional[
tuple[np.ndarray, np.ndarray]
] = None
# NOTE: this prevents redraws on mouse interaction which is # NOTE: this prevents redraws on mouse interaction which is
# a huge boon for avg interaction latency. # a huge boon for avg interaction latency.
@ -232,6 +222,7 @@ class BarItems(pg.GraphicsObject):
self._pi = plotitem self._pi = plotitem
self.path = QtGui.QPainterPath() self.path = QtGui.QPainterPath()
self.fast_path = QtGui.QPainterPath()
self._xrange: tuple[int, int] self._xrange: tuple[int, int]
self._yrange: tuple[float, float] self._yrange: tuple[float, float]
@ -246,7 +237,6 @@ class BarItems(pg.GraphicsObject):
# downsampler-line state # downsampler-line state
self._in_ds: bool = False self._in_ds: bool = False
self._ds_lines: dict[int, FastAppendCurve] = {}
self._ds_line: Optional[FastAppendCurve] = None self._ds_line: Optional[FastAppendCurve] = None
self._dsi: tuple[int, int] = 0, 0 self._dsi: tuple[int, int] = 0, 0
self._xs_in_px: float = 0 self._xs_in_px: float = 0
@ -278,96 +268,29 @@ class BarItems(pg.GraphicsObject):
) )
# up to last to avoid double draw of last bar # up to last to avoid double draw of last bar
self._last_bar_lines = lines_from_ohlc(last, self.w) self._last_bar_lines = bar_from_ohlc_row(last, self.w)
# trigger render # trigger render
# https://doc.qt.io/qt-5/qgraphicsitem.html#update # https://doc.qt.io/qt-5/qgraphicsitem.html#update
self.update() self.update()
# self.update_ds_line(ohlc) x, y = self._ds_line_xy = ohlc_flatten(ohlc)
# assert self._ds_line self.update_ds_line(x, y)
# self._ds_line.hide() self._ds_xrange = (index[0], index[-1])
self._array = ohlc
return self.path return self.path
def get_ds_line(
self,
ds: Optional[int] = None,
) -> tuple[FastAppendCurve, int]:
px_vecs = self.pixelVectors()[0]
if not px_vecs and self._ds_line:
px_vecs = self._ds_line.pixelVectors()[0]
if px_vecs:
xs_in_px = px_vecs.x()
ds = round(xs_in_px)
else:
ds = 0
return self._ds_line, ds
# return self._ds_line.get(ds), ds
def update_ds_line( def update_ds_line(
self, self,
ohlc: np.ndarray, x,
y,
) -> int: ) -> FastAppendCurve:
# determine current potential downsampling value (based on pixel # determine current potential downsampling value (based on pixel
# scaling) and return any existing curve for it. # scaling) and return any existing curve for it.
curve, uppx = self.get_ds_line() curve = self._ds_line
# print(f'uppx: {uppx}')
chart = self.linked.chart
if not chart:
return
else:
px_width = round(chart.curve_width_pxs())
if px_width == 0:
return
# if self._ds_line:
# self._pi.removeItem(self._ds_line)
# log.info(f'current dsi: {self._dsi}')
old_dsi = ds_uppx, ds_px_width = self._dsi
changed = False
if (
abs(uppx - ds_uppx) >= 4
# or not self._in_ds
):
changed = True
if curve:
# trigger a full redraw of the curve path since
# we have downsampled another "level" using m4.
curve.clear()
ds_uppx, ds_px_width = dsi = (uppx, px_width)
self._dsi = dsi
if changed:
log.info(f'sampler change: {old_dsi} -> {dsi}')
# always refresh data bounds until we get diffing
# working properly, see above..
x, y = ohlc_to_m4_line(
ohlc,
px_width=ds_px_width,
uppx=ds_uppx,
# pretrace=True,
# activate m4 ds?
downsample=True,
)
if not curve: if not curve:
# TODO: figuring out the most optimial size for the ideal # TODO: figuring out the most optimial size for the ideal
# curve-path by, # curve-path by,
# - calcing the display's max px width `.screen()` # - calcing the display's max px width `.screen()`
@ -381,16 +304,13 @@ class BarItems(pg.GraphicsObject):
curve = FastAppendCurve( curve = FastAppendCurve(
y=y, y=y,
x=x, x=x,
name='ds', name='OHLC',
color=self._color, color=self._color,
# color='dad_blue',
# use_polyline=True, # pretty sure this is slower?
) )
curve.hide() curve.hide()
self._pi.addItem(curve) self._pi.addItem(curve)
self._ds_lines[ds_uppx] = curve
self._ds_line = curve self._ds_line = curve
return curve, ds_uppx return curve
# TODO: we should be diffing the amount of new data which # TODO: we should be diffing the amount of new data which
# needs to be downsampled. Ideally we actually are just # needs to be downsampled. Ideally we actually are just
@ -403,7 +323,7 @@ class BarItems(pg.GraphicsObject):
y=y, y=y,
x=x, x=x,
) )
return curve, ds_uppx return curve
def update_from_array( def update_from_array(
self, self,
@ -424,14 +344,14 @@ class BarItems(pg.GraphicsObject):
This routine should be made (transitively) as fast as possible. This routine should be made (transitively) as fast as possible.
''' '''
# XXX: always do this? profiler = pg.debug.Profiler(
# if self._ds_line: disabled=not pg_profile_enabled(),
# del self._array gt=ms_slower_then,
self._array = ohlc )
self.update_ds_line(ohlc)
# index = self.start_index # index = self.start_index
istart, istop = self._xrange istart, istop = self._xrange
ds_istart, ds_istop = self._ds_xrange
index = ohlc['index'] index = ohlc['index']
first_index, last_index = index[0], index[-1] first_index, last_index = index[0], index[-1]
@ -440,52 +360,110 @@ class BarItems(pg.GraphicsObject):
prepend_length = istart - first_index prepend_length = istart - first_index
append_length = last_index - istop append_length = last_index - istop
ds_prepend_length = ds_istart - first_index
ds_append_length = last_index - ds_istop
flip_cache = False flip_cache = False
# TODO: allow mapping only a range of lines thus # TODO: to make the downsampling faster
# only drawing as many bars as exactly specified. # - allow mapping only a range of lines thus only drawing as
# many bars as exactly specified.
# - move ohlc "flattening" to a shmarr
# - maybe move all this embedded logic to a higher
# level type?
fx, fy = self._ds_line_xy
if prepend_length: if prepend_length:
# new history was added and we need to render a new path # new history was added and we need to render a new path
new_bars = ohlc[:prepend_length] prepend_bars = ohlc[:prepend_length]
prepend_path = gen_qpath(new_bars, 0, self.w)
# XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path if ds_prepend_length:
# y value not matching the first value from ds_prepend_bars = ohlc[:ds_prepend_length]
# ohlc[prepend_length + 1] ??? pre_x, pre_y = ohlc_flatten(ds_prepend_bars)
fx = np.concatenate((pre_x, fx))
# update path fy = np.concatenate((pre_y, fy))
old_path = self.path profiler('ds line prepend diff complete')
self.path = prepend_path
self.path.addPath(old_path)
# trigger redraw despite caching
self.prepareGeometryChange()
if append_length: if append_length:
# generate new lines objects for updatable "current bar"
self._last_bar_lines = lines_from_ohlc(ohlc[-1], self.w)
# generate new graphics to match provided array # generate new graphics to match provided array
# path appending logic: # path appending logic:
# we need to get the previous "current bar(s)" for the time step # we need to get the previous "current bar(s)" for the time step
# and convert it to a sub-path to append to the historical set # and convert it to a sub-path to append to the historical set
# new_bars = ohlc[istop - 1:istop + append_length - 1] # new_bars = ohlc[istop - 1:istop + append_length - 1]
new_bars = ohlc[-append_length - 1:-1] append_bars = ohlc[-append_length - 1:-1]
append_path = gen_qpath(new_bars, 0, self.w) # print(f'ohlc bars to append size: {append_bars.size}\n')
self.path.moveTo(float(istop - self.w), float(new_bars[0]['open']))
if ds_append_length:
ds_append_bars = ohlc[-ds_append_length - 1:-1]
post_x, post_y = ohlc_flatten(ds_append_bars)
# print(f'ds curve to append sizes: {(post_x.size, post_y.size)}')
fx = np.concatenate((fx, post_x))
fy = np.concatenate((fy, post_y))
profiler('ds line append diff complete')
profiler('array diffs complete')
# does this work?
last = ohlc[-1]
fy[-1] = last['close']
# incremental update and cache line datums
self._ds_line_xy = fx, fy
# maybe downsample to line
ds = self.maybe_downsample()
if ds:
# if we downsample to a line don't bother with
# any more path generation / updates
self._ds_xrange = first_index, last_index
profiler('downsampled to line')
return
# path updates
if prepend_length:
# XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path
# y value not matching the first value from
# ohlc[prepend_length + 1] ???
prepend_path = gen_qpath(prepend_bars, 0, self.w)
old_path = self.path
self.path = prepend_path
self.path.addPath(old_path)
profiler('path PREPEND')
if append_length:
append_path = gen_qpath(append_bars, 0, self.w)
self.path.moveTo(
float(istop - self.w),
float(append_bars[0]['open'])
)
self.path.addPath(append_path) self.path.addPath(append_path)
# trigger redraw despite caching profiler('path APPEND')
self.prepareGeometryChange() # fp = self.fast_path
self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) # if fp is None:
flip_cache = True # self.fast_path = append_path
# else:
# fp.moveTo(float(istop - self.w), float(new_bars[0]['open']))
# fp.addPath(append_path)
# self.setCacheMode(QtWidgets.QGraphicsItem.NoCache)
# flip_cache = True
self._xrange = first_index, last_index self._xrange = first_index, last_index
# trigger redraw despite caching
self.prepareGeometryChange()
# generate new lines objects for updatable "current bar"
self._last_bar_lines = bar_from_ohlc_row(last, self.w)
# last bar update # last bar update
i, o, h, l, last, v = ohlc[-1][ i, o, h, l, last, v = last[
['index', 'open', 'high', 'low', 'close', 'volume'] ['index', 'open', 'high', 'low', 'close', 'volume']
] ]
# assert i == self.start_index - 1 # assert i == self.start_index - 1
@ -514,7 +492,10 @@ class BarItems(pg.GraphicsObject):
# now out of date / from some previous sample. It's weird # now out of date / from some previous sample. It's weird
# though because i've seen it do this to bars i - 3 back? # though because i've seen it do this to bars i - 3 back?
profiler('last bar set')
self.update() self.update()
profiler('.update()')
if flip_cache: if flip_cache:
self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
@ -536,7 +517,20 @@ class BarItems(pg.GraphicsObject):
# apparently this a lot faster says the docs? # apparently this a lot faster says the docs?
# https://doc.qt.io/qt-5/qpainterpath.html#controlPointRect # https://doc.qt.io/qt-5/qpainterpath.html#controlPointRect
hb = self.path.controlPointRect() hb = self.path.controlPointRect()
hb_tl, hb_br = hb.topLeft(), hb.bottomRight() hb_tl, hb_br = (
hb.topLeft(),
hb.bottomRight(),
)
# fp = self.fast_path
# if fp:
# fhb = fp.controlPointRect()
# print((hb_tl, hb_br))
# print(fhb)
# hb_tl, hb_br = (
# fhb.topLeft() + hb.topLeft(),
# fhb.bottomRight() + hb.bottomRight(),
# )
# need to include last bar height or BR will be off # need to include last bar height or BR will be off
mx_y = hb_br.y() mx_y = hb_br.y()
@ -565,7 +559,7 @@ class BarItems(pg.GraphicsObject):
) )
def maybe_paint_line( def maybe_downsample(
self, self,
x_gt: float = 2., x_gt: float = 2.,
@ -583,54 +577,35 @@ class BarItems(pg.GraphicsObject):
# this is the ``float`` value of the "number of x units" (in # this is the ``float`` value of the "number of x units" (in
# view coords) that a pixel spans. # view coords) that a pixel spans.
xvec = self.pixelVectors()[0] xs_in_px = self._ds_line.x_uppx()
if xvec:
xs_in_px = xvec.x()
else:
xs_in_px = self._ds_line.pixelVectors()[0].x()
linked = self.linked linked = self.linked
if ( if (
# xs_in_px != self._xs_in_px self._ds_line_xy is not None
self._array is not None
): ):
# print('refreshing curve') curve = self.update_ds_line(
out = self.update_ds_line(self._array) *self._ds_line_xy,
if not out: )
print("NOTHING!?")
return
curve, ds = out
if ( if (
not self._in_ds not self._in_ds
and xs_in_px >= x_gt and xs_in_px >= x_gt
): ):
# TODO: a `.ui()` log level? # TODO: a `.ui()` log level?
log.info(f'downsampling to line graphic {linked.symbol.key}') log.info(
f'downsampling to line graphic {linked.symbol.key}'
)
self.hide() self.hide()
# XXX: is this actually any faster? # XXX: is this actually any faster?
# self._pi.removeItem(self) # self._pi.removeItem(self)
curve, ds = out
self._xs_in_px = xs_in_px self._xs_in_px = xs_in_px
# curve, ds = self.get_ds_line(ds=0)
curve.clear()
curve.update()
curve, out = self.update_ds_line(self._array)
# curve = self._ds_line
# assert last_curve is curve
# self._pi.addItem(curve) # self._pi.addItem(curve)
curve.show() curve.show()
curve.update()
linked.graphics_cycle()
self._in_ds = True self._in_ds = True
return True
elif ( elif (
self._in_ds self._in_ds
@ -638,10 +613,7 @@ class BarItems(pg.GraphicsObject):
): ):
log.info(f'showing bars graphic {linked.symbol.key}') log.info(f'showing bars graphic {linked.symbol.key}')
# curve, ds = self.get_ds_line()
curve = self._ds_line curve = self._ds_line
# assert last_curve is curve
curve.clear()
curve.hide() curve.hide()
# self._pi.removeItem(curve) # self._pi.removeItem(curve)
@ -651,31 +623,9 @@ class BarItems(pg.GraphicsObject):
self.update() self.update()
self._in_ds = False self._in_ds = False
linked.graphics_cycle()
return True
# elif (
# self._in_ds
# and self._dsi != ds
# ):
# # curve = self._ds_lines.get(ds)
# # assert self._ds_line is not curve
# if self._ds_line and self._ds_line is not curve:
# self._ds_line.hide()
# if curve:
# # self._pi.removeItem(curve)
# curve.show()
# curve.update()
# self._ds_line = curve
# self._dsi = ds
# linked.graphics_cycle()
# return True
# no curve change # no curve change
return False return self._in_ds
def paint( def paint(
self, self,
@ -690,7 +640,7 @@ class BarItems(pg.GraphicsObject):
profiler = pg.debug.Profiler( profiler = pg.debug.Profiler(
disabled=not pg_profile_enabled(), disabled=not pg_profile_enabled(),
delayed=False, gt=ms_slower_then,
) )
# p.setCompositionMode(0) # p.setCompositionMode(0)
@ -708,6 +658,11 @@ class BarItems(pg.GraphicsObject):
p.setPen(self.bars_pen) p.setPen(self.bars_pen)
p.drawPath(self.path) p.drawPath(self.path)
profiler('draw history path') profiler('draw history path')
# if self.fast_path:
# p.drawPath(self.fast_path)
# profiler('draw fast path')
profiler.finish() profiler.finish()
# NOTE: for testing paint frequency as throttled by display loop. # NOTE: for testing paint frequency as throttled by display loop.