Draw bars using `QPainterPath` magic

This gives a massive speedup when viewing large bar sets (think a day's
worth of 5s bars) by using the `pg.functions.arrayToQPath()` "magic"
binary array writing that is also used in `PlotCurveItem`.  We're using
this same (lower level) function directly to draw bars as part of one
large path and it seems to be painting 15k (ish) bars with around 3ms
`.paint()` latency. The only thing still a bit slow is the path array
generation despite doing it with `numba`. Likely, either having multiple
paths or, only regenerating the missing backing array elements should
speed this up further to avoid slight delays when incrementing the bar
step.

This is of course a first draft and more cleanups are coming.
vwap_backup
Tyler Goodlet 2020-11-23 23:32:55 -05:00
parent 4fe8c4487a
commit bcce0b5df3
1 changed files with 306 additions and 63 deletions

View File

@ -18,16 +18,16 @@
Chart graphics for displaying a slew of different data types. Chart graphics for displaying a slew of different data types.
""" """
# import time import time
from typing import List, Optional, Tuple from typing import List, Optional, Tuple
import numpy as np import numpy as np
import pyqtgraph as pg import pyqtgraph as pg
# from numba import jit, float64, optional, int64 from numba import jit, float64, optional, int64
from PyQt5 import QtCore, QtGui from PyQt5 import QtCore, QtGui
from PyQt5.QtCore import QLineF, QPointF from PyQt5.QtCore import QLineF, QPointF
# from .._profile import timeit from .._profile import timeit
from ._style import ( from ._style import (
_xaxis_at, _xaxis_at,
hcolor, hcolor,
@ -44,6 +44,8 @@ _debounce_delay = 1 / 2e3
_ch_label_opac = 1 _ch_label_opac = 1
# TODO: we need to handle the case where index is outside
# the underlying datums range
class LineDot(pg.CurvePoint): class LineDot(pg.CurvePoint):
def __init__( def __init__(
@ -149,6 +151,7 @@ class ContentsLabel(pg.LabelItem):
index: int, index: int,
array: np.ndarray, array: np.ndarray,
) -> None: ) -> None:
if index < len(array):
data = array[index][name] data = array[index][name]
self.setText(f"{name}: {data:.2f}") self.setText(f"{name}: {data:.2f}")
@ -246,7 +249,7 @@ class CrossHair(pg.GraphicsObject):
) -> LineDot: ) -> LineDot:
# if this plot contains curves add line dot "cursors" to denote # if this plot contains curves add line dot "cursors" to denote
# the current sample under the mouse # the current sample under the mouse
cursor = LineDot(curve, index=len(plot._array)) cursor = LineDot(curve, index=len(plot._ohlc))
plot.addItem(cursor) plot.addItem(cursor)
self.graphics[plot].setdefault('cursors', []).append(cursor) self.graphics[plot].setdefault('cursors', []).append(cursor)
return cursor return cursor
@ -341,18 +344,45 @@ class CrossHair(pg.GraphicsObject):
# nopython=True, # nopython=True,
# nogil=True # nogil=True
# ) # )
def _mk_lines_array(data: List, size: int) -> np.ndarray: def _mk_lines_array(
"""Create an ndarray to hold lines graphics objects. data: List,
size: int,
elements_step: int = 6,
) -> np.ndarray:
"""Create an ndarray to hold lines graphics info.
""" """
return np.zeros_like( return np.zeros_like(
data, data,
shape=(int(size), 3), shape=(int(size), elements_step),
dtype=object, dtype=object,
) )
# TODO: `numba` this? def lines_from_ohlc(row: np.ndarray, w: float) -> Tuple[QLineF]:
open, high, low, close, index = row[
['open', 'high', 'low', 'close', 'index']]
# high -> low vertical (body) line
if low != high:
hl = QLineF(index, low, index, high)
else:
# XXX: if we don't do it renders a weird rectangle?
# see below for filtering this later...
hl = None
# NOTE: place the x-coord start as "middle" of the drawing range such
# that the open arm line-graphic is at the left-most-side of
# the index's range according to the view mapping.
# open line
o = QLineF(index - w, open, index, open)
# close line
c = QLineF(index, close, index + w, close)
return [hl, o, c]
# TODO: `numba` this?
# @jit( # @jit(
# # float64[:]( # # float64[:](
# # float64[:], # # float64[:],
@ -370,7 +400,7 @@ def bars_from_ohlc(
"""Generate an array of lines objects from input ohlc data. """Generate an array of lines objects from input ohlc data.
""" """
lines = _mk_lines_array(data, data.shape[0]) lines = _mk_lines_array(data, data.shape[0], 3)
for i, q in enumerate(data[start:], start=start): for i, q in enumerate(data[start:], start=start):
open, high, low, close, index = q[ open, high, low, close, index = q[
@ -424,6 +454,94 @@ def bars_from_ohlc(
return lines return lines
# @timeit
@jit(
# float64[:](
# float64[:],
# optional(float64),
# optional(int64)
# ),
nopython=True,
nogil=True
)
def path_arrays_from_ohlc(
data: np.ndarray,
w: float64,
start: int64 = int64(0),
) -> np.ndarray:
"""Generate an array of lines objects from input ohlc data.
"""
size = int(data.shape[0] * 6)
x = np.zeros(
# data,
shape=size,
dtype=float64,
)
y = np.zeros(
# data,
shape=size,
dtype=float64,
)
c = np.zeros(
# data,
shape=size,
dtype=float64,
)
# TODO: report bug for assert
# @ /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
# for i, q in enumerate(data[start:], start):
for i, q in enumerate(data[start:], start):
# TODO: ask numba why this doesn't work..
# open, high, low, close, index = q[
# ['open', 'high', 'low', 'close', 'index']]
open = q['open']
high = q['high']
low = q['low']
close = q['close']
index = float64(q['index'])
istart = i * 6
istop = istart + 6
# write points for x, y, and connections
x[istart:istop] = (
index - w,
index,
index,
index,
index,
index + w,
)
y[istart:istop] = (
open,
open,
low,
high,
close,
close,
)
c[istart:istop] = (0, 1, 1, 1, 1, 1)
return x, y, c
@timeit
def gen_qpath(
data,
w,
start,
) -> QtGui.QPainterPath:
x, y, c = path_arrays_from_ohlc(data, w, start=start)
return pg.functions.arrayToQPath(x, y, connect=c)
class BarItems(pg.GraphicsObject): class BarItems(pg.GraphicsObject):
"""Price range bars graphics rendered from a OHLC sequence. """Price range bars graphics rendered from a OHLC sequence.
""" """
@ -431,6 +549,9 @@ class BarItems(pg.GraphicsObject):
# 0.5 is no overlap between arms, 1.0 is full overlap # 0.5 is no overlap between arms, 1.0 is full overlap
w: float = 0.43 w: float = 0.43
# XXX: for the mega-lulz increasing width here increases draw latency...
# so probably don't do it until we figure that out.
bars_pen = pg.mkPen(hcolor('bracket')) bars_pen = pg.mkPen(hcolor('bracket'))
# XXX: tina mode, see below # XXX: tina mode, see below
@ -443,7 +564,8 @@ class BarItems(pg.GraphicsObject):
plotitem: 'pg.PlotItem', # noqa plotitem: 'pg.PlotItem', # noqa
) -> None: ) -> None:
super().__init__() super().__init__()
self.last = QtGui.QPicture()
self.last_bar = QtGui.QPicture()
self.history = QtGui.QPicture() self.history = QtGui.QPicture()
# TODO: implement updateable pixmap solution # TODO: implement updateable pixmap solution
self._pi = plotitem self._pi = plotitem
@ -456,7 +578,11 @@ class BarItems(pg.GraphicsObject):
# XXX: not sure this actually needs to be an array other # XXX: not sure this actually needs to be an array other
# then for the old tina mode calcs for up/down bars below? # then for the old tina mode calcs for up/down bars below?
# lines container # lines container
self.lines = _mk_lines_array([], 50e3) day_in_s = 60 * 60 * 12
self.lines = _mk_lines_array([], 50e3, 6)
# TODO: don't render the full backing array each time
# self._path_data = None
self._last_bar_lines = None
# track the current length of drawable lines within the larger array # track the current length of drawable lines within the larger array
self.index: int = 0 self.index: int = 0
@ -471,67 +597,136 @@ class BarItems(pg.GraphicsObject):
This routine is usually only called to draw the initial history. This routine is usually only called to draw the initial history.
""" """
lines = bars_from_ohlc(data, self.w, start=start) # start_lines = time.time()
# lines = bars_from_ohlc(data, self.w, start=start)
# start_path = time.time()
# assert len(data) == 2000
self.path = gen_qpath(data, self.w, start=start)
# end = time.time()
# print(f"paths took {end - start_path}\n lines took {start_path - start_lines}")
# save graphics for later reference and keep track # save graphics for later reference and keep track
# of current internal "last index" # of current internal "last index"
index = len(lines) # index = len(lines)
self.lines[:index] = lines # index = len(data)
self.index = index # self.lines[:index] = lines
# lines = bars_from_ohlc(data[-1:], self.w, start=start)
self.index = len(data)
# up to last to avoid double draw of last bar # up to last to avoid double draw of last bar
self.draw_lines(just_history=True, iend=self.index - 1) # self.draw_lines(just_history=True, iend=self.index - 1, path=self.path)
self.draw_lines(iend=self.index)
# @timeit # self.draw_lines(iend=self.index)
self._last_bar_lines = lines_from_ohlc(data[-1], self.w)
# create pics
self.draw_history()
self.draw_last_bar()
# trigger render
# https://doc.qt.io/qt-5/qgraphicsitem.html#update
self.update()
def draw_last_bar(self) -> None:
# pic = self.last_bar
# pre-computing a QPicture object allows paint() to run much
# more quickly, rather than re-drawing the shapes every time.
p = QtGui.QPainter(self.last_bar)
p.setPen(self.bars_pen)
# print(self._last_bar_lines)
p.drawLines(*tuple(filter(bool, self._last_bar_lines)))
p.end()
# trigger re-render
# https://doc.qt.io/qt-5/qgraphicsitem.html#update
# self.update()
def draw_history(self) -> None:
p = QtGui.QPainter(self.history)
p.setPen(self.bars_pen)
p.drawPath(self.path)
p.end()
# self.update()
@timeit
def draw_lines( def draw_lines(
self, self,
istart=0,
iend=None, iend=None,
just_history=False, just_history=False,
istart=0,
path: QtGui.QPainterPath = None,
# TODO: could get even fancier and only update the single close line? # TODO: could get even fancier and only update the single close line?
lines=None, lines=None,
) -> None: ) -> None:
"""Draw the current line set using the painter. """Draw the current line set using the painter.
Currently this draws lines to a cached ``QPicture`` which
is supposed to speed things up on ``.paint()`` calls (which
is a call to ``QPainter.drawPicture()`` but I'm not so sure.
""" """
if just_history: # if path is None:
# draw bars for the "history" picture # if just_history:
iend = iend or self.index - 1 # raise RuntimeError
pic = self.history # # draw bars for the "history" picture
else: # iend = iend or self.index - 1
# draw the last bar # pic = self.history
istart = self.index - 1 # else:
iend = iend or self.index # # draw the last bar
pic = self.last # istart = self.index - 1
# iend = iend or self.index
# use 2d array of lines objects, see conlusion on speed: # pic = self.last_bar
# https://stackoverflow.com/a/60089929
flat = np.ravel(self.lines[istart:iend])
# TODO: do this with numba for speed gain: # # use 2d array of lines objects, see conlusion on speed:
# https://stackoverflow.com/questions/58422690/filtering-a-numpy-array-what-is-the-best-approach # # https://stackoverflow.com/a/60089929
to_draw = flat[np.where(flat != None)] # noqa # flat = np.ravel(self.lines[istart:iend])
# # TODO: do this with numba for speed gain:
# # https://stackoverflow.com/questions/58422690/filtering-a-numpy-array-what-is-the-best-approach
# to_draw = flat[np.where(flat != None)] # noqa
# else:
# pic = self.history
pic = self.last_bar
# pre-computing a QPicture object allows paint() to run much # pre-computing a QPicture object allows paint() to run much
# more quickly, rather than re-drawing the shapes every time. # more quickly, rather than re-drawing the shapes every time.
p = QtGui.QPainter(pic) p = QtGui.QPainter(pic)
p.setPen(self.bars_pen) p.setPen(self.bars_pen)
# TODO: is there any way to not have to pass all the lines every p.drawLines(*self._last_bar_lines)
# iteration? It seems they won't draw unless it's done this way..
p.drawLines(*to_draw)
p.end() p.end()
# XXX: if we ever try using `QPixmap` again...
# if self._pmi is None:
# self._pmi = self.scene().addPixmap(self.picture)
# else:
# self._pmi.setPixmap(self.picture)
# trigger re-render # trigger re-render
# https://doc.qt.io/qt-5/qgraphicsitem.html#update # https://doc.qt.io/qt-5/qgraphicsitem.html#update
self.update() self.update()
# TODO: is there any way to not have to pass all the lines every
# iteration? It seems they won't draw unless it's done this way..
# if path is None:
# # p.drawLines(*to_draw)
# p.drawLines(*self._last_bars_lines)
# else:
# p.drawPath(path)
# p.end()
# trigger re-render
# https://doc.qt.io/qt-5/qgraphicsitem.html#update
# self.update()
def update_from_array( def update_from_array(
self, self,
array: np.ndarray, array: np.ndarray,
@ -552,25 +747,51 @@ class BarItems(pg.GraphicsObject):
# start_bar_to_update = index - 100 # start_bar_to_update = index - 100
# TODO: allow mapping only a range of lines thus
# only drawing as many bars as exactly specified.
if extra > 0: if extra > 0:
# generate new graphics to match provided array # generate new graphics to match provided array
# lines = bars_from_ohlc(new, self.w)
# lines = bars_from_ohlc(array[-1:], self.w)
self._last_bar_lines = lines_from_ohlc(array[-1], self.w)
# TODO: only draw these new bars to the backing binary
# path array and then call arrayToQpath() on the whole
# -> will avoid multiple passes for path data we've already
# already generated
new = array[index:index + extra] new = array[index:index + extra]
lines = bars_from_ohlc(new, self.w)
bars_added = len(lines) self.path = gen_qpath(array[:-1], self.w, start=0)
self.lines[index:index + bars_added] = lines
self.index += bars_added # self.path.connectPath(path)
# bars_added = len(new)
# bars_added = extra
# self.lines[index:index + bars_added] = lines
self.index += extra
# start_bar_to_update = index - bars_added # start_bar_to_update = index - bars_added
self.draw_lines(just_history=True) # self.draw_lines(just_history=True, path=self.path)
# self.update()
self.draw_history()
if just_history: if just_history:
self.update()
return return
# current bar update # last bar update
i, o, h, l, last, v = array[-1][ i, o, h, l, last, v = array[-1][
['index', 'open', 'high', 'low', 'close', 'volume'] ['index', 'open', 'high', 'low', 'close', 'volume']
] ]
assert i == self.index - 1 # assert i == self.index - 1
body, larm, rarm = self.lines[i]
# body, larm, rarm = self.lines[i]
# body, larm, rarm = self._bars
body, larm, rarm = self._last_bar_lines
# XXX: is there a faster way to modify this? # XXX: is there a faster way to modify this?
rarm.setLine(rarm.x1(), last, rarm.x2(), last) rarm.setLine(rarm.x1(), last, rarm.x2(), last)
@ -579,18 +800,30 @@ class BarItems(pg.GraphicsObject):
if l != h: # noqa if l != h: # noqa
if body is None: if body is None:
body = self.lines[index - 1][0] = QLineF(i, l, i, h) # body = self.lines[index - 1][0] = QLineF(i, l, i, h)
body = self._last_bar_lines[0] = QLineF(i, l, i, h)
else: else:
# update body # update body
body.setLine(i, l, i, h) body.setLine(i, l, i, h)
else:
# XXX: h == l -> remove any HL line to avoid render bug
if body is not None:
body = self.lines[index - 1][0] = None
self.draw_lines(just_history=False) # XXX: pretty sure this is causing an issue where the bar has
# a large upward move right before the next sample and the body
# is getting set to None since the next bar is flat but the shm
# array index update wasn't read by the time this code runs. Iow
# we're doing this removal of the body for a bar index that is
# now out of date / from some previous sample. It's weird
# though because i've seen it do this to bars i - 3 back?
# @timeit # else:
# # XXX: h == l -> remove any HL line to avoid render bug
# if body is not None:
# body = self.lines[index - 1][0] = None
# self.draw_lines(just_history=False)
self.draw_last_bar()
self.update()
@timeit
def paint(self, p, opt, widget): def paint(self, p, opt, widget):
# profiler = pg.debug.Profiler(disabled=False, delayed=False) # profiler = pg.debug.Profiler(disabled=False, delayed=False)
@ -606,8 +839,17 @@ class BarItems(pg.GraphicsObject):
# as is necesarry for what's in "view". Not sure if this will # as is necesarry for what's in "view". Not sure if this will
# lead to any perf gains other then when zoomed in to less bars # lead to any perf gains other then when zoomed in to less bars
# in view. # in view.
p.drawPicture(0, 0, self.history) # p.drawPicture(0, 0, self.history)
p.drawPicture(0, 0, self.last) p.drawPicture(0, 0, self.last_bar)
# p = QtGui.QPainter(pic)
p.setPen(self.bars_pen)
# p.drawLines(*self._last_bar_lines)
# TODO: is there any way to not have to pass all the lines every
# iteration? It seems they won't draw unless it's done this way..
p.drawPath(self.path)
# TODO: if we can ever make pixmaps work... # TODO: if we can ever make pixmaps work...
# p.drawPixmap(0, 0, self.picture) # p.drawPixmap(0, 0, self.picture)
@ -616,6 +858,7 @@ class BarItems(pg.GraphicsObject):
# profiler('bars redraw:') # profiler('bars redraw:')
# @timeit
def boundingRect(self): def boundingRect(self):
# TODO: can we do rect caching to make this faster? # TODO: can we do rect caching to make this faster?
@ -626,7 +869,7 @@ class BarItems(pg.GraphicsObject):
# bounding rect for us). # bounding rect for us).
# compute aggregate bounding rectangle # compute aggregate bounding rectangle
lb = self.last.boundingRect() lb = self.last_bar.boundingRect()
hb = self.history.boundingRect() hb = self.history.boundingRect()
return QtCore.QRectF( return QtCore.QRectF(
# top left # top left