Add graphics incr-updated "formatter" subsys
After trying to hack epoch indexed time series and failing miserably, decided to properly factor out all formatting routines into a common subsystem API: ``IncrementalFormatter`` which provides the interface for incrementally updating and tracking pre-path-graphics formatted data. Previously this functionality was mangled into our `Renderer` (which also does the work of `QPath` generation and update) but splitting it out also preps for being able to do graphics-buffer downsampling and caching on a remote host B) The ``IncrementalFormatter`` (parent type) has the default behaviour of tracking a single field-array on some source `ShmArray`, updating a flattened `numpy.ndarray` in-mem allocation, and providing a default 1d conversion for pre-downsampling and path generation. Changed out of `Renderer`, - `.allocate_xy()`, `update_xy()` and `format_xy()` all are moved to more explicitly named formatter methods. - all `.x/y_data` nd array management and update - "last view range" tracking - `.last_read`, `.diff()` - now calls `IncrementalFormatter.format_to_1d()` inside `.render()` The new API gets, - `.diff()`, `.last_read` - all view range diff tracking through `.track_inview_range()`. - better nd format array names: `.x/y_nd`, `xy_nd_start/stop`. - `.format_to_1d()` which renders pre-path formatted arrays ready for both m4 sampling and path gen. - better explicit overloadable formatting method names: * `.allocate_xy()` -> `.allocate_xy_nd()` * `.update_xy()` -> `.incr_update_xy_nd()` * `.format_xy()` -> `.format_xy_nd_to_1d()` Finally this implements per-graphics-type formatters which define each set up related formatting routines: - `OHLCBarsFmtr`: std multi-line style bars - `OHLCBarsAsCurveFmtr`: draws an interpolated line for ohlc sampled data - `StepCurveFmtr`: handles vlm style curvesmultichartz_backup
Tyler Goodlet
parent
1a594fd219
commit
7c4d3e7f3b
|
|
@ -25,8 +25,6 @@ incremental update.
|
|||
from __future__ import annotations
|
||||
from typing import (
|
||||
Optional,
|
||||
Callable,
|
||||
Union,
|
||||
)
|
||||
|
||||
import msgspec
|
||||
|
|
@ -43,21 +41,10 @@ from .._profile import (
|
|||
# ms_slower_then,
|
||||
)
|
||||
from ._pathops import (
|
||||
by_index_and_key,
|
||||
|
||||
# Plain OHLC renderer
|
||||
gen_ohlc_qpath,
|
||||
|
||||
# OHLC -> line renderer
|
||||
ohlc_to_line,
|
||||
update_ohlc_to_line,
|
||||
ohlc_flat_to_xy,
|
||||
|
||||
# step curve renderer
|
||||
to_step_format,
|
||||
update_step_xy,
|
||||
step_to_xy,
|
||||
|
||||
IncrementalFormatter,
|
||||
OHLCBarsFmtr, # Plain OHLC renderer
|
||||
OHLCBarsAsCurveFmtr, # OHLC converted to line
|
||||
StepCurveFmtr, # "step" curve (like for vlm)
|
||||
xy_downsample,
|
||||
)
|
||||
from ._ohlc import (
|
||||
|
|
@ -76,16 +63,6 @@ from .._profile import Profiler
|
|||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# class FlowsTable(msgspec.Struct):
|
||||
# '''
|
||||
# Data-AGGRegate: high level API onto multiple (categorized)
|
||||
# ``Flow``s with high level processing routines for
|
||||
# multi-graphics computations and display.
|
||||
|
||||
# '''
|
||||
# flows: dict[str, np.ndarray] = {}
|
||||
|
||||
|
||||
def render_baritems(
|
||||
flow: Flow,
|
||||
graphics: BarItems,
|
||||
|
|
@ -117,21 +94,24 @@ def render_baritems(
|
|||
r = self._src_r
|
||||
if not r:
|
||||
show_bars = True
|
||||
|
||||
# OHLC bars path renderer
|
||||
r = self._src_r = Renderer(
|
||||
flow=self,
|
||||
format_xy=gen_ohlc_qpath,
|
||||
last_read=read,
|
||||
fmtr=OHLCBarsFmtr(
|
||||
shm=flow.shm,
|
||||
flow=flow,
|
||||
_last_read=read,
|
||||
),
|
||||
)
|
||||
|
||||
ds_curve_r = Renderer(
|
||||
flow=self,
|
||||
last_read=read,
|
||||
|
||||
# incr update routines
|
||||
allocate_xy=ohlc_to_line,
|
||||
update_xy=update_ohlc_to_line,
|
||||
format_xy=ohlc_flat_to_xy,
|
||||
fmtr=OHLCBarsAsCurveFmtr(
|
||||
shm=flow.shm,
|
||||
flow=flow,
|
||||
_last_read=read,
|
||||
),
|
||||
)
|
||||
|
||||
curve = FlattenedOHLC(
|
||||
|
|
@ -228,7 +208,7 @@ class Flow(msgspec.Struct): # , frozen=True):
|
|||
'''
|
||||
name: str
|
||||
plot: pg.PlotItem
|
||||
graphics: Union[Curve, BarItems]
|
||||
graphics: Curve | BarItems
|
||||
_shm: ShmArray
|
||||
yrange: tuple[float, float] = None
|
||||
|
||||
|
|
@ -237,7 +217,6 @@ class Flow(msgspec.Struct): # , frozen=True):
|
|||
# normally this is just a plain line.
|
||||
ds_graphics: Optional[Curve] = None
|
||||
|
||||
|
||||
is_ohlc: bool = False
|
||||
render: bool = True # toggle for display loop
|
||||
|
||||
|
|
@ -445,9 +424,14 @@ class Flow(msgspec.Struct): # , frozen=True):
|
|||
|
||||
slice_to_head: int = -1
|
||||
should_redraw: bool = False
|
||||
should_line: bool = False
|
||||
rkwargs = {}
|
||||
|
||||
should_line = False
|
||||
# TODO: probably specialize ``Renderer`` types instead of
|
||||
# these logic checks?
|
||||
# - put these blocks into a `.load_renderer()` meth?
|
||||
# - consider a OHLCRenderer, StepCurveRenderer, Renderer?
|
||||
r = self._src_r
|
||||
if isinstance(graphics, BarItems):
|
||||
# XXX: special case where we change out graphics
|
||||
# to a line after a certain uppx threshold.
|
||||
|
|
@ -467,16 +451,36 @@ class Flow(msgspec.Struct): # , frozen=True):
|
|||
should_redraw = changed_to_line or not should_line
|
||||
self._in_ds = should_line
|
||||
|
||||
else:
|
||||
r = self._src_r
|
||||
if not r:
|
||||
# just using for ``.diff()`` atm..
|
||||
elif not r:
|
||||
if isinstance(graphics, StepCurve):
|
||||
|
||||
r = self._src_r = Renderer(
|
||||
flow=self,
|
||||
# TODO: rename this to something with ohlc
|
||||
last_read=read,
|
||||
fmtr=StepCurveFmtr(
|
||||
shm=self.shm,
|
||||
flow=self,
|
||||
_last_read=read,
|
||||
),
|
||||
)
|
||||
|
||||
# TODO: append logic inside ``.render()`` isn't
|
||||
# correct yet for step curves.. remove this to see it.
|
||||
should_redraw = True
|
||||
slice_to_head = -2
|
||||
|
||||
else:
|
||||
r = self._src_r
|
||||
if not r:
|
||||
# just using for ``.diff()`` atm..
|
||||
r = self._src_r = Renderer(
|
||||
flow=self,
|
||||
fmtr=IncrementalFormatter(
|
||||
shm=self.shm,
|
||||
flow=self,
|
||||
_last_read=read,
|
||||
),
|
||||
)
|
||||
|
||||
# ``Curve`` derivative case(s):
|
||||
array_key = array_key or self.name
|
||||
# print(array_key)
|
||||
|
|
@ -486,19 +490,6 @@ class Flow(msgspec.Struct): # , frozen=True):
|
|||
should_ds: bool = r._in_ds
|
||||
showing_src_data: bool = not r._in_ds
|
||||
|
||||
# step_mode = getattr(graphics, '_step_mode', False)
|
||||
step_mode = isinstance(graphics, StepCurve)
|
||||
if step_mode:
|
||||
|
||||
r.allocate_xy = to_step_format
|
||||
r.update_xy = update_step_xy
|
||||
r.format_xy = step_to_xy
|
||||
|
||||
# TODO: append logic inside ``.render()`` isn't
|
||||
# correct yet for step curves.. remove this to see it.
|
||||
should_redraw = True
|
||||
slice_to_head = -2
|
||||
|
||||
# downsampling incremental state checking
|
||||
# check for and set std m4 downsample conditions
|
||||
uppx = graphics.x_uppx()
|
||||
|
|
@ -680,34 +671,7 @@ class Flow(msgspec.Struct): # , frozen=True):
|
|||
class Renderer(msgspec.Struct):
|
||||
|
||||
flow: Flow
|
||||
# last array view read
|
||||
last_read: Optional[tuple] = None
|
||||
|
||||
# default just returns index, and named array from data
|
||||
format_xy: Callable[
|
||||
[np.ndarray, str],
|
||||
tuple[np.ndarray]
|
||||
] = by_index_and_key
|
||||
|
||||
# optional pre-graphics xy formatted data which
|
||||
# is incrementally updated in sync with the source data.
|
||||
allocate_xy: Optional[Callable[
|
||||
[int, slice],
|
||||
tuple[np.ndarray, np.nd.array]
|
||||
]] = None
|
||||
|
||||
update_xy: Optional[Callable[
|
||||
[int, slice], None]
|
||||
] = None
|
||||
|
||||
x_data: Optional[np.ndarray] = None
|
||||
y_data: Optional[np.ndarray] = None
|
||||
|
||||
# indexes which slice into the above arrays (which are allocated
|
||||
# based on source data shm input size) and allow retrieving
|
||||
# incrementally updated data.
|
||||
_xy_first: int = 0
|
||||
_xy_last: int = 0
|
||||
fmtr: IncrementalFormatter
|
||||
|
||||
# output graphics rendering, the main object
|
||||
# processed in ``QGraphicsObject.paint()``
|
||||
|
|
@ -729,58 +693,11 @@ class Renderer(msgspec.Struct):
|
|||
_last_uppx: float = 0
|
||||
_in_ds: bool = False
|
||||
|
||||
# incremental update state(s)
|
||||
_last_vr: Optional[tuple[float, float]] = None
|
||||
_last_ivr: Optional[tuple[float, float]] = None
|
||||
|
||||
def diff(
|
||||
self,
|
||||
new_read: tuple[np.ndarray],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
(
|
||||
last_xfirst,
|
||||
last_xlast,
|
||||
last_array,
|
||||
last_ivl,
|
||||
last_ivr,
|
||||
last_in_view,
|
||||
) = self.last_read
|
||||
|
||||
# TODO: can the renderer just call ``Flow.read()`` directly?
|
||||
# unpack latest source data read
|
||||
(
|
||||
xfirst,
|
||||
xlast,
|
||||
array,
|
||||
ivl,
|
||||
ivr,
|
||||
in_view,
|
||||
) = new_read
|
||||
|
||||
# compute the length diffs between the first/last index entry in
|
||||
# the input data and the last indexes we have on record from the
|
||||
# last time we updated the curve index.
|
||||
prepend_length = int(last_xfirst - xfirst)
|
||||
append_length = int(xlast - last_xlast)
|
||||
|
||||
# blah blah blah
|
||||
# do diffing for prepend, append and last entry
|
||||
return (
|
||||
slice(xfirst, last_xfirst),
|
||||
prepend_length,
|
||||
append_length,
|
||||
slice(last_xlast, xlast),
|
||||
)
|
||||
|
||||
def draw_path(
|
||||
self,
|
||||
x: np.ndarray,
|
||||
y: np.ndarray,
|
||||
connect: Union[str, np.ndarray] = 'all',
|
||||
connect: str | np.ndarray = 'all',
|
||||
path: Optional[QPainterPath] = None,
|
||||
redraw: bool = False,
|
||||
|
||||
|
|
@ -858,166 +775,50 @@ class Renderer(msgspec.Struct):
|
|||
'''
|
||||
# TODO: can the renderer just call ``Flow.read()`` directly?
|
||||
# unpack latest source data read
|
||||
fmtr = self.fmtr
|
||||
|
||||
(
|
||||
xfirst,
|
||||
xlast,
|
||||
_,
|
||||
_,
|
||||
array,
|
||||
ivl,
|
||||
ivr,
|
||||
in_view,
|
||||
) = new_read
|
||||
|
||||
(
|
||||
pre_slice,
|
||||
prepend_length,
|
||||
append_length,
|
||||
post_slice,
|
||||
) = self.diff(new_read)
|
||||
|
||||
if self.update_xy:
|
||||
|
||||
shm = self.flow.shm
|
||||
|
||||
if self.y_data is None:
|
||||
# we first need to allocate xy data arrays
|
||||
# from the source data.
|
||||
assert self.allocate_xy
|
||||
self.x_data, self.y_data = self.allocate_xy(
|
||||
shm,
|
||||
array_key,
|
||||
)
|
||||
self._xy_first = shm._first.value
|
||||
self._xy_last = shm._last.value
|
||||
profiler('allocated xy history')
|
||||
|
||||
if prepend_length:
|
||||
y_prepend = shm._array[pre_slice]
|
||||
|
||||
if read_from_key:
|
||||
y_prepend = y_prepend[array_key]
|
||||
|
||||
xy_data, xy_slice = self.update_xy(
|
||||
shm,
|
||||
array_key,
|
||||
|
||||
# this is the pre-sliced, "normally expected"
|
||||
# new data that an updater would normally be
|
||||
# expected to process, however in some cases (like
|
||||
# step curves) the updater routine may want to do
|
||||
# the source history-data reading itself, so we pass
|
||||
# both here.
|
||||
y_prepend,
|
||||
|
||||
pre_slice,
|
||||
prepend_length,
|
||||
self._xy_first,
|
||||
self._xy_last,
|
||||
is_append=False,
|
||||
)
|
||||
self.y_data[xy_slice] = xy_data
|
||||
self._xy_first = shm._first.value
|
||||
profiler('prepended xy history: {prepend_length}')
|
||||
|
||||
if append_length:
|
||||
y_append = shm._array[post_slice]
|
||||
|
||||
if read_from_key:
|
||||
y_append = y_append[array_key]
|
||||
|
||||
xy_data, xy_slice = self.update_xy(
|
||||
shm,
|
||||
array_key,
|
||||
|
||||
y_append,
|
||||
post_slice,
|
||||
append_length,
|
||||
|
||||
self._xy_first,
|
||||
self._xy_last,
|
||||
is_append=True,
|
||||
)
|
||||
# self.y_data[post_slice] = xy_data
|
||||
# self.y_data[xy_slice or post_slice] = xy_data
|
||||
self.y_data[xy_slice] = xy_data
|
||||
self._xy_last = shm._last.value
|
||||
profiler('appened xy history: {append_length}')
|
||||
|
||||
if use_vr:
|
||||
array = in_view
|
||||
# else:
|
||||
# ivl, ivr = xfirst, xlast
|
||||
|
||||
hist = array[:slice_to_head]
|
||||
|
||||
# xy-path data transform: convert source data to a format
|
||||
# able to be passed to a `QPainterPath` rendering routine.
|
||||
if not len(hist):
|
||||
fmt_out = fmtr.format_to_1d(
|
||||
new_read,
|
||||
array_key,
|
||||
profiler,
|
||||
|
||||
slice_to_head=slice_to_head,
|
||||
read_src_from_key=read_from_key,
|
||||
slice_to_inview=use_vr,
|
||||
)
|
||||
|
||||
# no history in view case
|
||||
if not fmt_out:
|
||||
# XXX: this might be why the profiler only has exits?
|
||||
return
|
||||
|
||||
x_out, y_out, connect = self.format_xy(
|
||||
self,
|
||||
# TODO: hist here should be the pre-sliced
|
||||
# x/y_data in the case where allocate_xy is
|
||||
# defined?
|
||||
hist,
|
||||
array_key,
|
||||
(ivl, ivr),
|
||||
)
|
||||
(
|
||||
x_1d,
|
||||
y_1d,
|
||||
connect,
|
||||
prepend_length,
|
||||
append_length,
|
||||
view_changed,
|
||||
|
||||
profiler('sliced input arrays')
|
||||
|
||||
if (
|
||||
use_vr
|
||||
):
|
||||
# if a view range is passed, plan to draw the
|
||||
# source ouput that's "in view" of the chart.
|
||||
view_range = (ivl, ivr)
|
||||
# print(f'{self._name} vr: {view_range}')
|
||||
|
||||
profiler(f'view range slice {view_range}')
|
||||
|
||||
vl, vr = view_range
|
||||
|
||||
zoom_or_append = False
|
||||
last_vr = self._last_vr
|
||||
last_ivr = self._last_ivr or vl, vr
|
||||
|
||||
# incremental in-view data update.
|
||||
if last_vr:
|
||||
# relative slice indices
|
||||
lvl, lvr = last_vr
|
||||
# abs slice indices
|
||||
al, ar = last_ivr
|
||||
|
||||
# left_change = abs(x_iv[0] - al) >= 1
|
||||
# right_change = abs(x_iv[-1] - ar) >= 1
|
||||
|
||||
if (
|
||||
# likely a zoom view change
|
||||
(vr - lvr) > 2 or vl < lvl
|
||||
# append / prepend update
|
||||
# we had an append update where the view range
|
||||
# didn't change but the data-viewed (shifted)
|
||||
# underneath, so we need to redraw.
|
||||
# or left_change and right_change and last_vr == view_range
|
||||
|
||||
# not (left_change and right_change) and ivr
|
||||
# (
|
||||
# or abs(x_iv[ivr] - livr) > 1
|
||||
):
|
||||
zoom_or_append = True
|
||||
|
||||
self._last_vr = view_range
|
||||
if len(x_out):
|
||||
self._last_ivr = x_out[0], x_out[slice_to_head]
|
||||
) = fmt_out
|
||||
|
||||
# redraw conditions
|
||||
if (
|
||||
prepend_length > 0
|
||||
or new_sample_rate
|
||||
or append_length > 0
|
||||
or zoom_or_append
|
||||
or view_changed
|
||||
):
|
||||
should_redraw = True
|
||||
|
||||
|
|
@ -1039,9 +840,9 @@ class Renderer(msgspec.Struct):
|
|||
|
||||
elif should_ds and uppx > 1:
|
||||
|
||||
x_out, y_out, ymn, ymx = xy_downsample(
|
||||
x_out,
|
||||
y_out,
|
||||
x_1d, y_1d, ymn, ymx = xy_downsample(
|
||||
x_1d,
|
||||
y_1d,
|
||||
uppx,
|
||||
)
|
||||
self.flow.yrange = ymn, ymx
|
||||
|
|
@ -1052,8 +853,8 @@ class Renderer(msgspec.Struct):
|
|||
self._in_ds = True
|
||||
|
||||
path = self.draw_path(
|
||||
x=x_out,
|
||||
y=y_out,
|
||||
x=x_1d,
|
||||
y=y_1d,
|
||||
connect=connect,
|
||||
path=path,
|
||||
redraw=True,
|
||||
|
|
@ -1088,8 +889,8 @@ class Renderer(msgspec.Struct):
|
|||
and not should_redraw
|
||||
):
|
||||
print(f'{array_key} append len: {append_length}')
|
||||
new_x = x_out[-append_length - 2:] # slice_to_head]
|
||||
new_y = y_out[-append_length - 2:] # slice_to_head]
|
||||
new_x = x_1d[-append_length - 2:] # slice_to_head]
|
||||
new_y = y_1d[-append_length - 2:] # slice_to_head]
|
||||
profiler('sliced append path')
|
||||
|
||||
profiler(
|
||||
|
|
@ -1137,10 +938,4 @@ class Renderer(msgspec.Struct):
|
|||
self.path = path
|
||||
self.fast_path = fast_path
|
||||
|
||||
# TODO: eventually maybe we can implement some kind of
|
||||
# transform on the ``QPainterPath`` that will more or less
|
||||
# detect the diff in "elements" terms?
|
||||
# update diff state since we've now rendered paths.
|
||||
self.last_read = new_read
|
||||
|
||||
return self.path, array, reset
|
||||
|
|
|
|||
1036
piker/ui/_pathops.py
1036
piker/ui/_pathops.py
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue