pikers
/
piker
3
0
Fork 0
Code Issues 2 Pull Requests 5 Packages Projects Releases Wiki Activity

WIP incrementally update step array format

incremental_update_paths
Tyler Goodlet 2022-04-24 12:33:25 -04:00
parent b97ec38baf
commit b2b31b8f84
2 changed files with 215 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
piker/ui/_curve.py
View File

@ -58,36 +58,55 @@ def step_path_arrays_from_1d(
''' '''
y_out = y.copy() y_out = y.copy()
x_out = x.copy() x_out = x.copy()
x2 = np.empty(
# the data + 2 endpoints on either end for # x2 = np.empty(
# "termination of the path". # # the data + 2 endpoints on either end for
(len(x) + 1, 2), # # "termination of the path".
# we want to align with OHLC or other sampling style # (len(x) + 1, 2),
# bars likely so we need fractinal values # # we want to align with OHLC or other sampling style
dtype=float, # # bars likely so we need fractinal values
) # dtype=float,
x2[0] = x[0] - 0.5 # )
x2[1] = x[0] + 0.5
x2[1:] = x[:, np.newaxis] + 0.5 x2 = np.broadcast_to(
x[:, None],
(
x_out.size,
# 4, # only ohlc
2,
),
) + np.array([-0.5, 0.5])
# x2[0] = x[0] - 0.5
# x2[1] = x[0] + 0.5
# x2[0, 0] = x[0] - 0.5
# x2[0, 1] = x[0] + 0.5
# x2[1:] = x[:, np.newaxis] + 0.5
# import pdbpp
# pdbpp.set_trace()
# flatten to 1-d # flatten to 1-d
x_out = x2.reshape(x2.size) # x_out = x2.reshape(x2.size)
x_out = x2
# we create a 1d with 2 extra indexes to # we create a 1d with 2 extra indexes to
# hold the start and (current) end value for the steps # hold the start and (current) end value for the steps
# on either end # on either end
y2 = np.empty((len(y), 2), dtype=y.dtype) y2 = np.empty((len(y), 2), dtype=y.dtype)
y2[:] = y[:, np.newaxis] y2[:] = y[:, np.newaxis]
y2[-1] = 0
y_out = np.empty( y_out = y2
2*len(y) + 2,
dtype=y.dtype # y_out = np.empty(
) # 2*len(y) + 2,
# dtype=y.dtype
# )
# flatten and set 0 endpoints # flatten and set 0 endpoints
y_out[1:-1] = y2.reshape(y2.size) # y_out[1:-1] = y2.reshape(y2.size)
y_out[0] = 0 # y_out[0] = 0
y_out[-1] = 0 # y_out[-1] = 0
if not include_endpoints: if not include_endpoints:
return x_out[:-1], y_out[:-1] return x_out[:-1], y_out[:-1]
@ -414,16 +433,16 @@ class FastAppendCurve(pg.GraphicsObject):
# step mode: draw flat top discrete "step" # step mode: draw flat top discrete "step"
# over the index space for each datum. # over the index space for each datum.
if self._step_mode: # if self._step_mode:
x_out, y_out = step_path_arrays_from_1d( # x_out, y_out = step_path_arrays_from_1d(
x_out, # x_out,
y_out, # y_out,
) # )
# self.disable_cache() # # self.disable_cache()
# flip_cache = True # # flip_cache = True
# TODO: numba this bish # # TODO: numba this bish
profiler('generated step arrays') # profiler('generated step arrays')
if should_redraw: if should_redraw:
if self.path: if self.path:
@ -501,27 +520,26 @@ class FastAppendCurve(pg.GraphicsObject):
new_y = y[-append_length - 2:-1] new_y = y[-append_length - 2:-1]
profiler('sliced append path') profiler('sliced append path')
if self._step_mode: # if self._step_mode:
new_x, new_y = step_path_arrays_from_1d( # new_x, new_y = step_path_arrays_from_1d(
new_x, # new_x,
new_y, # new_y,
) # )
# [1:] since we don't need the vertical line normally at # # [1:] since we don't need the vertical line normally at
# the beginning of the step curve taking the first (x, # # the beginning of the step curve taking the first (x,
# y) poing down to the x-axis **because** this is an # # y) poing down to the x-axis **because** this is an
# appended path graphic. # # appended path graphic.
new_x = new_x[1:] # new_x = new_x[1:]
new_y = new_y[1:] # new_y = new_y[1:]
# self.disable_cache() # # self.disable_cache()
# flip_cache = True # # flip_cache = True
profiler('generated step data') # profiler('generated step data')
else: profiler(
profiler( f'diffed array input, append_length={append_length}'
f'diffed array input, append_length={append_length}' )
)
# if should_ds: # if should_ds:
# new_x, new_y = self.downsample( # new_x, new_y = self.downsample(
@ -655,6 +673,10 @@ class FastAppendCurve(pg.GraphicsObject):
# self.disable_cache() # self.disable_cache()
# self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
def reset_cache(self) -> None:
self.disable_cache()
self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
def disable_cache(self) -> None: def disable_cache(self) -> None:
''' '''
Disable the use of the pixel coordinate cache and trigger a geo event. Disable the use of the pixel coordinate cache and trigger a geo event.

170
piker/ui/_flows.py
View File

@ -46,6 +46,7 @@ from ._ohlc import (
) )
from ._curve import ( from ._curve import (
FastAppendCurve, FastAppendCurve,
step_path_arrays_from_1d,
) )
from ._compression import ( from ._compression import (
# ohlc_flatten, # ohlc_flatten,
@ -149,8 +150,8 @@ class Flow(msgspec.Struct): # , frozen=True):
is_ohlc: bool = False is_ohlc: bool = False
render: bool = True # toggle for display loop render: bool = True # toggle for display loop
flat: Optional[ShmArray] = None gy: Optional[ShmArray] = None
x_basis: Optional[np.ndarray] = None gx: Optional[np.ndarray] = None
_iflat_last: int = 0 _iflat_last: int = 0
_iflat_first: int = 0 _iflat_first: int = 0
@ -360,7 +361,7 @@ class Flow(msgspec.Struct): # , frozen=True):
flow=self, flow=self,
# just swap in the flat view # just swap in the flat view
# data_t=lambda array: self.flat.array, # data_t=lambda array: self.gy.array,
last_read=read, last_read=read,
draw_path=partial( draw_path=partial(
rowarr_to_path, rowarr_to_path,
@ -413,37 +414,37 @@ class Flow(msgspec.Struct): # , frozen=True):
if should_line: if should_line:
fields = ['open', 'high', 'low', 'close'] fields = ['open', 'high', 'low', 'close']
if self.flat is None: if self.gy is None:
# create a flattened view onto the OHLC array # create a flattened view onto the OHLC array
# which can be read as a line-style format # which can be read as a line-style format
shm = self.shm shm = self.shm
# flat = self.flat = self.shm.unstruct_view(fields) # flat = self.gy = self.shm.unstruct_view(fields)
self.flat = self.shm.ustruct(fields) self.gy = self.shm.ustruct(fields)
first = self._iflat_first = self.shm._first.value first = self._iflat_first = self.shm._first.value
last = self._iflat_last = self.shm._last.value last = self._iflat_last = self.shm._last.value
# write pushed data to flattened copy # write pushed data to flattened copy
self.flat[first:last] = rfn.structured_to_unstructured( self.gy[first:last] = rfn.structured_to_unstructured(
self.shm.array[fields] self.shm.array[fields]
) )
# generate an flat-interpolated x-domain # generate an flat-interpolated x-domain
self.x_basis = ( self.gx = (
np.broadcast_to( np.broadcast_to(
shm._array['index'][:, None], shm._array['index'][:, None],
( (
shm._array.size, shm._array.size,
# 4, # only ohlc # 4, # only ohlc
self.flat.shape[1], self.gy.shape[1],
), ),
) + np.array([-0.5, 0, 0, 0.5]) ) + np.array([-0.5, 0, 0, 0.5])
) )
assert self.flat.any() assert self.gy.any()
# print(f'unstruct diff: {time.time() - start}') # print(f'unstruct diff: {time.time() - start}')
# profiler('read unstr view bars to line') # profiler('read unstr view bars to line')
# start = self.flat._first.value # start = self.gy._first.value
# update flatted ohlc copy # update flatted ohlc copy
( (
iflat_first, iflat_first,
@ -461,15 +462,15 @@ class Flow(msgspec.Struct): # , frozen=True):
if iflat_first != ishm_first: if iflat_first != ishm_first:
# write newly prepended data to flattened copy # write newly prepended data to flattened copy
self.flat[ self.gy[
ishm_first:iflat_first ishm_first:iflat_first
] = rfn.structured_to_unstructured( ] = rfn.structured_to_unstructured(
self.shm.array[fields][:iflat_first] self.shm.array[fields][:iflat_first]
) )
self._iflat_first = ishm_first self._iflat_first = ishm_first
# # flat = self.flat = self.shm.unstruct_view(fields) # # flat = self.gy = self.shm.unstruct_view(fields)
# self.flat = self.shm.ustruct(fields) # self.gy = self.shm.ustruct(fields)
# # self._iflat_last = self.shm._last.value # # self._iflat_last = self.shm._last.value
# # self._iflat_first = self.shm._first.value # # self._iflat_first = self.shm._first.value
@ -481,12 +482,12 @@ class Flow(msgspec.Struct): # , frozen=True):
self.shm._array[iflat:ishm_last][fields] self.shm._array[iflat:ishm_last][fields]
) )
self.flat[iflat:ishm_last][:] = to_update self.gy[iflat:ishm_last][:] = to_update
profiler('updated ustruct OHLC data') profiler('updated ustruct OHLC data')
# slice out up-to-last step contents # slice out up-to-last step contents
y_flat = self.flat[ishm_first:ishm_last] y_flat = self.gy[ishm_first:ishm_last]
x_flat = self.x_basis[ishm_first:ishm_last] x_flat = self.gx[ishm_first:ishm_last]
# update local last-index tracking # update local last-index tracking
self._iflat_last = ishm_last self._iflat_last = ishm_last
@ -577,16 +578,139 @@ class Flow(msgspec.Struct): # , frozen=True):
# graphics.draw_last(last) # graphics.draw_last(last)
else: else:
# ``FastAppendCurve`` case:
array_key = array_key or self.name array_key = array_key or self.name
graphics.update_from_array( # ``FastAppendCurve`` case:
x=array['index'], if graphics._step_mode and self.gy is None:
y=array[array_key],
# create a flattened view onto the OHLC array
# which can be read as a line-style format
shm = self.shm
# fields = ['index', array_key]
i = shm._array['index']
out = shm._array[array_key]
self.gx, self.gy = step_path_arrays_from_1d(i, out)
# flat = self.gy = self.shm.unstruct_view(fields)
# self.gy = self.shm.ustruct(fields)
# first = self._iflat_first = self.shm._first.value
# last = self._iflat_last = self.shm._last.value
# # write pushed data to flattened copy
# self.gy[first:last] = rfn.structured_to_unstructured(
# self.shm.array[fields]
# )
# # generate an flat-interpolated x-domain
# self.gx = (
# np.broadcast_to(
# shm._array['index'][:, None],
# (
# shm._array.size,
# # 4, # only ohlc
# self.gy.shape[1],
# ),
# ) + np.array([-0.5, 0, 0, 0.5])
# )
# assert self.gy.any()
# print(f'unstruct diff: {time.time() - start}')
# profiler('read unstr view bars to line')
# start = self.gy._first.value
# update flatted ohlc copy
if graphics._step_mode:
(
iflat_first,
iflat,
ishm_last,
ishm_first,
) = (
self._iflat_first,
self._iflat_last,
self.shm._last.value,
self.shm._first.value
)
# check for shm prepend updates since last read.
if iflat_first != ishm_first:
# write newly prepended data to flattened copy
_gx, self.gy[
ishm_first:iflat_first
] = step_path_arrays_from_1d(
self.shm.array['index'][:iflat_first],
self.shm.array[array_key][:iflat_first],
)
self._iflat_first = ishm_first
# # flat = self.gy = self.shm.unstruct_view(fields)
# self.gy = self.shm.ustruct(fields)
# # self._iflat_last = self.shm._last.value
# # self._iflat_first = self.shm._first.value
# # do an update for the most recent prepend
# # index
# iflat = ishm_first
if iflat != ishm_last:
_x, to_update = step_path_arrays_from_1d(
self.shm._array[iflat:ishm_last]['index'],
self.shm._array[iflat:ishm_last][array_key],
)
# to_update = rfn.structured_to_unstructured(
# self.shm._array[iflat:ishm_last][fields]
# )
# import pdbpp
# pdbpp.set_trace()
self.gy[iflat:ishm_last-1] = to_update
self.gy[-1] = 0
print(f'updating step curve {to_update}')
profiler('updated step curve data')
# slice out up-to-last step contents
x_step = self.gx[ishm_first:ishm_last]
x = x_step.reshape(-1)
y_step = self.gy[ishm_first:ishm_last]
y = y_step.reshape(-1)
profiler('sliced step data')
# update local last-index tracking
self._iflat_last = ishm_last
# reshape to 1d for graphics rendering
# y = y_flat.reshape(-1)
# x = x_flat.reshape(-1)
# do all the same for only in-view data
y_iv = y_step[ivl:ivr].reshape(-1)
x_iv = x_step[ivl:ivr].reshape(-1)
# y_iv = y_iv_flat.reshape(-1)
# x_iv = x_iv_flat.reshape(-1)
profiler('flattened ustruct in-view OHLC data')
# legacy full-recompute-everytime method
# x, y = ohlc_flatten(array)
# x_iv, y_iv = ohlc_flatten(in_view)
# profiler('flattened OHLC data')
graphics.reset_cache()
else:
x = array['index']
y = array[array_key]
x_iv = in_view['index']
y_iv = in_view[array_key]
graphics.update_from_array(
x=x,
y=y,
x_iv=x_iv,
y_iv=y_iv,
x_iv=in_view['index'],
y_iv=in_view[array_key],
view_range=(ivl, ivr) if use_vr else None, view_range=(ivl, ivr) if use_vr else None,
**kwargs **kwargs