Set a `PlotItem.viz` for interaction lookup

Inside `._interaction` routines we need access to `Viz` instances.
Instead of doing `CharPlotWidget._vizs: dict` lookups this ensures each
plot can lookup it's (parent) viz without error.

Also, adjusts `Viz.maxmin()` output parsing to new signature.

piker/_daemon.py

@@ -257,7 +257,7 @@ async def open_piker_runtime(
     # and spawn the service tree distributed per that.
     start_method: str = 'trio',
 
-    tractor_kwargs: dict = {},
+    **tractor_kwargs,
 
 ) -> tuple[
     tractor.Actor,

piker/clearing/_ems.py

@@ -46,6 +46,7 @@ from ..data._normalize import iterticks
 from ..data._source import (
     unpack_fqsn,
     mk_fqsn,
+    float_digits,
 )
 from ..data.feed import (
     Feed,
@@ -1250,6 +1251,7 @@ async def process_client_order_cmds(
 
                 spread_slap: float = 5
                 min_tick = flume.symbol.tick_size
+                min_tick_digits = float_digits(min_tick)
 
                 if action == 'buy':
                     tickfilter = ('ask', 'last', 'trade')
@@ -1258,12 +1260,18 @@ async def process_client_order_cmds(
                     # TODO: we probably need to scale this based
                     # on some near term historical spread
                     # measure?
-                    abs_diff_away = spread_slap * min_tick
+                    abs_diff_away = round(
+                        spread_slap * min_tick,
+                        ndigits=min_tick_digits,
+                    )
 
                 elif action == 'sell':
                     tickfilter = ('bid', 'last', 'trade')
                     percent_away = -0.005
-                    abs_diff_away = -spread_slap * min_tick
+                    abs_diff_away = round(
+                        -spread_slap * min_tick,
+                        ndigits=min_tick_digits,
+                    )
 
                 else:  # alert
                     tickfilter = ('trade', 'utrade', 'last')

piker/data/_formatters.py

@@ -0,0 +1,837 @@
+# piker: trading gear for hackers
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Pre-(path)-graphics formatted x/y nd/1d rendering subsystem.
+
+"""
+from __future__ import annotations
+from typing import (
+    Optional,
+    TYPE_CHECKING,
+)
+
+import msgspec
+import numpy as np
+from numpy.lib import recfunctions as rfn
+
+from ._sharedmem import (
+    ShmArray,
+)
+from ._pathops import (
+    path_arrays_from_ohlc,
+)
+
+if TYPE_CHECKING:
+    from ._dataviz import (
+        Viz,
+    )
+    from .._profile import Profiler
+
+
+class IncrementalFormatter(msgspec.Struct):
+    '''
+    Incrementally updating, pre-path-graphics tracking, formatter.
+
+    Allows tracking source data state in an updateable pre-graphics
+    ``np.ndarray`` format (in local process memory) as well as
+    incrementally rendering from that format **to** 1d x/y for path
+    generation using ``pg.functions.arrayToQPath()``.
+
+    '''
+    shm: ShmArray
+    viz: Viz
+
+    @property
+    def index_field(self) -> 'str':
+        '''
+        Value (``str``) used to look up the "index series" from the
+        underlying source ``numpy`` struct-array; delegate directly to
+        the managing ``Viz``.
+
+        '''
+        return self.viz.index_field
+
+    # Incrementally updated xy ndarray formatted data, a pre-1d
+    # format which is updated and cached independently of the final
+    # pre-graphics-path 1d format.
+    x_nd: Optional[np.ndarray] = None
+    y_nd: Optional[np.ndarray] = None
+
+    @property
+    def xy_nd(self) -> tuple[np.ndarray, np.ndarray]:
+        return (
+            self.x_nd[self.xy_slice],
+            self.y_nd[self.xy_slice],
+        )
+
+    @property
+    def xy_slice(self) -> slice:
+        return slice(
+            self.xy_nd_start,
+            self.xy_nd_stop,
+        )
+
+    # indexes which slice into the above arrays (which are allocated
+    # based on source data shm input size) and allow retrieving
+    # incrementally updated data.
+    xy_nd_start: int | None = None
+    xy_nd_stop: int | None = None
+
+    # TODO: eventually incrementally update 1d-pre-graphics path data?
+    # x_1d: Optional[np.ndarray] = None
+    # y_1d: Optional[np.ndarray] = None
+
+    # incremental view-change state(s) tracking
+    _last_vr: tuple[float, float] | None = None
+    _last_ivdr: tuple[float, float] | None = None
+
+    @property
+    def index_step_size(self) -> float:
+        '''
+        Readonly value computed on first ``.diff()`` call.
+
+        '''
+        return self.viz.index_step()
+
+    def __repr__(self) -> str:
+        msg = (
+            f'{type(self)}: ->\n\n'
+            f'fqsn={self.viz.name}\n'
+            f'shm_name={self.shm.token["shm_name"]}\n\n'
+
+            f'last_vr={self._last_vr}\n'
+            f'last_ivdr={self._last_ivdr}\n\n'
+
+            f'xy_slice={self.xy_slice}\n'
+            # f'xy_nd_stop={self.xy_nd_stop}\n\n'
+        )
+
+        x_nd_len = 0
+        y_nd_len = 0
+        if self.x_nd is not None:
+            x_nd_len = len(self.x_nd)
+            y_nd_len = len(self.y_nd)
+
+        msg += (
+            f'x_nd_len={x_nd_len}\n'
+            f'y_nd_len={y_nd_len}\n'
+        )
+
+        return msg
+
+    def diff(
+        self,
+        new_read: tuple[np.ndarray],
+
+    ) -> tuple[
+        np.ndarray,
+        np.ndarray,
+    ]:
+        # TODO:
+        # - can the renderer just call ``Viz.read()`` directly? unpack
+        #   latest source data read
+        # - eventually maybe we can implement some kind of
+        #   transform on the ``QPainterPath`` that will more or less
+        #   detect the diff in "elements" terms? update diff state since
+        #   we've now rendered paths.
+        (
+            xfirst,
+            xlast,
+            array,
+            ivl,
+            ivr,
+            in_view,
+        ) = new_read
+
+        index = array['index']
+
+        # if the first index in the read array is 0 then
+        # it means the source buffer has bee completely backfilled to
+        # available space.
+        src_start = index[0]
+        src_stop = index[-1] + 1
+
+        # these are the "formatted output data" indices
+        # for the pre-graphics arrays.
+        nd_start = self.xy_nd_start
+        nd_stop = self.xy_nd_stop
+
+        if (
+            nd_start is None
+        ):
+            assert nd_stop is None
+
+            # setup to do a prepend of all existing src history
+            nd_start = self.xy_nd_start = src_stop
+            # set us in a zero-to-append state
+            nd_stop = self.xy_nd_stop = src_stop
+
+        # compute the length diffs between the first/last index entry in
+        # the input data and the last indexes we have on record from the
+        # last time we updated the curve index.
+        prepend_length = int(nd_start - src_start)
+        append_length = int(src_stop - nd_stop)
+
+        # blah blah blah
+        # do diffing for prepend, append and last entry
+        return (
+            slice(src_start, nd_start),
+            prepend_length,
+            append_length,
+            slice(nd_stop, src_stop),
+        )
+
+    def _track_inview_range(
+        self,
+        view_range: tuple[int, int],
+
+    ) -> bool:
+        # if a view range is passed, plan to draw the
+        # source ouput that's "in view" of the chart.
+        vl, vr = view_range
+        zoom_or_append = False
+        last_vr = self._last_vr
+
+        # incremental in-view data update.
+        if last_vr:
+            lvl, lvr = last_vr  # relative slice indices
+
+            # TODO: detecting more specifically the interaction changes
+            # last_ivr = self._last_ivdr or (vl, vr)
+            # al, ar = last_ivr  # abs slice indices
+            # left_change = abs(x_iv[0] - al) >= 1
+            # right_change = abs(x_iv[-1] - ar) >= 1
+
+            # likely a zoom/pan view change or data append update
+            if (
+                (vr - lvr) > 2
+                or vl < lvl
+
+                # append / prepend update
+                # we had an append update where the view range
+                # didn't change but the data-viewed (shifted)
+                # underneath, so we need to redraw.
+                # or left_change and right_change and last_vr == view_range
+
+                # not (left_change and right_change) and ivr
+                # (
+                # or abs(x_iv[ivr] - livr) > 1
+            ):
+                zoom_or_append = True
+
+        self._last_vr = view_range
+
+        return zoom_or_append
+
+    def format_to_1d(
+        self,
+        new_read: tuple,
+        array_key: str,
+        profiler: Profiler,
+
+        slice_to_inview: bool = True,
+
+    ) -> tuple[
+        np.ndarray,
+        np.ndarray,
+    ]:
+        shm = self.shm
+
+        (
+            _,
+            _,
+            array,
+            ivl,
+            ivr,
+            in_view,
+
+        ) = new_read
+
+        (
+            pre_slice,
+            prepend_len,
+            append_len,
+            post_slice,
+        ) = self.diff(new_read)
+
+        # we first need to allocate xy data arrays
+        # from the source data.
+        if self.y_nd is None:
+            self.xy_nd_start = shm._first.value
+            self.xy_nd_stop = shm._last.value
+            self.x_nd, self.y_nd = self.allocate_xy_nd(
+                shm,
+                array_key,
+            )
+            profiler('allocated xy history')
+
+        # once allocated we do incremental pre/append
+        # updates from the diff with the source buffer.
+        else:
+            if prepend_len:
+
+                self.incr_update_xy_nd(
+                    shm,
+                    array_key,
+
+                    # this is the pre-sliced, "normally expected"
+                    # new data that an updater would normally be
+                    # expected to process, however in some cases (like
+                    # step curves) the updater routine may want to do
+                    # the source history-data reading itself, so we pass
+                    # both here.
+                    shm._array[pre_slice],
+                    pre_slice,
+                    prepend_len,
+
+                    self.xy_nd_start,
+                    self.xy_nd_stop,
+                    is_append=False,
+                )
+
+                self.xy_nd_start -= prepend_len
+                profiler('prepended xy history: {prepend_length}')
+
+            if append_len:
+                self.incr_update_xy_nd(
+                    shm,
+                    array_key,
+
+                    shm._array[post_slice],
+                    post_slice,
+                    append_len,
+
+                    self.xy_nd_start,
+                    self.xy_nd_stop,
+                    is_append=True,
+                )
+                self.xy_nd_stop += append_len
+                profiler('appened xy history: {append_length}')
+                # sanity
+                # slice_ln = post_slice.stop - post_slice.start
+                # assert append_len == slice_ln
+
+        view_changed: bool = False
+        view_range: tuple[int, int] = (ivl, ivr)
+        if slice_to_inview:
+            view_changed = self._track_inview_range(view_range)
+            array = in_view
+            profiler(f'{self.viz.name} view range slice {view_range}')
+
+        # hist = array[:slice_to_head]
+
+        # XXX: WOA WTF TRACTOR DEBUGGING BUGGG
+        # assert 0
+
+        # xy-path data transform: convert source data to a format
+        # able to be passed to a `QPainterPath` rendering routine.
+        if not len(array):
+            # XXX: this might be why the profiler only has exits?
+            return
+
+        # TODO: hist here should be the pre-sliced
+        # x/y_data in the case where allocate_xy is
+        # defined?
+        x_1d, y_1d, connect = self.format_xy_nd_to_1d(
+            array,
+            array_key,
+            view_range,
+        )
+
+        # app_tres = None
+        # if append_len:
+        #     appended = array[-append_len-1:slice_to_head]
+        #     app_tres = self.format_xy_nd_to_1d(
+        #         appended,
+        #         array_key,
+        #         (
+        #             view_range[1] - append_len + slice_to_head,
+        #             view_range[1]
+        #         ),
+        #     )
+        #     # assert (len(appended) - 1) == append_len
+        #     # assert len(appended) == append_len
+        #     print(
+        #         f'{self.viz.name} APPEND LEN: {append_len}\n'
+        #         f'{self.viz.name} APPENDED: {appended}\n'
+        #         f'{self.viz.name} app_tres: {app_tres}\n'
+        #     )
+
+        # update the last "in view data range"
+        if len(x_1d):
+            self._last_ivdr = x_1d[0], x_1d[-1]
+
+        profiler('.format_to_1d()')
+
+        return (
+            x_1d,
+            y_1d,
+            connect,
+            prepend_len,
+            append_len,
+            view_changed,
+            # app_tres,
+        )
+
+    ###############################
+    # Sub-type override interface #
+    ###############################
+
+    x_offset: np.ndarray = np.array([0])
+
+    # optional pre-graphics xy formatted data which
+    # is incrementally updated in sync with the source data.
+    # XXX: was ``.allocate_xy()``
+    def allocate_xy_nd(
+        self,
+        src_shm: ShmArray,
+        data_field: str,
+
+    ) -> tuple[
+        np.ndarray,  # x
+        np.nd.array  # y
+    ]:
+        '''
+        Convert the structured-array ``src_shm`` format to
+        a equivalently shaped (and field-less) ``np.ndarray``.
+
+        Eg. a 4 field x N struct-array => (N, 4)
+
+        '''
+        y_nd = src_shm._array[data_field].copy()
+        x_nd = (
+            src_shm._array[self.index_field].copy()
+            +
+            self.x_offset
+        )
+        return x_nd, y_nd
+
+    # XXX: was ``.update_xy()``
+    def incr_update_xy_nd(
+        self,
+
+        src_shm: ShmArray,
+        data_field: str,
+
+        new_from_src: np.ndarray,  # portion of source that was updated
+
+        read_slc: slice,
+        ln: int,  # len of updated
+
+        nd_start: int,
+        nd_stop: int,
+
+        is_append: bool,
+
+    ) -> None:
+        # write pushed data to flattened copy
+        y_nd_new = new_from_src[data_field]
+        self.y_nd[read_slc] = y_nd_new
+
+        x_nd_new = self.x_nd[read_slc]
+        x_nd_new[:] = (
+            new_from_src[self.index_field]
+            +
+            self.x_offset
+        )
+
+        # x_nd = self.x_nd[self.xy_slice]
+        # y_nd = self.y_nd[self.xy_slice]
+        # name = self.viz.name
+        # if 'trade_rate' == name:
+        #     s = 4
+        #     print(
+        #         f'{name.upper()}:\n'
+        #         'NEW_FROM_SRC:\n'
+        #         f'new_from_src: {new_from_src}\n\n'
+
+        #         f'PRE self.x_nd:'
+        #         f'\n{list(x_nd[-s:])}\n'
+
+        #         f'PRE self.y_nd:\n'
+        #         f'{list(y_nd[-s:])}\n\n'
+
+        #         f'TO WRITE:\n'
+
+        #         f'x_nd_new:\n'
+        #         f'{x_nd_new[0]}\n'
+
+        #         f'y_nd_new:\n'
+        #         f'{y_nd_new}\n'
+        #     )
+
+    # XXX: was ``.format_xy()``
+    def format_xy_nd_to_1d(
+        self,
+
+        array: np.ndarray,
+        array_key: str,
+        vr: tuple[int, int],
+
+    ) -> tuple[
+        np.ndarray,  # 1d x
+        np.ndarray,  # 1d y
+        np.ndarray | str,  # connection array/style
+    ]:
+        '''
+        Default xy-nd array to 1d pre-graphics-path render routine.
+
+        Return single field column data verbatim
+
+        '''
+        # NOTE: we don't include the very last datum which is filled in
+        # normally by another graphics object.
+        x_1d = array[self.index_field][:-1]
+        y_1d = array[array_key][:-1]
+
+        # name = self.viz.name
+        # if 'trade_rate' == name:
+        #     s = 4
+        #     x_nd = list(self.x_nd[self.xy_slice][-s:-1])
+        #     y_nd = list(self.y_nd[self.xy_slice][-s:-1])
+        #     print(
+        #         f'{name}:\n'
+        #         f'XY data:\n'
+        #         f'x: {x_nd}\n'
+        #         f'y: {y_nd}\n\n'
+        #         f'x_1d: {list(x_1d[-s:])}\n'
+        #         f'y_1d: {list(y_1d[-s:])}\n\n'
+
+        #     )
+        return (
+            x_1d,
+            y_1d,
+
+            # 1d connection array or style-key to
+            # ``pg.functions.arrayToQPath()``
+            'all',
+        )
+
+
+class OHLCBarsFmtr(IncrementalFormatter):
+    x_offset: np.ndarray = np.array([
+        -0.5,
+        0,
+        0,
+        0.5,
+    ])
+
+    fields: list[str] = ['open', 'high', 'low', 'close']
+
+    def allocate_xy_nd(
+        self,
+
+        ohlc_shm: ShmArray,
+        data_field: str,
+
+    ) -> tuple[
+        np.ndarray,  # x
+        np.nd.array  # y
+    ]:
+        '''
+        Convert an input struct-array holding OHLC samples into a pair of
+        flattened x, y arrays with the same size (datums wise) as the source
+        data.
+
+        '''
+        y_nd = ohlc_shm.ustruct(self.fields)
+
+        # generate an flat-interpolated x-domain
+        x_nd = (
+            np.broadcast_to(
+                ohlc_shm._array[self.index_field][:, None],
+                (
+                    ohlc_shm._array.size,
+                    # 4,  # only ohlc
+                    y_nd.shape[1],
+                ),
+            )
+            +
+            self.x_offset
+        )
+        assert y_nd.any()
+
+        # write pushed data to flattened copy
+        return (
+            x_nd,
+            y_nd,
+        )
+
+    def incr_update_xy_nd(
+        self,
+
+        src_shm: ShmArray,
+        data_field: str,
+
+        new_from_src: np.ndarray,  # portion of source that was updated
+
+        read_slc: slice,
+        ln: int,  # len of updated
+
+        nd_start: int,
+        nd_stop: int,
+
+        is_append: bool,
+
+    ) -> None:
+        # write newly pushed data to flattened copy
+        # a struct-arr is always passed in.
+        new_y_nd = rfn.structured_to_unstructured(
+            new_from_src[self.fields]
+        )
+        self.y_nd[read_slc] = new_y_nd
+
+        # generate same-valued-per-row x support based on y shape
+        x_nd_new = self.x_nd[read_slc]
+        x_nd_new[:] = np.broadcast_to(
+            new_from_src[self.index_field][:, None],
+            new_y_nd.shape,
+        ) + self.x_offset
+
+    # TODO: can we drop this frame and just use the above?
+    def format_xy_nd_to_1d(
+        self,
+
+        array: np.ndarray,
+        array_key: str,
+        vr: tuple[int, int],
+
+        start: int = 0,  # XXX: do we need this?
+        # 0.5 is no overlap between arms, 1.0 is full overlap
+        w: float = 0.16,
+
+    ) -> tuple[
+        np.ndarray,
+        np.ndarray,
+        np.ndarray,
+    ]:
+        '''
+        More or less direct proxy to the ``numba``-fied
+        ``path_arrays_from_ohlc()`` (above) but with closed in kwargs
+        for line spacing.
+
+        '''
+        x, y, c = path_arrays_from_ohlc(
+            array,
+            start,
+            bar_w=self.index_step_size,
+            bar_gap=w * self.index_step_size,
+
+            # XXX: don't ask, due to a ``numba`` bug..
+            use_time_index=(self.index_field == 'time'),
+        )
+        return x, y, c
+
+
+class OHLCBarsAsCurveFmtr(OHLCBarsFmtr):
+
+    def format_xy_nd_to_1d(
+        self,
+
+        array: np.ndarray,
+        array_key: str,
+        vr: tuple[int, int],
+
+    ) -> tuple[
+        np.ndarray,
+        np.ndarray,
+        str,
+    ]:
+        # TODO: in the case of an existing ``.update_xy()``
+        # should we be passing in array as an xy arrays tuple?
+
+        # 2 more datum-indexes to capture zero at end
+        x_flat = self.x_nd[self.xy_nd_start:self.xy_nd_stop-1]
+        y_flat = self.y_nd[self.xy_nd_start:self.xy_nd_stop-1]
+
+        # slice to view
+        ivl, ivr = vr
+        x_iv_flat = x_flat[ivl:ivr]
+        y_iv_flat = y_flat[ivl:ivr]
+
+        # reshape to 1d for graphics rendering
+        y_iv = y_iv_flat.reshape(-1)
+        x_iv = x_iv_flat.reshape(-1)
+
+        return x_iv, y_iv, 'all'
+
+
+class StepCurveFmtr(IncrementalFormatter):
+
+    x_offset: np.ndarray = np.array([
+        0,
+        1,
+    ])
+
+    def allocate_xy_nd(
+        self,
+
+        shm: ShmArray,
+        data_field: str,
+
+    ) -> tuple[
+        np.ndarray,  # x
+        np.nd.array  # y
+    ]:
+        '''
+        Convert an input 1d shm array to a "step array" format
+        for use by path graphics generation.
+
+        '''
+        i = shm._array[self.index_field].copy()
+        out = shm._array[data_field].copy()
+
+        x_out = (
+            np.broadcast_to(
+                i[:, None],
+                (i.size, 2),
+            )
+            +
+            self.x_offset
+        )
+
+        # fill out Nx2 array to hold each step's left + right vertices.
+        y_out = np.empty(
+            x_out.shape,
+            dtype=out.dtype,
+        )
+        # fill in (current) values from source shm buffer
+        y_out[:] = out[:, np.newaxis]
+
+        # TODO: pretty sure we can drop this?
+        # start y at origin level
+        # y_out[0, 0] = 0
+        # y_out[self.xy_nd_start] = 0
+        return x_out, y_out
+
+    def incr_update_xy_nd(
+        self,
+
+        src_shm: ShmArray,
+        array_key: str,
+
+        new_from_src: np.ndarray,  # portion of source that was updated
+        read_slc: slice,
+        ln: int,  # len of updated
+
+        nd_start: int,
+        nd_stop: int,
+
+        is_append: bool,
+
+    ) -> tuple[
+        np.ndarray,
+        slice,
+    ]:
+        # NOTE: for a step curve we slice from one datum prior
+        # to the current "update slice" to get the previous
+        # "level".
+        #
+        # why this is needed,
+        # - the current new append slice will often have a zero
+        #   value in the latest datum-step (at least for zero-on-new
+        #   cases like vlm in the) as per configuration of the FSP
+        #   engine.
+        # - we need to look back a datum to get the last level which
+        #   will be used to terminate/complete the last step x-width
+        #   which will be set to pair with the last x-index THIS MEANS
+        #
+        # XXX: this means WE CAN'T USE the append slice since we need to
+        # "look backward" one step to get the needed back-to-zero level
+        # and the update data in ``new_from_src`` will only contain the
+        # latest new data.
+        back_1 = slice(
+            read_slc.start - 1,
+            read_slc.stop,
+        )
+
+        to_write = src_shm._array[back_1]
+        y_nd_new = self.y_nd[back_1]
+        y_nd_new[:] = to_write[array_key][:, None]
+
+        x_nd_new = self.x_nd[read_slc]
+        x_nd_new[:] = (
+            new_from_src[self.index_field][:, None]
+            +
+            self.x_offset
+        )
+
+        # XXX: uncomment for debugging
+        # x_nd = self.x_nd[self.xy_slice]
+        # y_nd = self.y_nd[self.xy_slice]
+        # name = self.viz.name
+        # if 'dolla_vlm' in name:
+        #     s = 4
+        #     print(
+        #         f'{name}:\n'
+        #         'NEW_FROM_SRC:\n'
+        #         f'new_from_src: {new_from_src}\n\n'
+
+        #         f'PRE self.x_nd:'
+        #         f'\n{x_nd[-s:]}\n'
+        #         f'PRE self.y_nd:\n'
+        #         f'{y_nd[-s:]}\n\n'
+
+        #         f'TO WRITE:\n'
+        #         f'x_nd_new:\n'
+        #         f'{x_nd_new}\n'
+        #         f'y_nd_new:\n'
+        #         f'{y_nd_new}\n'
+        #     )
+
+    def format_xy_nd_to_1d(
+        self,
+
+        array: np.ndarray,
+        array_key: str,
+        vr: tuple[int, int],
+
+    ) -> tuple[
+        np.ndarray,
+        np.ndarray,
+        str,
+    ]:
+        last_t, last = array[-1][[self.index_field, array_key]]
+
+        start = self.xy_nd_start
+        stop = self.xy_nd_stop
+
+        x_step = self.x_nd[start:stop]
+        y_step = self.y_nd[start:stop]
+
+        # slice out in-view data
+        ivl, ivr = vr
+
+        # NOTE: add an extra step to get the vertical-line-down-to-zero
+        # adjacent to the last-datum graphic (filled rect).
+        x_step_iv = x_step[ivl:ivr+1]
+        y_step_iv = y_step[ivl:ivr+1]
+
+        # flatten to 1d
+        x_1d = x_step_iv.reshape(x_step_iv.size)
+        y_1d = y_step_iv.reshape(y_step_iv.size)
+
+        # debugging
+        # if y_1d.any():
+        #     s = 6
+        #     print(
+        #         f'x_step_iv:\n{x_step_iv[-s:]}\n'
+        #         f'y_step_iv:\n{y_step_iv[-s:]}\n\n'
+        #         f'x_1d:\n{x_1d[-s:]}\n'
+        #         f'y_1d:\n{y_1d[-s:]}\n'
+        #     )
+
+        return x_1d, y_1d, 'all'

piker/data/_m4.py

@@ -15,17 +15,30 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 '''
-Graphics related downsampling routines for compressing to pixel
-limits on the display device.
+Graphics downsampling using the infamous M4 algorithm.
+
+This is one of ``piker``'s secret weapons allowing us to boss all other
+charting platforms B)
+
+(AND DON'T YOU DARE TAKE THIS CODE WITHOUT CREDIT OR WE'LL SUE UR F#&@* ASS).
+
+NOTES: this method is a so called "visualization driven data
+aggregation" approach. It gives error-free line chart
+downsampling, see
+further scientific paper resources:
+- http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
+- http://www.vldb.org/2014/program/papers/demo/p997-jugel.pdf
+
+Details on implementation of this algo are based in,
+https://github.com/pikers/piker/issues/109
 
 '''
 import math
 from typing import Optional
 
 import numpy as np
-from numpy.lib import recfunctions as rfn
 from numba import (
-    jit,
+    njit,
     # float64, optional, int64,
 )
 
@@ -35,109 +48,6 @@ from ..log import get_logger
 log = get_logger(__name__)
 
 
-def hl2mxmn(ohlc: np.ndarray) -> np.ndarray:
-    '''
-    Convert a OHLC struct-array containing 'high'/'low' columns
-    to a "joined" max/min 1-d array.
-
-    '''
-    index = ohlc['index']
-    hls = ohlc[[
-        'low',
-        'high',
-    ]]
-
-    mxmn = np.empty(2*hls.size, dtype=np.float64)
-    x = np.empty(2*hls.size, dtype=np.float64)
-    trace_hl(hls, mxmn, x, index[0])
-    x = x + index[0]
-
-    return mxmn, x
-
-
-@jit(
-    # TODO: the type annots..
-    # float64[:](float64[:],),
-    nopython=True,
-)
-def trace_hl(
-    hl: 'np.ndarray',
-    out: np.ndarray,
-    x: np.ndarray,
-    start: int,
-
-    # the "offset" values in the x-domain which
-    # place the 2 output points around each ``int``
-    # master index.
-    margin: float = 0.43,
-
-) -> None:
-    '''
-    "Trace" the outline of the high-low values of an ohlc sequence
-    as a line such that the maximum deviation (aka disperaion) between
-    bars if preserved.
-
-    This routine is expected to modify input arrays in-place.
-
-    '''
-    last_l = hl['low'][0]
-    last_h = hl['high'][0]
-
-    for i in range(hl.size):
-        row = hl[i]
-        l, h = row['low'], row['high']
-
-        up_diff = h - last_l
-        down_diff = last_h - l
-
-        if up_diff > down_diff:
-            out[2*i + 1] = h
-            out[2*i] = last_l
-        else:
-            out[2*i + 1] = l
-            out[2*i] = last_h
-
-        last_l = l
-        last_h = h
-
-        x[2*i] = int(i) - margin
-        x[2*i + 1] = int(i) + margin
-
-    return out
-
-
-def ohlc_flatten(
-    ohlc: np.ndarray,
-    use_mxmn: bool = True,
-
-) -> tuple[np.ndarray, np.ndarray]:
-    '''
-    Convert an OHLCV struct-array into a flat ready-for-line-plotting
-    1-d array that is 4 times the size with x-domain values distributed
-    evenly (by 0.5 steps) over each index.
-
-    '''
-    index = ohlc['index']
-
-    if use_mxmn:
-        # traces a line optimally over highs to lows
-        # using numba. NOTE: pretty sure this is faster
-        # and looks about the same as the below output.
-        flat, x = hl2mxmn(ohlc)
-
-    else:
-        flat = rfn.structured_to_unstructured(
-            ohlc[['open', 'high', 'low', 'close']]
-        ).flatten()
-
-        x = np.linspace(
-            start=index[0] - 0.5,
-            stop=index[-1] + 0.5,
-            num=len(flat),
-        )
-    return x, flat
-
-
 def ds_m4(
     x: np.ndarray,
     y: np.ndarray,
@@ -160,16 +70,6 @@ def ds_m4(
     This is more or less an OHLC style sampling of a line-style series.
 
     '''
-    # NOTE: this method is a so called "visualization driven data
-    # aggregation" approach. It gives error-free line chart
-    # downsampling, see
-    # further scientific paper resources:
-    # - http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
-    # - http://www.vldb.org/2014/program/papers/demo/p997-jugel.pdf
-
-    # Details on implementation of this algo are based in,
-    # https://github.com/pikers/piker/issues/109
-
     # XXX: from infinite on downsampling viewable graphics:
     # "one thing i remembered about the binning - if you are
     # picking a range within your timeseries the start and end bin
@@ -191,6 +91,14 @@ def ds_m4(
         x_end = x[-1]  # x end value/highest in domain
         xrange = (x_end - x_start)
 
+    if xrange < 0:
+        log.error(f'-VE M4 X-RANGE: {x_start} -> {x_end}')
+        # XXX: broken x-range calc-case, likely the x-end points
+        # are wrong and have some default value set (such as
+        # x_end -> <some epoch float> while x_start -> 0.5).
+        # breakpoint()
+        return None
+
     # XXX: always round up on the input pixels
     # lnx = len(x)
     # uppx *= max(4 / (1 + math.log(uppx, 2)), 1)
@@ -256,8 +164,7 @@ def ds_m4(
     return nb, x_out, y_out, ymn, ymx
 
 
-@jit(
-    nopython=True,
+@njit(
     nogil=True,
 )
 def _m4(

piker/data/_pathops.py

@@ -0,0 +1,455 @@
+# piker: trading gear for hackers
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Super fast ``QPainterPath`` generation related operator routines.
+
+"""
+from math import (
+    ceil,
+    floor,
+)
+
+import numpy as np
+from numpy.lib import recfunctions as rfn
+from numba import (
+    # types,
+    njit,
+    float64,
+    int64,
+    # optional,
+)
+
+# TODO: for ``numba`` typing..
+# from ._source import numba_ohlc_dtype
+from ._m4 import ds_m4
+from .._profile import (
+    Profiler,
+    pg_profile_enabled,
+    ms_slower_then,
+)
+
+
+def xy_downsample(
+    x,
+    y,
+    uppx,
+
+    x_spacer: float = 0.5,
+
+) -> tuple[
+    np.ndarray,
+    np.ndarray,
+    float,
+    float,
+]:
+    '''
+    Downsample 1D (flat ``numpy.ndarray``) arrays using M4 given an input
+    ``uppx`` (units-per-pixel) and add space between discreet datums.
+
+    '''
+    # downsample whenever more then 1 pixels per datum can be shown.
+    # always refresh data bounds until we get diffing
+    # working properly, see above..
+    m4_out = ds_m4(
+        x,
+        y,
+        uppx,
+    )
+
+    if m4_out is not None:
+        bins, x, y, ymn, ymx = m4_out
+        # flatten output to 1d arrays suitable for path-graphics generation.
+        x = np.broadcast_to(x[:, None], y.shape)
+        x = (x + np.array(
+            [-x_spacer, 0, 0, x_spacer]
+        )).flatten()
+        y = y.flatten()
+
+        return x, y, ymn, ymx
+
+    # XXX: we accept a None output for the case where the input range
+    # to ``ds_m4()`` is bad (-ve) and we want to catch and debug
+    # that (seemingly super rare) circumstance..
+    return None
+
+
+@njit(
+    # NOTE: need to construct this manually for readonly
+    # arrays, see https://github.com/numba/numba/issues/4511
+    # (
+    #     types.Array(
+    #         numba_ohlc_dtype,
+    #         1,
+    #         'C',
+    #         readonly=True,
+    #     ),
+    #     int64,
+    #     types.unicode_type,
+    #     optional(float64),
+    # ),
+    nogil=True
+)
+def path_arrays_from_ohlc(
+    data: np.ndarray,
+    start: int64,
+    bar_w: float64,
+    bar_gap: float64 = 0.16,
+    use_time_index: bool = True,
+
+    # XXX: ``numba`` issue: https://github.com/numba/numba/issues/8622
+    # index_field: str,
+
+) -> tuple[
+    np.ndarray,
+    np.ndarray,
+    np.ndarray,
+]:
+    '''
+    Generate an array of lines objects from input ohlc data.
+
+    '''
+    size = int(data.shape[0] * 6)
+
+    # XXX: see this for why the dtype might have to be defined outside
+    # the routine.
+    # https://github.com/numba/numba/issues/4098#issuecomment-493914533
+    x = np.zeros(
+        shape=size,
+        dtype=float64,
+    )
+    y, c = x.copy(), x.copy()
+
+    half_w: float = bar_w/2
+
+    # TODO: report bug for assert @
+    # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
+    for i, q in enumerate(data[start:], start):
+
+        open = q['open']
+        high = q['high']
+        low = q['low']
+        close = q['close']
+
+        if use_time_index:
+            index = float64(q['time'])
+        else:
+            index = float64(q['index'])
+
+        # XXX: ``numba`` issue: https://github.com/numba/numba/issues/8622
+        # index = float64(q[index_field])
+        # AND this (probably)
+        # open, high, low, close, index = q[
+        #     ['open', 'high', 'low', 'close', 'index']]
+
+        istart = i * 6
+        istop = istart + 6
+
+        # x,y detail the 6 points which connect all vertexes of a ohlc bar
+        mid: float = index + half_w
+        x[istart:istop] = (
+            index + bar_gap,
+            mid,
+            mid,
+            mid,
+            mid,
+            index + bar_w - bar_gap,
+        )
+        y[istart:istop] = (
+            open,
+            open,
+            low,
+            high,
+            close,
+            close,
+        )
+
+        # specifies that the first edge is never connected to the
+        # prior bars last edge thus providing a small "gap"/"space"
+        # between bars determined by ``bar_gap``.
+        c[istart:istop] = (1, 1, 1, 1, 1, 0)
+
+    return x, y, c
+
+
+def hl2mxmn(
+    ohlc: np.ndarray,
+    index_field: str = 'index',
+
+) -> np.ndarray:
+    '''
+    Convert a OHLC struct-array containing 'high'/'low' columns
+    to a "joined" max/min 1-d array.
+
+    '''
+    index = ohlc[index_field]
+    hls = ohlc[[
+        'low',
+        'high',
+    ]]
+
+    mxmn = np.empty(2*hls.size, dtype=np.float64)
+    x = np.empty(2*hls.size, dtype=np.float64)
+    trace_hl(hls, mxmn, x, index[0])
+    x = x + index[0]
+
+    return mxmn, x
+
+
+@njit(
+    # TODO: the type annots..
+    # float64[:](float64[:],),
+)
+def trace_hl(
+    hl: 'np.ndarray',
+    out: np.ndarray,
+    x: np.ndarray,
+    start: int,
+
+    # the "offset" values in the x-domain which
+    # place the 2 output points around each ``int``
+    # master index.
+    margin: float = 0.43,
+
+) -> None:
+    '''
+    "Trace" the outline of the high-low values of an ohlc sequence
+    as a line such that the maximum deviation (aka disperaion) between
+    bars if preserved.
+
+    This routine is expected to modify input arrays in-place.
+
+    '''
+    last_l = hl['low'][0]
+    last_h = hl['high'][0]
+
+    for i in range(hl.size):
+        row = hl[i]
+        l, h = row['low'], row['high']
+
+        up_diff = h - last_l
+        down_diff = last_h - l
+
+        if up_diff > down_diff:
+            out[2*i + 1] = h
+            out[2*i] = last_l
+        else:
+            out[2*i + 1] = l
+            out[2*i] = last_h
+
+        last_l = l
+        last_h = h
+
+        x[2*i] = int(i) - margin
+        x[2*i + 1] = int(i) + margin
+
+    return out
+
+
+def ohlc_flatten(
+    ohlc: np.ndarray,
+    use_mxmn: bool = True,
+    index_field: str = 'index',
+
+) -> tuple[np.ndarray, np.ndarray]:
+    '''
+    Convert an OHLCV struct-array into a flat ready-for-line-plotting
+    1-d array that is 4 times the size with x-domain values distributed
+    evenly (by 0.5 steps) over each index.
+
+    '''
+    index = ohlc[index_field]
+
+    if use_mxmn:
+        # traces a line optimally over highs to lows
+        # using numba. NOTE: pretty sure this is faster
+        # and looks about the same as the below output.
+        flat, x = hl2mxmn(ohlc)
+
+    else:
+        flat = rfn.structured_to_unstructured(
+            ohlc[['open', 'high', 'low', 'close']]
+        ).flatten()
+
+        x = np.linspace(
+            start=index[0] - 0.5,
+            stop=index[-1] + 0.5,
+            num=len(flat),
+        )
+    return x, flat
+
+
+def slice_from_time(
+    arr: np.ndarray,
+    start_t: float,
+    stop_t: float,
+    step: int | None = None,
+
+) -> tuple[
+    slice,
+    slice,
+]:
+    '''
+    Calculate array indices mapped from a time range and return them in
+    a slice.
+
+    Given an input array with an epoch `'time'` series entry, calculate
+    the indices which span the time range and return in a slice. Presume
+    each `'time'` step increment is uniform and when the time stamp
+    series contains gaps (the uniform presumption is untrue) use
+    ``np.searchsorted()`` binary search to look up the appropriate
+    index.
+
+    '''
+    profiler = Profiler(
+        msg='slice_from_time()',
+        disabled=not pg_profile_enabled(),
+        ms_threshold=ms_slower_then,
+    )
+
+    times = arr['time']
+    t_first = floor(times[0])
+    t_last = ceil(times[-1])
+
+    # the greatest index we can return which slices to the
+    # end of the input array.
+    read_i_max = arr.shape[0]
+
+    # TODO: require this is always passed in?
+    if step is None:
+        step = round(t_last - times[-2])
+        if step == 0:
+            step = 1
+
+    # compute (presumed) uniform-time-step index offsets
+    i_start_t = floor(start_t)
+    read_i_start = floor(((i_start_t - t_first) // step)) - 1
+
+    i_stop_t = ceil(stop_t)
+
+    # XXX: edge case -> always set stop index to last in array whenever
+    # the input stop time is detected to be greater then the equiv time
+    # stamp at that last entry.
+    if i_stop_t >= t_last:
+        read_i_stop = read_i_max
+    else:
+        read_i_stop = ceil((i_stop_t - t_first) // step) + 1
+
+    # always clip outputs to array support
+    # for read start:
+    # - never allow a start < the 0 index
+    # - never allow an end index > the read array len
+    read_i_start = min(
+        max(0, read_i_start),
+        read_i_max - 1,
+    )
+    read_i_stop = max(
+        0,
+        min(read_i_stop, read_i_max),
+    )
+
+    # check for larger-then-latest calculated index for given start
+    # time, in which case we do a binary search for the correct index.
+    # NOTE: this is usually the result of a time series with time gaps
+    # where it is expected that each index step maps to a uniform step
+    # in the time stamp series.
+    t_iv_start = times[read_i_start]
+    if (
+        t_iv_start > i_start_t
+    ):
+        # do a binary search for the best index mapping to ``start_t``
+        # given we measured an overshoot using the uniform-time-step
+        # calculation from above.
+
+        # TODO: once we start caching these per source-array,
+        # we can just overwrite ``read_i_start`` directly.
+        new_read_i_start = np.searchsorted(
+            times,
+            i_start_t,
+            side='left',
+        )
+
+        # TODO: minimize binary search work as much as possible:
+        # - cache these remap values which compensate for gaps in the
+        #   uniform time step basis where we calc a later start
+        #   index for the given input ``start_t``.
+        # - can we shorten the input search sequence by heuristic?
+        #   up_to_arith_start = index[:read_i_start]
+
+        if (
+            new_read_i_start <= read_i_start
+        ):
+            # t_diff = t_iv_start - start_t
+            # print(
+            #     f"WE'RE CUTTING OUT TIME - STEP:{step}\n"
+            #     f'start_t:{start_t} -> 0index start_t:{t_iv_start}\n'
+            #     f'diff: {t_diff}\n'
+            #     f'REMAPPED START i: {read_i_start} -> {new_read_i_start}\n'
+            # )
+            read_i_start = new_read_i_start - 1
+
+    t_iv_stop = times[read_i_stop - 1]
+    if (
+        t_iv_stop > i_stop_t
+    ):
+        # t_diff = stop_t - t_iv_stop
+        # print(
+        #     f"WE'RE CUTTING OUT TIME - STEP:{step}\n"
+        #     f'calced iv stop:{t_iv_stop} -> stop_t:{stop_t}\n'
+        #     f'diff: {t_diff}\n'
+        #     # f'SHOULD REMAP STOP: {read_i_start} -> {new_read_i_start}\n'
+        # )
+        new_read_i_stop = np.searchsorted(
+            times[read_i_start:],
+            # times,
+            i_stop_t,
+            side='left',
+        )
+
+        if (
+            new_read_i_stop <= read_i_stop
+        ):
+            read_i_stop = read_i_start + new_read_i_stop + 1
+
+    # sanity checks for range size
+    # samples = (i_stop_t - i_start_t) // step
+    # index_diff = read_i_stop - read_i_start + 1
+    # if index_diff > (samples + 3):
+    #     breakpoint()
+
+    # read-relative indexes: gives a slice where `shm.array[read_slc]`
+    # will be the data spanning the input time range `start_t` ->
+    # `stop_t`
+    read_slc = slice(
+        int(read_i_start),
+        int(read_i_stop),
+    )
+
+    profiler(
+        'slicing complete'
+        # f'{start_t} -> {abs_slc.start} | {read_slc.start}\n'
+        # f'{stop_t} -> {abs_slc.stop} | {read_slc.stop}\n'
+    )
+
+    # NOTE: if caller needs absolute buffer indices they can
+    # slice the buffer abs index like so:
+    # index = arr['index']
+    # abs_indx = index[read_slc]
+    # abs_slc = slice(
+    #     int(abs_indx[0]),
+    #     int(abs_indx[-1]),
+    # )
+
+    return read_slc

piker/data/feed.py

@@ -207,7 +207,7 @@ def get_feed_bus(
 
 ) -> _FeedsBus:
     '''
-    Retreive broker-daemon-local data feeds bus from process global
+    Retrieve broker-daemon-local data feeds bus from process global
     scope. Serialize task access to lock.
 
     '''
@@ -250,6 +250,7 @@ async def start_backfill(
     shm: ShmArray,
     timeframe: float,
     sampler_stream: tractor.MsgStream,
+    feed_is_live: trio.Event,
 
     last_tsdb_dt: Optional[datetime] = None,
     storage: Optional[Storage] = None,
@@ -281,9 +282,30 @@ async def start_backfill(
             - pendulum.from_timestamp(times[-2])
         ).seconds
 
-        if step_size_s == 60:
+        # if the market is open (aka we have a live feed) but the
+        # history sample step index seems off we report the surrounding
+        # data and drop into a bp. this case shouldn't really ever
+        # happen if we're doing history retrieval correctly.
+        if (
+            step_size_s == 60
+            and feed_is_live.is_set()
+        ):
             inow = round(time.time())
-            if (inow - times[-1]) > 60:
+            diff = inow - times[-1]
+            if abs(diff) > 60:
+                surr = array[-6:]
+                diff_in_mins = round(diff/60., ndigits=2)
+                log.warning(
+                    f'STEP ERROR `{bfqsn}` for period {step_size_s}s:\n'
+                    f'Off by `{diff}` seconds (or `{diff_in_mins}` mins)\n'
+                    'Surrounding 6 time stamps:\n'
+                    f'{list(surr["time"])}\n'
+                    'Here is surrounding 6 samples:\n'
+                    f'{surr}\nn'
+                )
+
+                # for now we expect a hacker to investigate this case
+                # manually..
                 await tractor.breakpoint()
 
         # frame's worth of sample-period-steps, in seconds
@@ -485,6 +507,7 @@ async def basic_backfill(
     bfqsn: str,
     shms: dict[int, ShmArray],
     sampler_stream: tractor.MsgStream,
+    feed_is_live: trio.Event,
 
 ) -> None:
 
@@ -504,6 +527,7 @@ async def basic_backfill(
                     shm,
                     timeframe,
                     sampler_stream,
+                    feed_is_live,
                 )
             )
         except DataUnavailable:
@@ -520,6 +544,7 @@ async def tsdb_backfill(
     bfqsn: str,
     shms: dict[int, ShmArray],
     sampler_stream: tractor.MsgStream,
+    feed_is_live: trio.Event,
 
     task_status: TaskStatus[
         tuple[ShmArray, ShmArray]
@@ -554,6 +579,8 @@ async def tsdb_backfill(
                     shm,
                     timeframe,
                     sampler_stream,
+                    feed_is_live,
+
                     last_tsdb_dt=last_tsdb_dt,
                     tsdb_is_up=True,
                     storage=storage,
@@ -856,6 +883,7 @@ async def manage_history(
                         60: hist_shm,
                     },
                     sample_stream,
+                    feed_is_live,
                 )
 
                 # yield back after client connect with filled shm
@@ -890,6 +918,7 @@ async def manage_history(
                     60: hist_shm,
                 },
                 sample_stream,
+                feed_is_live,
             )
             task_status.started((
                 hist_zero_index,
@@ -1023,12 +1052,11 @@ async def allocate_persistent_feed(
 
     flume = Flume(
         symbol=symbol,
-        _hist_shm_token=hist_shm.token,
-        _rt_shm_token=rt_shm.token,
         first_quote=first_quote,
+        _rt_shm_token=rt_shm.token,
+        _hist_shm_token=hist_shm.token,
         izero_hist=izero_hist,
         izero_rt=izero_rt,
-        # throttle_rate=tick_throttle,
     )
 
     # for ambiguous names we simply apply the retreived
@@ -1052,7 +1080,10 @@ async def allocate_persistent_feed(
     # seed the buffer with a history datum - this is most handy
     # for many backends which don't sample @ 1s OHLC but do have
     # slower data such as 1m OHLC.
-    if not len(rt_shm.array):
+    if (
+        not len(rt_shm.array)
+        and hist_shm.array.size
+    ):
         rt_shm.push(hist_shm.array[-3:-1])
         ohlckeys = ['open', 'high', 'low', 'close']
         rt_shm.array[ohlckeys][-2:] = hist_shm.array['close'][-1]
@@ -1063,6 +1094,9 @@ async def allocate_persistent_feed(
         rt_shm.array['time'][0] = ts
         rt_shm.array['time'][1] = ts + 1
 
+    elif hist_shm.array.size == 0:
+        await tractor.breakpoint()
+
     # wait the spawning parent task to register its subscriber
     # send-stream entry before we start the sample loop.
     await sub_registered.wait()

piker/data/flows.py

@@ -22,17 +22,11 @@ real-time data processing data-structures.
 
 """
 from __future__ import annotations
-from contextlib import asynccontextmanager as acm
-from functools import partial
 from typing import (
-    AsyncIterator,
     TYPE_CHECKING,
 )
 
 import tractor
-from tractor.trionics import (
-    maybe_open_context,
-)
 import pendulum
 import numpy as np
 
@@ -45,12 +39,13 @@ from ._sharedmem import (
     ShmArray,
     _Token,
 )
-from ._sampling import (
-    open_sample_stream,
-)
+# from .._profile import (
+#     Profiler,
+#     pg_profile_enabled,
+# )
 
 if TYPE_CHECKING:
-    from pyqtgraph import PlotItem
+    # from pyqtgraph import PlotItem
     from .feed import Feed
 
 
@@ -147,26 +142,6 @@ class Flume(Struct):
     async def receive(self) -> dict:
         return await self.stream.receive()
 
-    @acm
-    async def index_stream(
-        self,
-        delay_s: float = 1,
-
-    ) -> AsyncIterator[int]:
-
-        if not self.feed:
-            raise RuntimeError('This flume is not part of any ``Feed``?')
-
-        # TODO: maybe a public (property) API for this in ``tractor``?
-        portal = self.stream._ctx._portal
-        assert portal
-
-        # XXX: this should be singleton on a host,
-        # a lone broker-daemon per provider should be
-        # created for all practical purposes
-        async with open_sample_stream(float(delay_s)) as stream:
-            yield stream
-
     def get_ds_info(
         self,
     ) -> tuple[float, float, float]:
@@ -218,104 +193,18 @@ class Flume(Struct):
     def get_index(
         self,
         time_s: float,
+        array: np.ndarray,
 
-    ) -> int:
+    ) -> int | float:
         '''
         Return array shm-buffer index for for epoch time.
 
         '''
-        array = self.rt_shm.array
         times = array['time']
-        mask = (times >= time_s)
-
-        if any(mask):
-            return array['index'][mask][0]
-
-        # just the latest index
-        array['index'][-1]
-
-    def slice_from_time(
-        self,
-        array: np.ndarray,
-        start_t: float,
-        stop_t: float,
-        timeframe_s: int = 1,
-        return_data: bool = False,
-
-    ) -> np.ndarray:
-        '''
-        Slice an input struct array providing only datums
-        "in view" of this chart.
-
-        '''
-        arr = {
-            1: self.rt_shm.array,
-            60: self.hist_shm.arry,
-        }[timeframe_s]
-
-        times = arr['time']
-        index = array['index']
-
-        # use advanced indexing to map the
-        # time range to the index range.
-        mask = (
-            (times >= start_t)
-            &
-            (times < stop_t)
+        first = np.searchsorted(
+            times,
+            time_s,
+            side='left',
         )
-
-        # TODO: if we can ensure each time field has a uniform
-        # step we can instead do some arithmetic to determine
-        # the equivalent index like we used to?
-        # return array[
-        #     lbar - ifirst:
-        #     (rbar - ifirst) + 1
-        # ]
-
-        i_by_t = index[mask]
-        i_0 = i_by_t[0]
-
-        abs_slc = slice(
-            i_0,
-            i_by_t[-1],
-        )
-        # slice data by offset from the first index
-        # available in the passed datum set.
-        read_slc = slice(
-            0,
-            i_by_t[-1] - i_0,
-        )
-        if not return_data:
-            return (
-                abs_slc,
-                read_slc,
-            )
-
-        # also return the readable data from the timerange
-        return (
-            abs_slc,
-            read_slc,
-            arr[mask],
-        )
-
-    def view_data(
-        self,
-        plot: PlotItem,
-        timeframe_s: int = 1,
-
-    ) -> np.ndarray:
-
-        # get far-side x-indices plot view
-        vr = plot.viewRect()
-
-        (
-            abs_slc,
-            buf_slc,
-            iv_arr,
-        ) = self.slice_from_time(
-            start_t=vr.left(),
-            stop_t=vr.right(),
-            timeframe_s=timeframe_s,
-            return_data=True,
-        )
-        return iv_arr
+        imx = times.shape[0] - 1
+        return min(first, imx)

piker/fsp/_engine.py

@@ -188,6 +188,8 @@ async def fsp_compute(
 
     history_by_field['time'] = src_time[-len(history_by_field):]
 
+    history_output['time'] = src.array['time']
+
     # TODO: XXX:
     # THERE'S A BIG BUG HERE WITH THE `index` field since we're
     # prepending a copy of the first value a few times to make

piker/pp.py

@@ -29,6 +29,7 @@ import re
 import time
 from typing import (
     Any,
+    Iterator,
     Optional,
     Union,
 )
@@ -53,7 +54,7 @@ def open_trade_ledger(
     broker: str,
     account: str,
 
-) -> str:
+) -> dict:
     '''
     Indempotently create and read in a trade log file from the
     ``<configuration_dir>/ledgers/`` directory.
@@ -116,6 +117,21 @@ class Transaction(Struct, frozen=True):
     # from: Optional[str] = None
 
 
+def iter_by_dt(
+    clears: dict[str, Any],
+) -> Iterator[tuple[str, dict]]:
+    '''
+    Iterate entries of a ``clears: dict`` table sorted by entry recorded
+    datetime presumably set at the ``'dt'`` field in each entry.
+
+    '''
+    for tid, data in sorted(
+        list(clears.items()),
+        key=lambda item: item[1]['dt'],
+    ):
+        yield tid, data
+
+
 class Position(Struct):
     '''
     Basic pp (personal/piker position) model with attached clearing
@@ -183,12 +199,7 @@ class Position(Struct):
         toml_clears_list = []
 
         # reverse sort so latest clears are at top of section?
-        for tid, data in sorted(
-            list(clears.items()),
-
-            # sort by datetime
-            key=lambda item: item[1]['dt'],
-        ):
+        for tid, data in iter_by_dt(clears):
             inline_table = toml.TomlDecoder().get_empty_inline_table()
 
             # serialize datetime to parsable `str`
@@ -301,6 +312,14 @@ class Position(Struct):
     # def lifo_price() -> float:
     #     ...
 
+    def iter_clears(self) -> Iterator[tuple[str, dict]]:
+        '''
+        Iterate the internally managed ``.clears: dict`` table in
+        datetime-stamped order.
+
+        '''
+        return iter_by_dt(self.clears)
+
     def calc_ppu(
         self,
         # include transaction cost in breakeven price
@@ -331,10 +350,9 @@ class Position(Struct):
         asize_h: list[float] = []  # historical accumulative size
         ppu_h: list[float] = []  # historical price-per-unit
 
-        clears = list(self.clears.items())
-
-        for i, (tid, entry) in enumerate(clears):
-
+        tid: str
+        entry: dict[str, Any]
+        for (tid, entry) in self.iter_clears():
             clear_size = entry['size']
             clear_price = entry['price']
 
@@ -344,6 +362,11 @@ class Position(Struct):
 
             sign_change: bool = False
 
+            if accum_size == 0:
+                ppu_h.append(0)
+                asize_h.append(0)
+                continue
+
             if accum_size == 0:
                 ppu_h.append(0)
                 asize_h.append(0)

piker/ui/_app.py

@@ -118,17 +118,10 @@ async def _async_main(
         # godwidget.hbox.addWidget(search)
         godwidget.search = search
 
-        symbols: list[str] = []
-
-        for sym in syms:
-            symbol, _, provider = sym.rpartition('.')
-            symbols.append(symbol)
-
         # this internally starts a ``display_symbol_data()`` task above
         order_mode_ready = await godwidget.load_symbols(
-            provider,
-            symbols,
-            loglevel
+            fqsns=syms,
+            loglevel=loglevel,
         )
 
         # spin up a search engine for the local cached symbol set
@@ -185,8 +178,7 @@ def _main(
     tractor_kwargs,
 ) -> None:
     '''
-    Sync entry point to start a chart: a ``tractor`` + Qt runtime
-    entry point
+    Sync entry point to start a chart: a ``tractor`` + Qt runtime.
 
     '''
     run_qtractor(

piker/ui/_axes.py

@@ -18,6 +18,7 @@
 Chart axes graphics and behavior.
 
 """
+from __future__ import annotations
 from functools import lru_cache
 from typing import Optional, Callable
 from math import floor
@@ -27,6 +28,7 @@ import pyqtgraph as pg
 from PyQt5 import QtCore, QtGui, QtWidgets
 from PyQt5.QtCore import QPointF
 
+from . import _pg_overrides as pgo
 from ..data._source import float_digits
 from ._label import Label
 from ._style import DpiAwareFont, hcolor, _font
@@ -46,7 +48,7 @@ class Axis(pg.AxisItem):
     '''
     def __init__(
         self,
-        linkedsplits,
+        plotitem: pgo.PlotItem,
         typical_max_str: str = '100 000.000  ',
         text_color: str = 'bracket',
         lru_cache_tick_strings: bool = True,
@@ -61,36 +63,42 @@ class Axis(pg.AxisItem):
         # XXX: pretty sure this makes things slower
         # self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
 
-        self.linkedsplits = linkedsplits
+        self.pi = plotitem
         self._dpi_font = _font
 
         self.setTickFont(_font.font)
         font_size = self._dpi_font.font.pixelSize()
 
+        style_conf = {
+            'textFillLimits': [(0, 0.5)],
+            'tickFont': self._dpi_font.font,
+
+        }
+        text_offset = None
         if self.orientation in ('bottom',):
             text_offset = floor(0.25 * font_size)
 
         elif self.orientation in ('left', 'right'):
             text_offset = floor(font_size / 2)
 
-        self.setStyle(**{
-            'textFillLimits': [(0, 0.5)],
-            'tickFont': self._dpi_font.font,
-
+        if text_offset:
+            style_conf.update({
                 # offset of text *away from* axis line in px
                 # use approx. half the font pixel size (height)
                 'tickTextOffset': text_offset,
             })
 
+        self.setStyle(**style_conf)
         self.setTickFont(_font.font)
 
         # NOTE: this is for surrounding "border"
         self.setPen(_axis_pen)
 
         # this is the text color
-        # self.setTextPen(pg.mkPen(hcolor(text_color)))
         self.text_color = text_color
 
+        # generate a bounding rect based on sizing to a "typical"
+        # maximum length-ed string defined as init default.
         self.typical_br = _font._qfm.boundingRect(typical_max_str)
 
         # size the pertinent axis dimension to a "typical value"
@@ -102,6 +110,9 @@ class Axis(pg.AxisItem):
                 maxsize=2**20
             )(self.tickStrings)
 
+        # axis "sticky" labels
+        self._stickies: dict[str, YAxisLabel] = {}
+
     # NOTE: only overriden to cast tick values entries into tuples
     # for use with the lru caching.
     def tickValues(
@@ -139,6 +150,38 @@ class Axis(pg.AxisItem):
     def txt_offsets(self) -> tuple[int, int]:
         return tuple(self.style['tickTextOffset'])
 
+    def add_sticky(
+        self,
+        pi: pgo.PlotItem,
+        name: None | str = None,
+        digits: None | int = 2,
+        bg_color='default',
+        fg_color='black',
+
+    ) -> YAxisLabel:
+
+        # if the sticky is for our symbol
+        # use the tick size precision for display
+        name = name or pi.name
+        digits = digits or 2
+
+        # TODO: ``._ysticks`` should really be an attr on each
+        # ``PlotItem`` now instead of the containing widget (because of
+        # overlays) ?
+
+        # add y-axis "last" value label
+        sticky = self._stickies[name] = YAxisLabel(
+            pi=pi,
+            parent=self,
+            digits=digits,  # TODO: pass this from symbol data
+            opacity=0.9,  # slight see-through
+            bg_color=bg_color,
+            fg_color=fg_color,
+        )
+
+        pi.sigRangeChanged.connect(sticky.update_on_resize)
+        return sticky
+
 
 class PriceAxis(Axis):
 
@@ -200,7 +243,6 @@ class PriceAxis(Axis):
         self._min_tick = size
 
     def size_to_values(self) -> None:
-        # self.typical_br = _font._qfm.boundingRect(typical_max_str)
         self.setWidth(self.typical_br.width())
 
     # XXX: drop for now since it just eats up h space
@@ -255,28 +297,50 @@ class DynamicDateAxis(Axis):
 
     ) -> list[str]:
 
-        chart = self.linkedsplits.chart
-        flow = chart._flows[chart.name]
-        shm = flow.shm
-        bars = shm.array
+        # XX: ARGGGGG AG:LKSKDJF:LKJSDFD
+        chart = self.pi.chart_widget
+
+        viz = chart._vizs[chart.name]
+        shm = viz.shm
+        array = shm.array
+        times = array['time']
+        i_0, i_l = times[0], times[-1]
+
+        # edge cases
+        if (
+            not indexes
+            or
+            (indexes[0] < i_0
+             and indexes[-1] < i_l)
+            or
+            (indexes[0] > i_0
+             and indexes[-1] > i_l)
+        ):
+            return []
+
+        if viz.index_field == 'index':
+            arr_len = times.shape[0]
             first = shm._first.value
-
-        bars_len = len(bars)
-        times = bars['time']
-
-        epochs = times[list(
+            epochs = times[
+                list(
                     map(
                         int,
                         filter(
-                    lambda i: i > 0 and i < bars_len,
+                            lambda i: i > 0 and i < arr_len,
                             (i - first for i in indexes)
                         )
                     )
-        )]
+                )
+            ]
+        else:
+            epochs = list(map(int, indexes))
 
         # TODO: **don't** have this hard coded shift to EST
         # delay = times[-1] - times[-2]
-        dts = np.array(epochs, dtype='datetime64[s]')
+        dts = np.array(
+            epochs,
+            dtype='datetime64[s]',
+        )
 
         # see units listing:
         # https://numpy.org/devdocs/reference/arrays.datetime.html#datetime-units
@@ -294,24 +358,39 @@ class DynamicDateAxis(Axis):
         spacing: float,
 
     ) -> list[str]:
+
+        return self._indexes_to_timestrs(values)
+
+        # NOTE: handy for debugging the lru cache
         # info = self.tickStrings.cache_info()
         # print(info)
-        return self._indexes_to_timestrs(values)
 
 
 class AxisLabel(pg.GraphicsObject):
 
-    _x_margin = 0
-    _y_margin = 0
+    # relative offsets *OF* the bounding rect relative
+    # to parent graphics object.
+    # eg.  <parent>| => <_x_br_offset> => | <text> |
+    _x_br_offset: float = 0
+    _y_br_offset: float = 0
+
+    # relative offsets of text *within* bounding rect
+    # eg. | <_x_margin> => <text> |
+    _x_margin: float = 0
+    _y_margin: float = 0
+
+    # multiplier of the text content's height in order
+    # to force a larger (y-dimension) bounding rect.
+    _y_txt_h_scaling: float = 1
 
     def __init__(
         self,
         parent: pg.GraphicsItem,
         digits: int = 2,
 
-        bg_color: str = 'bracket',
+        bg_color: str = 'default',
         fg_color: str = 'black',
-        opacity: int = 1,  # XXX: seriously don't set this to 0
+        opacity: int = .8,  # XXX: seriously don't set this to 0
         font_size: str = 'default',
 
         use_arrow: bool = True,
@@ -322,6 +401,7 @@ class AxisLabel(pg.GraphicsObject):
         self.setParentItem(parent)
 
         self.setFlag(self.ItemIgnoresTransformations)
+        self.setZValue(100)
 
         # XXX: pretty sure this is faster
         self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
@@ -353,14 +433,14 @@ class AxisLabel(pg.GraphicsObject):
         p: QtGui.QPainter,
         opt: QtWidgets.QStyleOptionGraphicsItem,
         w: QtWidgets.QWidget
+
     ) -> None:
-        """Draw a filled rectangle based on the size of ``.label_str`` text.
+        '''
+        Draw a filled rectangle based on the size of ``.label_str`` text.
 
         Subtypes can customize further by overloading ``.draw()``.
 
-        """
-        # p.setCompositionMode(QtWidgets.QPainter.CompositionMode_SourceOver)
-
+        '''
         if self.label_str:
 
             # if not self.rect:
@@ -371,7 +451,11 @@ class AxisLabel(pg.GraphicsObject):
 
             p.setFont(self._dpifont.font)
             p.setPen(self.fg_color)
-            p.drawText(self.rect, self.text_flags, self.label_str)
+            p.drawText(
+                self.rect,
+                self.text_flags,
+                self.label_str,
+            )
 
     def draw(
         self,
@@ -379,6 +463,8 @@ class AxisLabel(pg.GraphicsObject):
         rect: QtCore.QRectF
     ) -> None:
 
+        p.setOpacity(self.opacity)
+
         if self._use_arrow:
             if not self.path:
                 self._draw_arrow_path()
@@ -386,15 +472,13 @@ class AxisLabel(pg.GraphicsObject):
             p.drawPath(self.path)
             p.fillPath(self.path, pg.mkBrush(self.bg_color))
 
-        # this adds a nice black outline around the label for some odd
-        # reason; ok by us
-        p.setOpacity(self.opacity)
-
         # this cause the L1 labels to glitch out if used in the subtype
         # and it will leave a small black strip with the arrow path if
         # done before the above
-        p.fillRect(self.rect, self.bg_color)
-
+        p.fillRect(
+            self.rect,
+            self.bg_color,
+        )
 
     def boundingRect(self):  # noqa
         '''
@@ -438,15 +522,18 @@ class AxisLabel(pg.GraphicsObject):
         txt_h, txt_w = txt_br.height(), txt_br.width()
         # print(f'wsw: {self._dpifont.boundingRect(" ")}')
 
-        # allow subtypes to specify a static width and height
+        # allow subtypes to override width and height
         h, w = self.size_hint()
-        # print(f'axis size: {self._parent.size()}')
-        # print(f'axis geo: {self._parent.geometry()}')
 
         self.rect = QtCore.QRectF(
-            0, 0,
+
+            # relative bounds offsets
+            self._x_br_offset,
+            self._y_br_offset,
+
             (w or txt_w) + self._x_margin / 2,
-            (h or txt_h) + self._y_margin / 2,
+
+            (h or txt_h) * self._y_txt_h_scaling + (self._y_margin / 2),
         )
         # print(self.rect)
         # hb = self.path.controlPointRect()
@@ -522,7 +609,7 @@ class XAxisLabel(AxisLabel):
 
 
 class YAxisLabel(AxisLabel):
-    _y_margin = 4
+    _y_margin: int = 4
 
     text_flags = (
         QtCore.Qt.AlignLeft
@@ -533,19 +620,19 @@ class YAxisLabel(AxisLabel):
 
     def __init__(
         self,
-        chart,
+        pi: pgo.PlotItem,
         *args,
         **kwargs
     ) -> None:
 
         super().__init__(*args, **kwargs)
 
-        self._chart = chart
-
-        chart.sigRangeChanged.connect(self.update_on_resize)
+        self._pi = pi
+        pi.sigRangeChanged.connect(self.update_on_resize)
 
         self._last_datum = (None, None)
 
+        self.x_offset = 0
         # pull text offset from axis from parent axis
         if getattr(self._parent, 'txt_offsets', False):
             self.x_offset, y_offset = self._parent.txt_offsets()
@@ -564,7 +651,8 @@ class YAxisLabel(AxisLabel):
         value: float,  # data for text
 
         # on odd dimension and/or adds nice black line
-        x_offset: Optional[int] = None
+        x_offset: int = 0,
+
     ) -> None:
 
         # this is read inside ``.paint()``
@@ -610,7 +698,7 @@ class YAxisLabel(AxisLabel):
             self._last_datum = (index, value)
 
         self.update_label(
-            self._chart.mapFromView(QPointF(index, value)),
+            self._pi.mapFromView(QPointF(index, value)),
             value
         )
 

piker/ui/_cursor.py

@@ -71,7 +71,7 @@ class LineDot(pg.CurvePoint):
 
         plot: ChartPlotWidget,  # type: ingore # noqa
         pos=None,
-        color: str = 'default_light',
+        color: str = 'bracket',
 
     ) -> None:
         # scale from dpi aware font size
@@ -198,12 +198,11 @@ class ContentsLabel(pg.LabelItem):
         self,
 
         name: str,
-        index: int,
+        ix: int,
         array: np.ndarray,
 
     ) -> None:
         # this being "html" is the dumbest shit :eyeroll:
-        first = array[0]['index']
 
         self.setText(
             "<b>i</b>:{index}<br/>"
@@ -216,7 +215,7 @@ class ContentsLabel(pg.LabelItem):
             "<b>C</b>:{}<br/>"
             "<b>V</b>:{}<br/>"
             "<b>wap</b>:{}".format(
-                *array[index - first][
+                *array[ix][
                     [
                         'time',
                         'open',
@@ -228,7 +227,7 @@ class ContentsLabel(pg.LabelItem):
                     ]
                 ],
                 name=name,
-                index=index,
+                index=ix,
             )
         )
 
@@ -236,14 +235,11 @@ class ContentsLabel(pg.LabelItem):
         self,
 
         name: str,
-        index: int,
+        ix: int,
         array: np.ndarray,
 
     ) -> None:
-
-        first = array[0]['index']
-        if index < array[-1]['index'] and index > first:
-            data = array[index - first][name]
+        data = array[ix][name]
         self.setText(f"{name}: {data:.2f}")
 
 
@@ -269,17 +265,20 @@ class ContentsLabels:
 
     def update_labels(
         self,
-        index: int,
+        x_in: int,
 
     ) -> None:
         for chart, name, label, update in self._labels:
 
-            flow = chart._flows[name]
-            array = flow.shm.array
+            viz = chart.get_viz(name)
+            array = viz.shm.array
+            index = array[viz.index_field]
+            start = index[0]
+            stop = index[-1]
 
             if not (
-                index >= 0
-                and index < array[-1]['index']
+                x_in >= start
+                and x_in <= stop
             ):
                 # out of range
                 print('WTF out of range?')
@@ -288,7 +287,10 @@ class ContentsLabels:
             # call provided update func with data point
             try:
                 label.show()
-                update(index, array)
+                ix = np.searchsorted(index, x_in)
+                if ix > len(array):
+                    breakpoint()
+                update(ix, array)
 
             except IndexError:
                 log.exception(f"Failed to update label: {name}")
@@ -349,7 +351,7 @@ class Cursor(pg.GraphicsObject):
         # XXX: not sure why these are instance variables?
         # It's not like we can change them on the fly..?
         self.pen = pg.mkPen(
-            color=hcolor('default'),
+            color=hcolor('bracket'),
             style=QtCore.Qt.DashLine,
         )
         self.lines_pen = pg.mkPen(
@@ -365,7 +367,7 @@ class Cursor(pg.GraphicsObject):
         self._lw = self.pixelWidth() * self.lines_pen.width()
 
         # xhair label's color name
-        self.label_color: str = 'default'
+        self.label_color: str = 'bracket'
 
         self._y_label_update: bool = True
 
@@ -418,7 +420,7 @@ class Cursor(pg.GraphicsObject):
         hl.hide()
 
         yl = YAxisLabel(
-            chart=plot,
+            pi=plot.plotItem,
             # parent=plot.getAxis('right'),
             parent=plot.pi_overlay.get_axis(plot.plotItem, 'right'),
             digits=digits or self.digits,
@@ -482,25 +484,32 @@ class Cursor(pg.GraphicsObject):
 
     def add_curve_cursor(
         self,
-        plot: ChartPlotWidget,  # noqa
+        chart: ChartPlotWidget,  # noqa
         curve: 'PlotCurveItem',  # noqa
 
     ) -> LineDot:
-        # if this plot contains curves add line dot "cursors" to denote
+        # if this chart contains curves add line dot "cursors" to denote
         # the current sample under the mouse
-        main_flow = plot._flows[plot.name]
+        main_viz = chart.get_viz(chart.name)
+
         # read out last index
-        i = main_flow.shm.array[-1]['index']
+        i = main_viz.shm.array[-1]['index']
         cursor = LineDot(
             curve,
             index=i,
-            plot=plot
+            plot=chart
         )
-        plot.addItem(cursor)
-        self.graphics[plot].setdefault('cursors', []).append(cursor)
+        chart.addItem(cursor)
+        self.graphics[chart].setdefault('cursors', []).append(cursor)
         return cursor
 
-    def mouseAction(self, action, plot):  # noqa
+    def mouseAction(
+        self,
+        action: str,
+        plot: ChartPlotWidget,
+
+    ) -> None:  # noqa
+
         log.debug(f"{(action, plot.name)}")
         if action == 'Enter':
             self.active_plot = plot

piker/ui/_curve.py

@@ -28,10 +28,7 @@ from PyQt5.QtWidgets import QGraphicsItem
 from PyQt5.QtCore import (
     Qt,
     QLineF,
-    QSizeF,
     QRectF,
-    # QRect,
-    QPointF,
 )
 from PyQt5.QtGui import (
     QPainter,
@@ -39,10 +36,6 @@ from PyQt5.QtGui import (
 )
 from .._profile import pg_profile_enabled, ms_slower_then
 from ._style import hcolor
-# from ._compression import (
-#     # ohlc_to_m4_line,
-#     ds_m4,
-# )
 from ..log import get_logger
 from .._profile import Profiler
 
@@ -58,7 +51,39 @@ _line_styles: dict[str, int] = {
 }
 
 
-class Curve(pg.GraphicsObject):
+class FlowGraphic(pg.GraphicsObject):
+    '''
+    Base class with minimal interface for `QPainterPath` implemented,
+    real-time updated "data flow" graphics.
+
+    See subtypes below.
+
+    '''
+    # sub-type customization methods
+    declare_paintables: Optional[Callable] = None
+    sub_paint: Optional[Callable] = None
+
+    # TODO: can we remove this?
+    # sub_br: Optional[Callable] = None
+
+    def x_uppx(self) -> int:
+
+        px_vecs = self.pixelVectors()[0]
+        if px_vecs:
+            return px_vecs.x()
+        else:
+            return 0
+
+    def x_last(self) -> float | None:
+        '''
+        Return the last most x value of the last line segment or if not
+        drawn yet, ``None``.
+
+        '''
+        return self._last_line.x1() if self._last_line else None
+
+
+class Curve(FlowGraphic):
     '''
     A faster, simpler, append friendly version of
     ``pyqtgraph.PlotCurveItem`` built for highly customizable real-time
@@ -75,7 +100,7 @@ class Curve(pg.GraphicsObject):
     lower level graphics data can be rendered in different threads and
     then read and drawn in this main thread without having to worry
     about dealing with Qt's concurrency primitives. See
-    ``piker.ui._flows.Renderer`` for details and logic related to lower
+    ``piker.ui._render.Renderer`` for details and logic related to lower
     level path generation and incremental update. The main differences in
     the path generation code include:
 
@@ -88,11 +113,6 @@ class Curve(pg.GraphicsObject):
 
     '''
 
-    # sub-type customization methods
-    sub_br: Optional[Callable] = None
-    sub_paint: Optional[Callable] = None
-    declare_paintables: Optional[Callable] = None
-
     def __init__(
         self,
         *args,
@@ -102,7 +122,6 @@ class Curve(pg.GraphicsObject):
         fill_color: Optional[str] = None,
         style: str = 'solid',
         name: Optional[str] = None,
-        use_fpath: bool = True,
 
         **kwargs
 
@@ -117,11 +136,11 @@ class Curve(pg.GraphicsObject):
         # self._last_cap: int = 0
         self.path: Optional[QPainterPath] = None
 
-        # additional path used for appends which tries to avoid
-        # triggering an update/redraw of the presumably larger
-        # historical ``.path`` above.
-        self.use_fpath = use_fpath
-        self.fast_path: Optional[QPainterPath] = None
+        # additional path that can be optionally used for appends which
+        # tries to avoid triggering an update/redraw of the presumably
+        # larger historical ``.path`` above. the flag to enable
+        # this behaviour is found in `Renderer.render()`.
+        self.fast_path: QPainterPath | None = None
 
         # TODO: we can probably just dispense with the parent since
         # we're basically only using the pen setting now...
@@ -140,9 +159,7 @@ class Curve(pg.GraphicsObject):
         # self.last_step_pen = pg.mkPen(hcolor(color), width=2)
         self.last_step_pen = pg.mkPen(pen, width=2)
 
-        # self._last_line: Optional[QLineF] = None
-        self._last_line = QLineF()
-        self._last_w: float = 1
+        self._last_line: QLineF = QLineF()
 
         # flat-top style histogram-like discrete curve
         # self._step_mode: bool = step_mode
@@ -163,51 +180,19 @@ class Curve(pg.GraphicsObject):
         # endpoint (something we saw on trade rate curves)
         self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
 
-        # XXX: see explanation for different caching modes:
-        # https://stackoverflow.com/a/39410081
-        # seems to only be useful if we don't re-generate the entire
-        # QPainterPath every time
-        # curve.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
-
+        # XXX-NOTE-XXX: graphics caching.
+        # see explanation for different caching modes:
+        # https://stackoverflow.com/a/39410081 seems to only be useful
+        # if we don't re-generate the entire QPainterPath every time
         # don't ever use this - it's a colossal nightmare of artefacts
         # and is disastrous for performance.
-        # curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache)
+        # self.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache)
 
         # allow sub-type customization
         declare = self.declare_paintables
         if declare:
             declare()
 
-    # TODO: probably stick this in a new parent
-    # type which will contain our own version of
-    # what ``PlotCurveItem`` had in terms of base
-    # functionality? A `FlowGraphic` maybe?
-    def x_uppx(self) -> int:
-
-        px_vecs = self.pixelVectors()[0]
-        if px_vecs:
-            xs_in_px = px_vecs.x()
-            return round(xs_in_px)
-        else:
-            return 0
-
-    def px_width(self) -> float:
-
-        vb = self.getViewBox()
-        if not vb:
-            return 0
-
-        vr = self.viewRect()
-        l, r = int(vr.left()), int(vr.right())
-
-        start, stop = self._xrange
-        lbar = max(l, start)
-        rbar = min(r, stop)
-
-        return vb.mapViewToDevice(
-            QLineF(lbar, 0, rbar, 0)
-        ).length()
-
     # XXX: lol brutal, the internals of `CurvePoint` (inherited by
     # our `LineDot`) required ``.getData()`` to work..
     def getData(self):
@@ -231,8 +216,8 @@ class Curve(pg.GraphicsObject):
             self.path.clear()
 
             if self.fast_path:
-                # self.fast_path.clear()
-                self.fast_path = None
+                self.fast_path.clear()
+                # self.fast_path = None
 
     @cm
     def reset_cache(self) -> None:
@@ -252,77 +237,65 @@ class Curve(pg.GraphicsObject):
             self.boundingRect = self._path_br
             return self._path_br()
 
+    # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect
     def _path_br(self):
         '''
         Post init ``.boundingRect()```.
 
         '''
-        # hb = self.path.boundingRect()
-        hb = self.path.controlPointRect()
-        hb_size = hb.size()
-
-        fp = self.fast_path
-        if fp:
-            fhb = fp.controlPointRect()
-            hb_size = fhb.size() + hb_size
-
-        # print(f'hb_size: {hb_size}')
-
-        # if self._last_step_rect:
-        #     hb_size += self._last_step_rect.size()
-
-        # if self._line:
-        #     br = self._last_step_rect.bottomRight()
-
-        # tl = QPointF(
-        #     # self._vr[0],
-        #     # hb.topLeft().y(),
-        #     # 0,
-        #     # hb_size.height() + 1
+        # profiler = Profiler(
+        #     msg=f'Curve.boundingRect(): `{self._name}`',
+        #     disabled=not pg_profile_enabled(),
+        #     ms_threshold=ms_slower_then,
         # )
+        pr = self.path.controlPointRect()
+        hb_tl, hb_br = (
+            pr.topLeft(),
+            pr.bottomRight(),
+        )
+        mn_y = hb_tl.y()
+        mx_y = hb_br.y()
+        most_left = hb_tl.x()
+        most_right = hb_br.x()
+        # profiler('calc path vertices')
 
-        #     br = self._last_step_rect.bottomRight()
+        # TODO: if/when we get fast path appends working in the
+        # `Renderer`, then we might need to actually use this..
+        # fp = self.fast_path
+        # if fp:
+        #     fhb = fp.controlPointRect()
+        #     # hb_size = fhb.size() + hb_size
+        #     br = pr.united(fhb)
 
-        w = hb_size.width()
-        h = hb_size.height()
+        # XXX: *was* a way to allow sub-types to extend the
+        # boundingrect calc, but in the one use case for a step curve
+        # doesn't seem like we need it as long as the last line segment
+        # is drawn as it is?
+
+        # sbr = self.sub_br
+        # if sbr:
+        #     # w, h = self.sub_br(w, h)
+        #     sub_br = sbr()
+        #     br = br.united(sub_br)
 
-        sbr = self.sub_br
-        if sbr:
-            w, h = self.sub_br(w, h)
-        else:
         # assume plain line graphic and use
         # default unit step in each direction.
+        ll = self._last_line
+        y1, y2 = ll.y1(), ll.y2()
+        x1, x2 = ll.x1(), ll.x2()
 
-            # only on a plane line do we include
-            # and extra index step's worth of width
-            # since in the step case the end of the curve
-            # actually terminates earlier so we don't need
-            # this for the last step.
-            w += self._last_w
-            # ll = self._last_line
-            h += 1  # ll.y2() - ll.y1()
+        ymn = min(y1, y2, mn_y)
+        ymx = max(y1, y2, mx_y)
+        most_left = min(x1, x2, most_left)
+        most_right = max(x1, x2, most_right)
+        # profiler('calc last line vertices')
 
-        # br = QPointF(
-        #     self._vr[-1],
-        #     # tl.x() + w,
-        #     tl.y() + h,
-        # )
-
-        br = QRectF(
-
-            # top left
-            # hb.topLeft()
-            # tl,
-            QPointF(hb.topLeft()),
-
-            # br,
-            # total size
-            # QSizeF(hb_size)
-            # hb_size,
-            QSizeF(w, h)
+        return QRectF(
+            most_left,
+            ymn,
+            most_right - most_left + 1,
+            ymx,
         )
-        # print(f'bounding rect: {br}')
-        return br
 
     def paint(
         self,
@@ -340,7 +313,7 @@ class Curve(pg.GraphicsObject):
 
         sub_paint = self.sub_paint
         if sub_paint:
-            sub_paint(p, profiler)
+            sub_paint(p)
 
         p.setPen(self.last_step_pen)
         p.drawLine(self._last_line)
@@ -374,22 +347,30 @@ class Curve(pg.GraphicsObject):
         self,
         path: QPainterPath,
         src_data: np.ndarray,
-        render_data: np.ndarray,
         reset: bool,
         array_key: str,
+        index_field: str,
 
     ) -> None:
         # default line draw last call
         # with self.reset_cache():
-        x = render_data['index']
-        y = render_data[array_key]
+        x = src_data[index_field]
+        y = src_data[array_key]
+
+        x_last = x[-1]
+        x_2last = x[-2]
 
         # draw the "current" step graphic segment so it
         # lines up with the "middle" of the current
         # (OHLC) sample.
         self._last_line = QLineF(
-            x[-2], y[-2],
-            x[-1], y[-1],
+
+            # NOTE: currently we draw in x-domain
+            # from last datum to current such that
+            # the end of line touches the "beginning"
+            # of the current datum step span.
+            x_2last , y[-2],
+            x_last, y[-1],
         )
 
         return x, y
@@ -405,13 +386,13 @@ class FlattenedOHLC(Curve):
         self,
         path: QPainterPath,
         src_data: np.ndarray,
-        render_data: np.ndarray,
         reset: bool,
         array_key: str,
+        index_field: str,
 
     ) -> None:
         lasts = src_data[-2:]
-        x = lasts['index']
+        x = lasts[index_field]
         y = lasts['close']
 
         # draw the "current" step graphic segment so it
@@ -435,9 +416,9 @@ class StepCurve(Curve):
         self,
         path: QPainterPath,
         src_data: np.ndarray,
-        render_data: np.ndarray,
         reset: bool,
         array_key: str,
+        index_field: str,
 
         w: float = 0.5,
 
@@ -446,40 +427,31 @@ class StepCurve(Curve):
         # TODO: remove this and instead place all step curve
         # updating into pre-path data render callbacks.
         # full input data
-        x = src_data['index']
+        x = src_data[index_field]
         y = src_data[array_key]
 
         x_last = x[-1]
+        x_2last = x[-2]
         y_last = y[-1]
+        step_size = x_last - x_2last
 
         # lol, commenting this makes step curves
         # all "black" for me :eyeroll:..
         self._last_line = QLineF(
-            x_last - w, 0,
-            x_last + w, 0,
+            x_2last, 0,
+            x_last, 0,
         )
         self._last_step_rect = QRectF(
-            x_last - w, 0,
-            x_last + w, y_last,
+            x_last, 0,
+            step_size, y_last,
         )
         return x, y
 
     def sub_paint(
         self,
         p: QPainter,
-        profiler: Profiler,
 
     ) -> None:
         # p.drawLines(*tuple(filter(bool, self._last_step_lines)))
         # p.drawRect(self._last_step_rect)
         p.fillRect(self._last_step_rect, self._brush)
-        profiler('.fillRect()')
-
-    def sub_br(
-        self,
-        path_w: float,
-        path_h: float,
-
-    ) -> (float, float):
-        # passthrough
-        return path_w, path_h

piker/ui/_editors.py

@@ -377,7 +377,7 @@ class SelectRect(QtWidgets.QGraphicsRectItem):
         nbars = ixmx - ixmn + 1
 
         chart = self._chart
-        data = chart._flows[chart.name].shm.array[ixmn:ixmx]
+        data = chart.get_viz(chart.name).shm.array[ixmn:ixmx]
 
         if len(data):
             std = data['close'].std()

piker/ui/_fsp.py

@@ -42,6 +42,8 @@ from ..data._sharedmem import (
     _Token,
     try_read,
 )
+from ..data.feed import Flume
+from ..data._source import Symbol
 from ._chart import (
     ChartPlotWidget,
     LinkedSplits,
@@ -77,14 +79,14 @@ def has_vlm(ohlcv: ShmArray) -> bool:
 
 def update_fsp_chart(
     chart: ChartPlotWidget,
-    flow,
+    viz,
     graphics_name: str,
     array_key: Optional[str],
     **kwargs,
 
 ) -> None:
 
-    shm = flow.shm
+    shm = viz.shm
     if not shm:
         return
 
@@ -110,7 +112,8 @@ def update_fsp_chart(
     # sub-charts reference it under different 'named charts'.
 
     # read from last calculated value and update any label
-    last_val_sticky = chart._ysticks.get(graphics_name)
+    last_val_sticky = chart.plotItem.getAxis(
+        'right')._stickies.get(graphics_name)
     if last_val_sticky:
         last = last_row[array_key]
         last_val_sticky.update_from_data(-1, last)
@@ -211,7 +214,7 @@ async def open_fsp_actor_cluster(
 async def run_fsp_ui(
 
     linkedsplits: LinkedSplits,
-    shm: ShmArray,
+    flume: Flume,
     started: trio.Event,
     target: Fsp,
     conf: dict[str, dict],
@@ -248,9 +251,11 @@ async def run_fsp_ui(
             else:
                 chart = linkedsplits.subplots[overlay_with]
 
+            shm = flume.rt_shm
             chart.draw_curve(
-                name=name,
-                shm=shm,
+                name,
+                shm,
+                flume,
                 overlay=True,
                 color='default_light',
                 array_key=name,
@@ -260,8 +265,9 @@ async def run_fsp_ui(
         else:
             # create a new sub-chart widget for this fsp
             chart = linkedsplits.add_plot(
-                name=name,
-                shm=shm,
+                name,
+                shm,
+                flume,
 
                 array_key=name,
                 sidepane=sidepane,
@@ -283,7 +289,7 @@ async def run_fsp_ui(
         # first UI update, usually from shm pushed history
         update_fsp_chart(
             chart,
-            chart._flows[array_key],
+            chart.get_viz(array_key),
             name,
             array_key=array_key,
         )
@@ -351,6 +357,9 @@ async def run_fsp_ui(
         #         last = time.time()
 
 
+# TODO: maybe this should be our ``Viz`` type since it maps
+# one flume to the next? The machinery for task/actor mgmt should
+# be part of the instantiation API?
 class FspAdmin:
     '''
     Client API for orchestrating FSP actors and displaying
@@ -362,7 +371,7 @@ class FspAdmin:
         tn: trio.Nursery,
         cluster: dict[str, tractor.Portal],
         linked: LinkedSplits,
-        src_shm: ShmArray,
+        flume: Flume,
 
     ) -> None:
         self.tn = tn
@@ -374,7 +383,11 @@ class FspAdmin:
             tuple[tractor.MsgStream, ShmArray]
         ] = {}
         self._flow_registry: dict[_Token, str] = {}
-        self.src_shm = src_shm
+
+        # TODO: make this a `.src_flume` and add
+        # a `dst_flume`?
+        # (=> but then wouldn't this be the most basic `Viz`?)
+        self.flume = flume
 
     def rr_next_portal(self) -> tractor.Portal:
         name, portal = next(self._rr_next_actor)
@@ -387,7 +400,7 @@ class FspAdmin:
         complete: trio.Event,
         started: trio.Event,
         fqsn: str,
-        dst_shm: ShmArray,
+        dst_fsp_flume: Flume,
         conf: dict,
         target: Fsp,
         loglevel: str,
@@ -408,9 +421,10 @@ class FspAdmin:
                 # data feed key
                 fqsn=fqsn,
 
+                # TODO: pass `Flume.to_msg()`s here?
                 # mems
-                src_shm_token=self.src_shm.token,
-                dst_shm_token=dst_shm.token,
+                src_shm_token=self.flume.rt_shm.token,
+                dst_shm_token=dst_fsp_flume.rt_shm.token,
 
                 # target
                 ns_path=ns_path,
@@ -427,12 +441,14 @@ class FspAdmin:
             ctx.open_stream() as stream,
         ):
 
+            dst_fsp_flume.stream: tractor.MsgStream = stream
+
             # register output data
             self._registry[
                 (fqsn, ns_path)
             ] = (
                 stream,
-                dst_shm,
+                dst_fsp_flume.rt_shm,
                 complete
             )
 
@@ -467,9 +483,9 @@ class FspAdmin:
         worker_name: Optional[str] = None,
         loglevel: str = 'info',
 
-    ) -> (ShmArray, trio.Event):
+    ) -> (Flume, trio.Event):
 
-        fqsn = self.linked.symbol.front_fqsn()
+        fqsn = self.flume.symbol.fqsn
 
         # allocate an output shm array
         key, dst_shm, opened = maybe_mk_fsp_shm(
@@ -477,8 +493,28 @@ class FspAdmin:
             target=target,
             readonly=True,
         )
+
+        portal = self.cluster.get(worker_name) or self.rr_next_portal()
+        provider_tag = portal.channel.uid
+
+        symbol = Symbol(
+            key=key,
+            broker_info={
+                provider_tag: {'asset_type': 'fsp'},
+            },
+        )
+        dst_fsp_flume = Flume(
+            symbol=symbol,
+            _rt_shm_token=dst_shm.token,
+            first_quote={},
+
+            # set to 0 presuming for now that we can't load
+            # FSP history (though we should eventually).
+            izero_hist=0,
+            izero_rt=0,
+        )
         self._flow_registry[(
-            self.src_shm._token,
+            self.flume.rt_shm._token,
             target.name
         )] = dst_shm._token
 
@@ -487,7 +523,6 @@ class FspAdmin:
         #         f'Already started FSP `{fqsn}:{func_name}`'
         #     )
 
-        portal = self.cluster.get(worker_name) or self.rr_next_portal()
         complete = trio.Event()
         started = trio.Event()
         self.tn.start_soon(
@@ -496,13 +531,13 @@ class FspAdmin:
             complete,
             started,
             fqsn,
-            dst_shm,
+            dst_fsp_flume,
             conf,
             target,
             loglevel,
         )
 
-        return dst_shm, started
+        return dst_fsp_flume, started
 
     async def open_fsp_chart(
         self,
@@ -514,7 +549,7 @@ class FspAdmin:
 
     ) -> (trio.Event, ChartPlotWidget):
 
-        shm, started = await self.start_engine_task(
+        flume, started = await self.start_engine_task(
             target,
             conf,
             loglevel,
@@ -526,7 +561,7 @@ class FspAdmin:
                 run_fsp_ui,
 
                 self.linked,
-                shm,
+                flume,
                 started,
                 target,
 
@@ -540,7 +575,7 @@ class FspAdmin:
 @acm
 async def open_fsp_admin(
     linked: LinkedSplits,
-    src_shm: ShmArray,
+    flume: Flume,
     **kwargs,
 
 ) -> AsyncGenerator[dict, dict[str, tractor.Portal]]:
@@ -561,7 +596,7 @@ async def open_fsp_admin(
             tn,
             cluster_map,
             linked,
-            src_shm,
+            flume,
         )
         try:
             yield admin
@@ -575,7 +610,7 @@ async def open_fsp_admin(
 async def open_vlm_displays(
 
     linked: LinkedSplits,
-    ohlcv: ShmArray,
+    flume: Flume,
     dvlm: bool = True,
 
     task_status: TaskStatus[ChartPlotWidget] = trio.TASK_STATUS_IGNORED,
@@ -597,6 +632,8 @@ async def open_vlm_displays(
     sig = inspect.signature(flow_rates.func)
     params = sig.parameters
 
+    ohlcv: ShmArray = flume.rt_shm
+
     async with (
         open_fsp_sidepane(
             linked, {
@@ -616,7 +653,7 @@ async def open_vlm_displays(
                 }
             },
         ) as sidepane,
-        open_fsp_admin(linked, ohlcv) as admin,
+        open_fsp_admin(linked, flume) as admin,
     ):
         # TODO: support updates
         # period_field = sidepane.fields['period']
@@ -624,14 +661,21 @@ async def open_vlm_displays(
         #     str(period_param.default)
         # )
 
+        # use slightly less light (then bracket) gray
+        # for volume from "main exchange" and a more "bluey"
+        # gray for "dark" vlm.
+        vlm_color = 'i3'
+        dark_vlm_color = 'charcoal'
+
         # built-in vlm which we plot ASAP since it's
         # usually data provided directly with OHLC history.
         shm = ohlcv
         ohlc_chart = linked.chart
 
-        chart = linked.add_plot(
+        vlm_chart = linked.add_plot(
             name='volume',
             shm=shm,
+            flume=flume,
 
             array_key='volume',
             sidepane=sidepane,
@@ -644,8 +688,12 @@ async def open_vlm_displays(
             # the curve item internals are pretty convoluted.
             style='step',
         )
+        vlm_chart.view.enable_auto_yrange()
+
+        # back-link the volume chart to trigger y-autoranging
+        # in the ohlc (parent) chart.
         ohlc_chart.view.enable_auto_yrange(
-            src_vb=chart.view,
+            src_vb=vlm_chart.view,
         )
 
         # force 0 to always be in view
@@ -654,7 +702,7 @@ async def open_vlm_displays(
 
         ) -> tuple[float, float]:
             '''
-            Flows "group" maxmin loop; assumes all named flows
+            Viz "group" maxmin loop; assumes all named flows
             are in the same co-domain and thus can be sorted
             as one set.
 
@@ -667,7 +715,7 @@ async def open_vlm_displays(
             '''
             mx = 0
             for name in names:
-                ymn, ymx = chart.maxmin(name=name)
+                ymn, ymx = vlm_chart.maxmin(name=name)
                 mx = max(mx, ymx)
 
             return 0, mx
@@ -675,40 +723,40 @@ async def open_vlm_displays(
         # TODO: fix the x-axis label issue where if you put
         # the axis on the left it's totally not lined up...
         # show volume units value on LHS (for dinkus)
-        # chart.hideAxis('right')
-        # chart.showAxis('left')
+        # vlm_chart.hideAxis('right')
+        # vlm_chart.showAxis('left')
 
         # send back new chart to caller
-        task_status.started(chart)
+        task_status.started(vlm_chart)
 
         # should **not** be the same sub-chart widget
-        assert chart.name != linked.chart.name
+        assert vlm_chart.name != linked.chart.name
 
         # sticky only on sub-charts atm
-        last_val_sticky = chart._ysticks[chart.name]
+        last_val_sticky = vlm_chart.plotItem.getAxis(
+            'right')._stickies.get(vlm_chart.name)
 
         # read from last calculated value
         value = shm.array['volume'][-1]
 
         last_val_sticky.update_from_data(-1, value)
 
-        vlm_curve = chart.update_graphics_from_flow(
+        vlm_curve = vlm_chart.update_graphics_from_flow(
             'volume',
-            # shm.array,
         )
 
         # size view to data once at outset
-        chart.view._set_yrange()
+        vlm_chart.view._set_yrange()
 
         # add axis title
-        axis = chart.getAxis('right')
+        axis = vlm_chart.getAxis('right')
         axis.set_title(' vlm')
 
         if dvlm:
 
             tasks_ready = []
             # spawn and overlay $ vlm on the same subchart
-            dvlm_shm, started = await admin.start_engine_task(
+            dvlm_flume, started = await admin.start_engine_task(
                 dolla_vlm,
 
                 {  # fsp engine conf
@@ -727,7 +775,7 @@ async def open_vlm_displays(
             # FIXME: we should error on starting the same fsp right
             # since it might collide with existing shm.. or wait we
             # had this before??
-            # dolla_vlm,
+            # dolla_vlm
 
             tasks_ready.append(started)
             # profiler(f'created shm for fsp actor: {display_name}')
@@ -741,22 +789,27 @@ async def open_vlm_displays(
             # XXX: the main chart already contains a vlm "units" axis
             # so here we add an overlay wth a y-range in
             # $ liquidity-value units (normally a fiat like USD).
-            dvlm_pi = chart.overlay_plotitem(
+            dvlm_pi = vlm_chart.overlay_plotitem(
                 'dolla_vlm',
                 index=0,  # place axis on inside (nearest to chart)
+
                 axis_title=' $vlm',
-                axis_side='right',
+                axis_side='left',
+
                 axis_kwargs={
                     'typical_max_str': ' 100.0 M ',
                     'formatter': partial(
                         humanize,
                         digits=2,
                     ),
+                    'text_color': vlm_color,
                 },
             )
 
-            dvlm_pi.hideAxis('left')
+            # TODO: should this maybe be implicit based on input args to
+            # `.overlay_plotitem()` above?
             dvlm_pi.hideAxis('bottom')
+
             # all to be overlayed curve names
             fields = [
                'dolla_vlm',
@@ -781,17 +834,12 @@ async def open_vlm_displays(
             # add custom auto range handler
             dvlm_pi.vb._maxmin = group_mxmn
 
-            # use slightly less light (then bracket) gray
-            # for volume from "main exchange" and a more "bluey"
-            # gray for "dark" vlm.
-            vlm_color = 'i3'
-            dark_vlm_color = 'charcoal'
-
             # add dvlm (step) curves to common view
             def chart_curves(
                 names: list[str],
                 pi: pg.PlotItem,
                 shm: ShmArray,
+                flume: Flume,
                 step_mode: bool = False,
                 style: str = 'solid',
 
@@ -805,9 +853,13 @@ async def open_vlm_displays(
                     else:
                         color = 'bracket'
 
-                    curve, _ = chart.draw_curve(
-                        name=name,
-                        shm=shm,
+                    assert isinstance(shm, ShmArray)
+                    assert isinstance(flume, Flume)
+
+                    viz = vlm_chart.draw_curve(
+                        name,
+                        shm,
+                        flume,
                         array_key=name,
                         overlay=pi,
                         color=color,
@@ -815,29 +867,24 @@ async def open_vlm_displays(
                         style=style,
                         pi=pi,
                     )
-
-                    # TODO: we need a better API to do this..
-                    # specially store ref to shm for lookup in display loop
-                    # since only a placeholder of `None` is entered in
-                    # ``.draw_curve()``.
-                    flow = chart._flows[name]
-                    assert flow.plot is pi
+                    assert viz.plot is pi
 
             chart_curves(
                 fields,
                 dvlm_pi,
-                dvlm_shm,
+                dvlm_flume.rt_shm,
+                dvlm_flume,
                 step_mode=True,
             )
 
             # spawn flow rates fsp **ONLY AFTER** the 'dolla_vlm' fsp is
             # up since this one depends on it.
 
-            fr_shm, started = await admin.start_engine_task(
+            fr_flume, started = await admin.start_engine_task(
                 flow_rates,
                 {  # fsp engine conf
                     'func_name': 'flow_rates',
-                    'zero_on_step': False,
+                    'zero_on_step': True,
                 },
                 # loglevel,
             )
@@ -846,7 +893,7 @@ async def open_vlm_displays(
             # chart_curves(
             #     dvlm_rate_fields,
             #     dvlm_pi,
-            #     fr_shm,
+            #     fr_flume.rt_shm,
             # )
 
             # TODO: is there a way to "sync" the dual axes such that only
@@ -855,24 +902,24 @@ async def open_vlm_displays(
             # displayed and the curves are effectively the same minus
             # liquidity events (well at least on low OHLC periods - 1s).
             vlm_curve.hide()
-            chart.removeItem(vlm_curve)
-            vflow = chart._flows['volume']
-            vflow.render = False
+            vlm_chart.removeItem(vlm_curve)
+            vlm_viz = vlm_chart._vizs['volume']
+            vlm_viz.render = False
 
             # avoid range sorting on volume once disabled
-            chart.view.disable_auto_yrange()
+            vlm_chart.view.disable_auto_yrange()
 
             # Trade rate overlay
             # XXX: requires an additional overlay for
             # a trades-per-period (time) y-range.
-            tr_pi = chart.overlay_plotitem(
+            tr_pi = vlm_chart.overlay_plotitem(
                 'trade_rates',
 
                 # TODO: dynamically update period (and thus this axis?)
                 # title from user input.
                 axis_title='clears',
-
                 axis_side='left',
+
                 axis_kwargs={
                     'typical_max_str': ' 10.0 M ',
                     'formatter': partial(
@@ -894,7 +941,8 @@ async def open_vlm_displays(
             chart_curves(
                 trade_rate_fields,
                 tr_pi,
-                fr_shm,
+                fr_flume.rt_shm,
+                fr_flume,
                 # step_mode=True,
 
                 # dashed line to represent "individual trades" being
@@ -928,7 +976,7 @@ async def open_vlm_displays(
 async def start_fsp_displays(
 
     linked: LinkedSplits,
-    ohlcv: ShmArray,
+    flume: Flume,
     group_status_key: str,
     loglevel: str,
 
@@ -971,7 +1019,10 @@ async def start_fsp_displays(
     async with (
 
         # NOTE: this admin internally opens an actor cluster
-        open_fsp_admin(linked, ohlcv) as admin,
+        open_fsp_admin(
+            linked,
+            flume,
+        ) as admin,
     ):
         statuses = []
         for target, conf in fsp_conf.items():

piker/ui/_interaction.py

@@ -21,7 +21,11 @@ Chart view box primitives
 from __future__ import annotations
 from contextlib import asynccontextmanager
 import time
-from typing import Optional, Callable
+from typing import (
+    Optional,
+    Callable,
+    TYPE_CHECKING,
+)
 
 import pyqtgraph as pg
 # from pyqtgraph.GraphicsScene import mouseEvents
@@ -39,6 +43,9 @@ from .._profile import pg_profile_enabled, ms_slower_then
 from ._editors import SelectRect
 from . import _event
 
+if TYPE_CHECKING:
+    from ._chart import ChartPlotWidget
+
 
 log = get_logger(__name__)
 
@@ -76,7 +83,6 @@ async def handle_viewmode_kb_inputs(
     pressed: set[str] = set()
 
     last = time.time()
-    trigger_mode: str
     action: str
 
     on_next_release: Optional[Callable] = None
@@ -375,7 +381,7 @@ class ChartView(ViewBox):
         )
 
         self.linked = None
-        self._chart: 'ChartPlotWidget' = None  # noqa
+        self._chart: ChartPlotWidget | None = None  # noqa
 
         # add our selection box annotator
         self.select_box = SelectRect(self)
@@ -446,11 +452,11 @@ class ChartView(ViewBox):
             yield self
 
     @property
-    def chart(self) -> 'ChartPlotWidget':  # type: ignore # noqa
+    def chart(self) -> ChartPlotWidget:  # type: ignore # noqa
         return self._chart
 
     @chart.setter
-    def chart(self, chart: 'ChartPlotWidget') -> None:  # type: ignore # noqa
+    def chart(self, chart: ChartPlotWidget) -> None:  # type: ignore # noqa
         self._chart = chart
         self.select_box.chart = chart
         if self._maxmin is None:
@@ -468,7 +474,6 @@ class ChartView(ViewBox):
         self,
         ev,
         axis=None,
-        # relayed_from: ChartView = None,
     ):
         '''
         Override "center-point" location for scrolling.
@@ -483,7 +488,6 @@ class ChartView(ViewBox):
         if (
             not linked
         ):
-            # print(f'{self.name} not linked but relay from {relayed_from.name}')
             return
 
         if axis in (0, 1):
@@ -495,18 +499,19 @@ class ChartView(ViewBox):
         chart = self.linked.chart
 
         # don't zoom more then the min points setting
-        l, lbar, rbar, r = chart.bars_range()
-        # vl = r - l
+        viz = chart.get_viz(chart.name)
+        vl, lbar, rbar, vr = viz.bars_range()
 
-        # if ev.delta() > 0 and vl <= _min_points_to_show:
-        #     log.debug("Max zoom bruh...")
+        # TODO: max/min zoom limits incorporating time step size.
+        # rl = vr - vl
+        # if ev.delta() > 0 and rl <= _min_points_to_show:
+        #     log.warning("Max zoom bruh...")
         #     return
-
         # if (
         #     ev.delta() < 0
-        #     and vl >= len(chart._flows[chart.name].shm.array) + 666
+        #     and rl >= len(chart._vizs[chart.name].shm.array) + 666
         # ):
-        #     log.debug("Min zoom bruh...")
+        #     log.warning("Min zoom bruh...")
         #     return
 
         # actual scaling factor
@@ -537,49 +542,17 @@ class ChartView(ViewBox):
             self.scaleBy(s, center)
 
         else:
-
-            # center = pg.Point(
-            #     fn.invertQTransform(self.childGroup.transform()).map(ev.pos())
-            # )
-
-            # XXX: scroll "around" the right most element in the view
-            # which stays "pinned" in place.
-
-            # furthest_right_coord = self.boundingRect().topRight()
-
-            # yaxis = pg.Point(
-            #     fn.invertQTransform(
-            #         self.childGroup.transform()
-            #     ).map(furthest_right_coord)
-            # )
-
-            # This seems like the most "intuitive option, a hybrid of
-            # tws and tv styles
-            last_bar = pg.Point(int(rbar)) + 1
-
-            ryaxis = chart.getAxis('right')
-            r_axis_x = ryaxis.pos().x()
-
-            end_of_l1 = pg.Point(
-                round(
-                    chart.cv.mapToView(
-                        pg.Point(r_axis_x - chart._max_l1_line_len)
-                        # QPointF(chart._max_l1_line_len, 0)
-                    ).x()
-                )
-            )  # .x()
-
-            # self.state['viewRange'][0][1] = end_of_l1
-            # focal = pg.Point((last_bar.x() + end_of_l1)/2)
-
+            # use right-most point of current curve graphic
+            xl = viz.graphics.x_last()
             focal = min(
-                last_bar,
-                end_of_l1,
-                key=lambda p: p.x()
+                xl,
+                vr,
             )
-            # focal = pg.Point(last_bar.x() + end_of_l1)
 
             self._resetTarget()
+
+            # NOTE: scroll "around" the right most datum-element in view
+            # gives the feeling of staying "pinned" in place.
             self.scaleBy(s, focal)
 
             # XXX: the order of the next 2 lines i'm pretty sure
@@ -605,21 +578,8 @@ class ChartView(ViewBox):
         self,
         ev,
         axis: Optional[int] = None,
-        # relayed_from: ChartView = None,
 
     ) -> None:
-        # if relayed_from:
-        #     print(f'PAN: {self.name} -> RELAYED FROM: {relayed_from.name}')
-
-        # NOTE since in the overlay case axes are already
-        # "linked" any x-range change will already be mirrored
-        # in all overlaid ``PlotItems``, so we need to simply
-        # ignore the signal here since otherwise we get N-calls
-        # from N-overlays resulting in an "accelerated" feeling
-        # panning motion instead of the expect linear shift.
-        # if relayed_from:
-        #     return
-
         pos = ev.pos()
         lastPos = ev.lastPos()
         dif = pos - lastPos
@@ -689,9 +649,6 @@ class ChartView(ViewBox):
 
             # PANNING MODE
             else:
-                # XXX: WHY
-                ev.accept()
-
                 try:
                     self.start_ic()
                 except RuntimeError:
@@ -723,6 +680,9 @@ class ChartView(ViewBox):
                     # self._ic = None
                     # self.chart.resume_all_feeds()
 
+                # XXX: WHY
+                ev.accept()
+
         # WEIRD "RIGHT-CLICK CENTER ZOOM" MODE
         elif button & QtCore.Qt.RightButton:
 
@@ -768,7 +728,11 @@ class ChartView(ViewBox):
         *,
 
         yrange: Optional[tuple[float, float]] = None,
-        range_margin: float = 0.06,
+
+        # NOTE: this value pairs (more or less) with L1 label text
+        # height offset from from the bid/ask lines.
+        range_margin: float = 0.09,
+
         bars_range: Optional[tuple[int, int, int, int]] = None,
 
         # flag to prevent triggering sibling charts from the same linked
@@ -821,16 +785,14 @@ class ChartView(ViewBox):
             # XXX: only compute the mxmn range
             # if none is provided as input!
             if not yrange:
-                # flow = chart._flows[name]
+                # flow = chart._vizs[name]
                 yrange = self._maxmin()
 
                 if yrange is None:
                     log.warning(f'No yrange provided for {name}!?')
-                    print(f"WTF NO YRANGE {name}")
                     return
 
             ylow, yhigh = yrange
-
             profiler(f'callback ._maxmin(): {yrange}')
 
             # view margins: stay within a % of the "true range"
@@ -912,7 +874,7 @@ class ChartView(ViewBox):
         graphics items which are our children.
 
         '''
-        graphics = [f.graphics for f in self._chart._flows.values()]
+        graphics = [f.graphics for f in self._chart._vizs.values()]
         if not graphics:
             return 0
 
@@ -925,7 +887,7 @@ class ChartView(ViewBox):
 
     def maybe_downsample_graphics(
         self,
-        autoscale_overlays: bool = True,
+        autoscale_overlays: bool = False,
     ):
         profiler = Profiler(
             msg=f'ChartView.maybe_downsample_graphics() for {self.name}',
@@ -948,7 +910,7 @@ class ChartView(ViewBox):
             plots |= linked.subplots
 
         for chart_name, chart in plots.items():
-            for name, flow in chart._flows.items():
+            for name, flow in chart._vizs.items():
 
                 if (
                     not flow.render
@@ -961,10 +923,7 @@ class ChartView(ViewBox):
 
                 # pass in no array which will read and render from the last
                 # passed array (normally provided by the display loop.)
-                chart.update_graphics_from_flow(
-                    name,
-                    use_vr=True,
-                )
+                chart.update_graphics_from_flow(name)
 
                 # for each overlay on this chart auto-scale the
                 # y-range to max-min values.

piker/ui/_l1.py

@@ -26,22 +26,24 @@ from PyQt5.QtCore import QPointF
 
 from ._axes import YAxisLabel
 from ._style import hcolor
+from ._pg_overrides import PlotItem
 
 
 class LevelLabel(YAxisLabel):
-    """Y-axis (vertically) oriented, horizontal label that sticks to
+    '''
+    Y-axis (vertically) oriented, horizontal label that sticks to
     where it's placed despite chart resizing and supports displaying
     multiple fields.
 
 
     TODO: replace the rectangle-text part with our new ``Label`` type.
 
-    """
-    _x_margin = 0
-    _y_margin = 0
+    '''
+    _x_br_offset: float = -16
+    _y_txt_h_scaling: float = 2
 
     # adjustment "further away from" anchor point
-    _x_offset = 9
+    _x_offset = 0
     _y_offset = 0
 
     # fields to be displayed in the label string
@@ -57,12 +59,12 @@ class LevelLabel(YAxisLabel):
         chart,
         parent,
 
-        color: str = 'bracket',
+        color: str = 'default_light',
 
         orient_v: str = 'bottom',
-        orient_h: str = 'left',
+        orient_h: str = 'right',
 
-        opacity: float = 0,
+        opacity: float = 1,
 
         # makes order line labels offset from their parent axis
         # such that they don't collide with the L1/L2 lines/prices
@@ -98,13 +100,15 @@ class LevelLabel(YAxisLabel):
 
         self._h_shift = {
             'left': -1.,
-            'right': 0.
+            'right': 0.,
         }[orient_h]
 
         self.fields = self._fields.copy()
         # ensure default format fields are in correct
         self.set_fmt_str(self._fmt_str, self.fields)
 
+        self.setZValue(10)
+
     @property
     def color(self):
         return self._hcolor
@@ -112,7 +116,10 @@ class LevelLabel(YAxisLabel):
     @color.setter
     def color(self, color: str) -> None:
         self._hcolor = color
-        self._pen = self.pen = pg.mkPen(hcolor(color))
+        self._pen = self.pen = pg.mkPen(
+            hcolor(color),
+            width=3,
+        )
 
     def update_on_resize(self, vr, r):
         """Tiis is a ``.sigRangeChanged()`` handler.
@@ -124,15 +131,16 @@ class LevelLabel(YAxisLabel):
         self,
         fields: dict = None,
     ) -> None:
-        """Update the label's text contents **and** position from
+        '''
+        Update the label's text contents **and** position from
         a view box coordinate datum.
 
-        """
+        '''
         self.fields.update(fields)
         level = self.fields['level']
 
         # map "level" to local coords
-        abs_xy = self._chart.mapFromView(QPointF(0, level))
+        abs_xy = self._pi.mapFromView(QPointF(0, level))
 
         self.update_label(
             abs_xy,
@@ -149,7 +157,7 @@ class LevelLabel(YAxisLabel):
         h, w = self.set_label_str(fields)
 
         if self._adjust_to_l1:
-            self._x_offset = self._chart._max_l1_line_len
+            self._x_offset = self._pi.chart_widget._max_l1_line_len
 
         self.setPos(QPointF(
             self._h_shift * (w + self._x_offset),
@@ -174,7 +182,8 @@ class LevelLabel(YAxisLabel):
         fields: dict,
     ):
         # use space as e3 delim
-        self.label_str = self._fmt_str.format(**fields).replace(',', ' ')
+        self.label_str = self._fmt_str.format(
+            **fields).replace(',', ' ')
 
         br = self.boundingRect()
         h, w = br.height(), br.width()
@@ -187,14 +196,14 @@ class LevelLabel(YAxisLabel):
         self,
         p: QtGui.QPainter,
         rect: QtCore.QRectF
-    ) -> None:
-        p.setPen(self._pen)
 
+    ) -> None:
+
+        p.setPen(self._pen)
         rect = self.rect
 
         if self._orient_v == 'bottom':
             lp, rp = rect.topLeft(), rect.topRight()
-            # p.drawLine(rect.topLeft(), rect.topRight())
 
         elif self._orient_v == 'top':
             lp, rp = rect.bottomLeft(), rect.bottomRight()
@@ -208,6 +217,11 @@ class LevelLabel(YAxisLabel):
             ])
         )
 
+        p.fillRect(
+            self.rect,
+            self.bg_color,
+        )
+
     def highlight(self, pen) -> None:
         self._pen = pen
         self.update()
@@ -236,43 +250,46 @@ class L1Label(LevelLabel):
         # Set a global "max L1 label length" so we can
         # look it up on order lines and adjust their
         # labels not to overlap with it.
-        chart = self._chart
+        chart = self._pi.chart_widget
         chart._max_l1_line_len: float = max(
             chart._max_l1_line_len,
-            w
+            w,
         )
 
         return h, w
 
 
 class L1Labels:
-    """Level 1 bid ask labels for dynamic update on price-axis.
+    '''
+    Level 1 bid ask labels for dynamic update on price-axis.
 
-    """
+    '''
     def __init__(
         self,
-        chart: 'ChartPlotWidget',  # noqa
+        plotitem: PlotItem,
         digits: int = 2,
         size_digits: int = 3,
         font_size: str = 'small',
     ) -> None:
 
-        self.chart = chart
+        chart = self.chart = plotitem.chart_widget
 
-        raxis = chart.getAxis('right')
+        raxis = plotitem.getAxis('right')
         kwargs = {
-            'chart': chart,
+            'chart': plotitem,
             'parent': raxis,
 
-            'opacity': 1,
+            'opacity': .9,
             'font_size': font_size,
-            'fg_color': chart.pen_color,
-            'bg_color': chart.view_color,
+            'fg_color': 'default_light',
+            'bg_color': chart.view_color,  # normally 'papas_special'
         }
 
+        # TODO: add humanized source-asset
+        # info format.
         fmt_str = (
-            ' {size:.{size_digits}f} x '
-            '{level:,.{level_digits}f} '
+            ' {size:.{size_digits}f} u'
+            # '{level:,.{level_digits}f} '
         )
         fields = {
             'level': 0,
@@ -285,12 +302,17 @@ class L1Labels:
             orient_v='bottom',
             **kwargs,
         )
-        bid.set_fmt_str(fmt_str=fmt_str, fields=fields)
+        bid.set_fmt_str(
+            fmt_str='\n' + fmt_str,
+            fields=fields,
+        )
         bid.show()
 
         ask = self.ask_label = L1Label(
             orient_v='top',
             **kwargs,
         )
-        ask.set_fmt_str(fmt_str=fmt_str, fields=fields)
+        ask.set_fmt_str(
+            fmt_str=fmt_str,
+            fields=fields)
         ask.show()

piker/ui/_label.py

@@ -233,6 +233,36 @@ class Label:
     def delete(self) -> None:
         self.vb.scene().removeItem(self.txt)
 
+    # NOTE: pulled out from ``ChartPlotWidget`` from way way old code.
+    # def _label_h(self, yhigh: float, ylow: float) -> float:
+    #     # compute contents label "height" in view terms
+    #     # to avoid having data "contents" overlap with them
+    #     if self._labels:
+    #         label = self._labels[self.name][0]
+
+    #         rect = label.itemRect()
+    #         tl, br = rect.topLeft(), rect.bottomRight()
+    #         vb = self.plotItem.vb
+
+    #         try:
+    #             # on startup labels might not yet be rendered
+    #             top, bottom = (vb.mapToView(tl).y(), vb.mapToView(br).y())
+
+    #             # XXX: magic hack, how do we compute exactly?
+    #             label_h = (top - bottom) * 0.42
+
+    #         except np.linalg.LinAlgError:
+    #             label_h = 0
+    #     else:
+    #         label_h = 0
+
+    #     # print(f'label height {self.name}: {label_h}')
+
+    #     if label_h > yhigh - ylow:
+    #         label_h = 0
+
+    #     print(f"bounds (ylow, yhigh): {(ylow, yhigh)}")
+
 
 class FormatLabel(QLabel):
     '''

piker/ui/_ohlc.py

@@ -25,10 +25,18 @@ from typing import (
 
 import numpy as np
 import pyqtgraph as pg
-from PyQt5 import QtCore, QtGui, QtWidgets
-from PyQt5.QtCore import QLineF, QPointF
+from PyQt5 import (
+    QtGui,
+    QtWidgets,
+)
+from PyQt5.QtCore import (
+    QLineF,
+    QRectF,
+)
+
 from PyQt5.QtGui import QPainterPath
 
+from ._curve import FlowGraphic
 from .._profile import pg_profile_enabled, ms_slower_then
 from ._style import hcolor
 from ..log import get_logger
@@ -44,7 +52,8 @@ log = get_logger(__name__)
 def bar_from_ohlc_row(
     row: np.ndarray,
     # 0.5 is no overlap between arms, 1.0 is full overlap
-    w: float = 0.43
+    bar_w: float,
+    bar_gap: float = 0.16
 
 ) -> tuple[QLineF]:
     '''
@@ -52,8 +61,7 @@ def bar_from_ohlc_row(
     OHLC "bar" for use in the "last datum" of a series.
 
     '''
-    open, high, low, close, index = row[
-        ['open', 'high', 'low', 'close', 'index']]
+    open, high, low, close, index = row
 
     # TODO: maybe consider using `QGraphicsLineItem` ??
     # gives us a ``.boundingRect()`` on the objects which may make
@@ -61,9 +69,11 @@ def bar_from_ohlc_row(
     # history path faster since it's done in C++:
     # https://doc.qt.io/qt-5/qgraphicslineitem.html
 
+    mid: float = (bar_w / 2) + index
+
     # high -> low vertical (body) line
     if low != high:
-        hl = QLineF(index, low, index, high)
+        hl = QLineF(mid, low, mid, high)
     else:
         # XXX: if we don't do it renders a weird rectangle?
         # see below for filtering this later...
@@ -74,15 +84,18 @@ def bar_from_ohlc_row(
     # the index's range according to the view mapping coordinates.
 
     # open line
-    o = QLineF(index - w, open, index, open)
+    o = QLineF(index + bar_gap, open, mid, open)
 
     # close line
-    c = QLineF(index, close, index + w, close)
+    c = QLineF(
+        mid, close,
+        index + bar_w - bar_gap, close,
+    )
 
     return [hl, o, c]
 
 
-class BarItems(pg.GraphicsObject):
+class BarItems(FlowGraphic):
     '''
     "Price range" bars graphics rendered from a OHLC sampled sequence.
 
@@ -91,8 +104,8 @@ class BarItems(pg.GraphicsObject):
         self,
         linked: LinkedSplits,
         plotitem: 'pg.PlotItem',  # noqa
-        pen_color: str = 'bracket',
-        last_bar_color: str = 'bracket',
+        color: str = 'bracket',
+        last_bar_color: str = 'original',
 
         name: Optional[str] = None,
 
@@ -101,21 +114,37 @@ class BarItems(pg.GraphicsObject):
         self.linked = linked
         # XXX: for the mega-lulz increasing width here increases draw
         # latency...  so probably don't do it until we figure that out.
-        self._color = pen_color
-        self.bars_pen = pg.mkPen(hcolor(pen_color), width=1)
+        self._color = color
+        self.bars_pen = pg.mkPen(hcolor(color), width=1)
         self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2)
         self._name = name
 
+        # XXX: causes this weird jitter bug when click-drag panning
+        # where the path curve will awkwardly flicker back and forth?
         self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
+
         self.path = QPainterPath()
-        self._last_bar_lines: Optional[tuple[QLineF, ...]] = None
+        self._last_bar_lines: tuple[QLineF, ...] | None = None
 
-    def x_uppx(self) -> int:
-        # we expect the downsample curve report this.
-        return 0
+    def x_last(self) -> None | float:
+        '''
+        Return the last most x value of the close line segment
+        or if not drawn yet, ``None``.
+
+        '''
+        if self._last_bar_lines:
+            close_arm_line = self._last_bar_lines[-1]
+            return close_arm_line.x2() if close_arm_line else None
+        else:
+            return None
 
-    def boundingRect(self):
     # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect
+    def boundingRect(self):
+        # profiler = Profiler(
+        #     msg=f'BarItems.boundingRect(): `{self._name}`',
+        #     disabled=not pg_profile_enabled(),
+        #     ms_threshold=ms_slower_then,
+        # )
 
         # TODO: Can we do rect caching to make this faster
         # like `pg.PlotCurveItem` does? In theory it's just
@@ -135,32 +164,37 @@ class BarItems(pg.GraphicsObject):
             hb.topLeft(),
             hb.bottomRight(),
         )
+        mn_y = hb_tl.y()
+        mx_y = hb_br.y()
+        most_left = hb_tl.x()
+        most_right = hb_br.x()
+        # profiler('calc path vertices')
 
         # need to include last bar height or BR will be off
-        mx_y = hb_br.y()
-        mn_y = hb_tl.y()
-
-        last_lines = self._last_bar_lines
+        # OHLC line segments: [hl, o, c]
+        last_lines: tuple[QLineF] | None = self._last_bar_lines
         if last_lines:
-            body_line = self._last_bar_lines[0]
-            if body_line:
-                mx_y = max(mx_y, max(body_line.y1(), body_line.y2()))
-                mn_y = min(mn_y, min(body_line.y1(), body_line.y2()))
+            (
+                hl,
+                o,
+                c,
+            ) = last_lines
+            most_right = c.x2() + 1
+            ymx = ymn = c.y2()
 
-        return QtCore.QRectF(
+            if hl:
+                y1, y2 = hl.y1(), hl.y2()
+                ymn = min(y1, y2)
+                ymx = max(y1, y2)
+                mx_y = max(ymx, mx_y)
+                mn_y = min(ymn, mn_y)
+                # profiler('calc last bar vertices')
 
-            # top left
-            QPointF(
-                hb_tl.x(),
+        return QRectF(
+            most_left,
             mn_y,
-            ),
-
-            # bottom right
-            QPointF(
-                hb_br.x() + 1,
-                mx_y,
-            )
-
+            most_right - most_left + 1,
+            mx_y - mn_y,
         )
 
     def paint(
@@ -197,29 +231,40 @@ class BarItems(pg.GraphicsObject):
         self,
         path: QPainterPath,
         src_data: np.ndarray,
-        render_data: np.ndarray,
         reset: bool,
         array_key: str,
-
-        fields: list[str] = [
-            'index',
-            'open',
-            'high',
-            'low',
-            'close',
-        ],
+        index_field: str,
 
     ) -> None:
 
         # relevant fields
+        fields: list[str] = [
+            'open',
+            'high',
+            'low',
+            'close',
+            index_field,
+        ]
         ohlc = src_data[fields]
-        last_row = ohlc[-1:]
+        # last_row = ohlc[-1:]
 
         # individual values
-        last_row = i, o, h, l, last = ohlc[-1]
+        last_row = o, h, l, last, i = ohlc[-1]
+
+        # times = src_data['time']
+        # if times[-1] - times[-2]:
+        #     breakpoint()
+
+        index = src_data[index_field]
+        step_size = index[-1] - index[-2]
 
         # generate new lines objects for updatable "current bar"
-        self._last_bar_lines = bar_from_ohlc_row(last_row)
+        bg: float = 0.16 * step_size
+        self._last_bar_lines = bar_from_ohlc_row(
+            last_row,
+            bar_w=step_size,
+            bar_gap=bg,
+        )
 
         # assert i == graphics.start_index - 1
         # assert i == last_index
@@ -234,10 +279,16 @@ class BarItems(pg.GraphicsObject):
         if l != h:  # noqa
 
             if body is None:
-                body = self._last_bar_lines[0] = QLineF(i, l, i, h)
+                body = self._last_bar_lines[0] = QLineF(
+                    i + bg, l,
+                    i + step_size - bg, h,
+                )
             else:
                 # update body
-                body.setLine(i, l, i, h)
+                body.setLine(
+                    body.x1(), l,
+                    body.x2(), h,
+                )
 
             # XXX: pretty sure this is causing an issue where the
             # bar has a large upward move right before the next
@@ -248,4 +299,5 @@ class BarItems(pg.GraphicsObject):
             # date / from some previous sample. It's weird though
             # because i've seen it do this to bars i - 3 back?
 
-        return ohlc['index'], ohlc['close']
+        # return ohlc['time'], ohlc['close']
+        return ohlc[index_field], ohlc['close']

piker/ui/_overlay.py

@@ -22,7 +22,6 @@ from collections import defaultdict
 from functools import partial
 from typing import (
     Callable,
-    Optional,
 )
 
 from pyqtgraph.graphicsItems.AxisItem import AxisItem
@@ -92,11 +91,11 @@ class ComposedGridLayout:
     '''
     def __init__(
         self,
-        item: PlotItem,
+        pi: PlotItem,
 
     ) -> None:
 
-        self.items: list[PlotItem] = []
+        self.pitems: list[PlotItem] = []
         self._pi2axes: dict[  # TODO: use a ``bidict`` here?
             int,
             dict[str, AxisItem],
@@ -125,7 +124,7 @@ class ComposedGridLayout:
 
             layout.setOrientation(orient)
 
-        self.insert_plotitem(0, item)
+        self.insert_plotitem(0, pi)
 
         # insert surrounding linear layouts into the parent pi's layout
         # such that additional axes can be appended arbitrarily without
@@ -135,13 +134,14 @@ class ComposedGridLayout:
             # TODO: do we need this?
             # axis should have been removed during insert above
             index = _axes_layout_indices[name]
-            axis = item.layout.itemAt(*index)
+            axis = pi.layout.itemAt(*index)
             if axis and axis.isVisible():
                 assert linlayout.itemAt(0) is axis
 
-            # item.layout.removeItem(axis)
-            item.layout.addItem(linlayout, *index)
-            layout = item.layout.itemAt(*index)
+            # XXX: see comment in ``.insert_plotitem()``...
+            # pi.layout.removeItem(axis)
+            pi.layout.addItem(linlayout, *index)
+            layout = pi.layout.itemAt(*index)
             assert layout is linlayout
 
     def _register_item(
@@ -157,14 +157,14 @@ class ComposedGridLayout:
             self._pi2axes.setdefault(name, {})[index] = axis
 
         # enter plot into list for index tracking
-        self.items.insert(index, plotitem)
+        self.pitems.insert(index, plotitem)
 
     def insert_plotitem(
         self,
         index: int,
         plotitem: PlotItem,
 
-    ) -> (int, int):
+    ) -> tuple[int, list[AxisItem]]:
         '''
         Place item at index by inserting all axes into the grid
         at list-order appropriate position.
@@ -175,11 +175,14 @@ class ComposedGridLayout:
                 '`.insert_plotitem()` only supports an index >= 0'
             )
 
+        inserted_axes: list[AxisItem] = []
+
         # add plot's axes in sequence to the embedded linear layouts
         # for each "side" thus avoiding graphics collisions.
         for name, axis_info in plotitem.axes.copy().items():
             linlayout, axes = self.sides[name]
             axis = axis_info['item']
+            inserted_axes.append(axis)
 
             if axis in axes:
                 # TODO: re-order using ``.pop()`` ?
@@ -192,19 +195,20 @@ class ComposedGridLayout:
             if (
                 not axis.isVisible()
 
-                # XXX: we never skip moving the axes for the *first*
+                # XXX: we never skip moving the axes for the *root*
                 # plotitem inserted (even if not shown) since we need to
                 # move all the hidden axes into linear sub-layouts for
                 # that "central" plot in the overlay. Also if we don't
                 # do it there's weird geomoetry calc offsets that make
                 # view coords slightly off somehow .. smh
-                and not len(self.items) == 0
+                and not len(self.pitems) == 0
             ):
                 continue
 
-            # XXX: Remove old axis? No, turns out we don't need this?
-            # DON'T unlink it since we the original ``ViewBox``
-            # to still drive it B)
+            # XXX: Remove old axis?
+            # No, turns out we don't need this?
+            # DON'T UNLINK IT since we need the original ``ViewBox`` to
+            # still drive it with events/handlers B)
             # popped = plotitem.removeAxis(name, unlink=False)
             # assert axis is popped
 
@@ -220,7 +224,7 @@ class ComposedGridLayout:
 
         self._register_item(index, plotitem)
 
-        return index
+        return (index, inserted_axes)
 
     def append_plotitem(
         self,
@@ -234,20 +238,20 @@ class ComposedGridLayout:
         '''
         # for left and bottom axes we have to first remove
         # items and re-insert to maintain a list-order.
-        return self.insert_plotitem(len(self.items), item)
+        return self.insert_plotitem(len(self.pitems), item)
 
     def get_axis(
         self,
         plot: PlotItem,
         name: str,
 
-    ) -> Optional[AxisItem]:
+    ) -> AxisItem | None:
         '''
         Retrieve the named axis for overlayed ``plot`` or ``None``
         if axis for that name is not shown.
 
         '''
-        index = self.items.index(plot)
+        index = self.pitems.index(plot)
         named = self._pi2axes[name]
         return named.get(index)
 
@@ -306,14 +310,17 @@ class PlotItemOverlay:
         # events/signals.
         root_plotitem.vb.setZValue(10)
 
-        self.overlays: list[PlotItem] = []
         self.layout = ComposedGridLayout(root_plotitem)
         self._relays: dict[str, Signal] = {}
 
+    @property
+    def overlays(self) -> list[PlotItem]:
+        return self.layout.pitems
+
     def add_plotitem(
         self,
         plotitem: PlotItem,
-        index: Optional[int] = None,
+        index: int | None = None,
 
         # event/signal names which will be broadcasted to all added
         # (relayee) ``PlotItem``s (eg. ``ViewBox.mouseDragEvent``).
@@ -324,11 +331,9 @@ class PlotItemOverlay:
         # (0, 1),  # link both
         link_axes: tuple[int] = (),
 
-    ) -> None:
+    ) -> tuple[int, list[AxisItem]]:
 
-        index = index or len(self.overlays)
         root = self.root_plotitem
-        self.overlays.insert(index, plotitem)
         vb: ViewBox = plotitem.vb
 
         # TODO: some sane way to allow menu event broadcast XD
@@ -370,7 +375,7 @@ class PlotItemOverlay:
 
                         # TODO: drop this viewbox specific input and
                         # allow a predicate to be passed in by user.
-                        axis: 'Optional[int]' = None,
+                        axis: int | None = None,
 
                         *,
 
@@ -476,7 +481,10 @@ class PlotItemOverlay:
         # ``PlotItem`` dynamically.
 
         # append-compose into the layout all axes from this plot
-        self.layout.insert_plotitem(index, plotitem)
+        if index is None:
+            insert_index, axes = self.layout.append_plotitem(plotitem)
+        else:
+            insert_index, axes = self.layout.insert_plotitem(index, plotitem)
 
         plotitem.setGeometry(root.vb.sceneBoundingRect())
 
@@ -496,6 +504,11 @@ class PlotItemOverlay:
 
         vb.setZValue(100)
 
+        return (
+            index,
+            axes,
+        )
+
     def get_axis(
         self,
         plot: PlotItem,
@@ -564,3 +577,93 @@ class PlotItemOverlay:
 
     #     '''
     #     ...
+
+    def group_maxmin(
+        self,
+        focus_around: str | None = None,
+        force_min: float | None = None,
+
+    ) -> tuple[
+        float,  # mn
+        float,  # mx
+        float,  # max range in % terms of highest sigma plot's y-range
+        PlotItem,  # front/selected plot
+    ]:
+        '''
+        Overlay "group" maxmin sorting.
+
+        Assumes all named flows are in the same co-domain and thus can
+        be sorted as one set.
+
+        Iterates all the named flows and calls the chart api to find
+        their range values and return.
+
+        TODO: really we should probably have a more built-in API for
+        this?
+
+        '''
+        # TODO:
+        # - use this in the ``.ui._fsp`` mutli-maxmin stuff
+        # -
+
+        # force 0 to always be in view
+        group_mx: float = 0
+        group_mn: float = 0
+        mx_up_rng: float = 0
+        mn_down_rng: float = 0
+        pis2ranges: dict[
+            PlotItem,
+            tuple[float, float],
+        ] = {}
+
+        for pi in self.overlays:
+
+            # TODO: can we remove this from the widget
+            # and place somewhere more related to UX/Viz?
+            # name = pi.name
+            # chartw = pi.chart_widget
+            viz = pi.viz
+            # viz = chartw._vizs[name]
+
+            out = viz.maxmin()
+            if out is None:
+                return None
+
+            (
+                (x_start, x_stop),
+                read_slc,
+                (ymn, ymx),
+            ) = out
+
+            arr = viz.shm.array
+
+            y_start = arr[read_slc.start - 1]
+            y_stop = arr[read_slc.stop - 1]
+            if viz.is_ohlc:
+                y_start = y_start['open']
+                y_stop = y_stop['close']
+            else:
+                y_start = y_start[viz.name]
+                y_stop = y_stop[viz.name]
+
+            # update max for group
+            up_rng = (ymx - y_start) / y_start
+            down_rng = (y_stop - ymn) / y_stop
+
+            # compute directional (up/down) y-range % swing/dispersion
+            mx_up_rng = max(mx_up_rng, up_rng)
+            mn_down_rng = min(mn_down_rng, down_rng)
+
+            pis2ranges[pi] = (ymn, ymx)
+
+            group_mx = max(group_mx, ymx)
+            if force_min is None:
+                group_mn = min(group_mn, ymn)
+
+        return (
+           group_mn if force_min is None else force_min,
+           group_mx,
+           mn_down_rng,
+           mx_up_rng,
+           pis2ranges,
+        )

piker/ui/_pathops.py

@@ -1,241 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-"""
-Super fast ``QPainterPath`` generation related operator routines.
-
-"""
-from __future__ import annotations
-from typing import (
-    # Optional,
-    TYPE_CHECKING,
-)
-
-import numpy as np
-from numpy.lib import recfunctions as rfn
-from numba import njit, float64, int64  # , optional
-# import pyqtgraph as pg
-from PyQt5 import QtGui
-# from PyQt5.QtCore import QLineF, QPointF
-
-from ..data._sharedmem import (
-    ShmArray,
-)
-# from .._profile import pg_profile_enabled, ms_slower_then
-from ._compression import (
-    ds_m4,
-)
-
-if TYPE_CHECKING:
-    from ._flows import Renderer
-
-
-def xy_downsample(
-    x,
-    y,
-    uppx,
-
-    x_spacer: float = 0.5,
-
-) -> tuple[
-    np.ndarray,
-    np.ndarray,
-    float,
-    float,
-]:
-
-    # downsample whenever more then 1 pixels per datum can be shown.
-    # always refresh data bounds until we get diffing
-    # working properly, see above..
-    bins, x, y, ymn, ymx = ds_m4(
-        x,
-        y,
-        uppx,
-    )
-
-    # flatten output to 1d arrays suitable for path-graphics generation.
-    x = np.broadcast_to(x[:, None], y.shape)
-    x = (x + np.array(
-        [-x_spacer, 0, 0, x_spacer]
-    )).flatten()
-    y = y.flatten()
-
-    return x, y, ymn, ymx
-
-
-@njit(
-    # TODO: for now need to construct this manually for readonly arrays, see
-    # https://github.com/numba/numba/issues/4511
-    # ntypes.tuple((float64[:], float64[:], float64[:]))(
-    #     numba_ohlc_dtype[::1],  # contiguous
-    #     int64,
-    #     optional(float64),
-    # ),
-    nogil=True
-)
-def path_arrays_from_ohlc(
-    data: np.ndarray,
-    start: int64,
-    bar_gap: float64 = 0.43,
-
-) -> np.ndarray:
-    '''
-    Generate an array of lines objects from input ohlc data.
-
-    '''
-    size = int(data.shape[0] * 6)
-
-    x = np.zeros(
-        # data,
-        shape=size,
-        dtype=float64,
-    )
-    y, c = x.copy(), x.copy()
-
-    # TODO: report bug for assert @
-    # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
-    for i, q in enumerate(data[start:], start):
-
-        # TODO: ask numba why this doesn't work..
-        # open, high, low, close, index = q[
-        #     ['open', 'high', 'low', 'close', 'index']]
-
-        open = q['open']
-        high = q['high']
-        low = q['low']
-        close = q['close']
-        index = float64(q['index'])
-
-        istart = i * 6
-        istop = istart + 6
-
-        # x,y detail the 6 points which connect all vertexes of a ohlc bar
-        x[istart:istop] = (
-            index - bar_gap,
-            index,
-            index,
-            index,
-            index,
-            index + bar_gap,
-        )
-        y[istart:istop] = (
-            open,
-            open,
-            low,
-            high,
-            close,
-            close,
-        )
-
-        # specifies that the first edge is never connected to the
-        # prior bars last edge thus providing a small "gap"/"space"
-        # between bars determined by ``bar_gap``.
-        c[istart:istop] = (1, 1, 1, 1, 1, 0)
-
-    return x, y, c
-
-
-def gen_ohlc_qpath(
-    r: Renderer,
-    data: np.ndarray,
-    array_key: str,  # we ignore this
-    vr: tuple[int, int],
-
-    start: int = 0,  # XXX: do we need this?
-    # 0.5 is no overlap between arms, 1.0 is full overlap
-    w: float = 0.43,
-
-) -> QtGui.QPainterPath:
-    '''
-    More or less direct proxy to ``path_arrays_from_ohlc()``
-    but with closed in kwargs for line spacing.
-
-    '''
-    x, y, c = path_arrays_from_ohlc(
-        data,
-        start,
-        bar_gap=w,
-    )
-    return x, y, c
-
-
-def ohlc_to_line(
-    ohlc_shm: ShmArray,
-    data_field: str,
-    fields: list[str] = ['open', 'high', 'low', 'close']
-
-) -> tuple[
-    np.ndarray,
-    np.ndarray,
-]:
-    '''
-    Convert an input struct-array holding OHLC samples into a pair of
-    flattened x, y arrays with the same size (datums wise) as the source
-    data.
-
-    '''
-    y_out = ohlc_shm.ustruct(fields)
-    first = ohlc_shm._first.value
-    last = ohlc_shm._last.value
-
-    # write pushed data to flattened copy
-    y_out[first:last] = rfn.structured_to_unstructured(
-        ohlc_shm.array[fields]
-    )
-
-    # generate an flat-interpolated x-domain
-    x_out = (
-        np.broadcast_to(
-            ohlc_shm._array['index'][:, None],
-            (
-                ohlc_shm._array.size,
-                # 4,  # only ohlc
-                y_out.shape[1],
-            ),
-        ) + np.array([-0.5, 0, 0, 0.5])
-    )
-    assert y_out.any()
-
-    return (
-        x_out,
-        y_out,
-    )
-
-
-def to_step_format(
-    shm: ShmArray,
-    data_field: str,
-    index_field: str = 'index',
-
-) -> tuple[int, np.ndarray, np.ndarray]:
-    '''
-    Convert an input 1d shm array to a "step array" format
-    for use by path graphics generation.
-
-    '''
-    i = shm._array['index'].copy()
-    out = shm._array[data_field].copy()
-
-    x_out = np.broadcast_to(
-        i[:, None],
-        (i.size, 2),
-    ) + np.array([-0.5, 0.5])
-
-    y_out = np.empty((len(out), 2), dtype=out.dtype)
-    y_out[:] = out[:, np.newaxis]
-
-    # start y at origin level
-    y_out[0, 0] = 0
-    return x_out, y_out

piker/ui/_pg_overrides.py

@@ -26,6 +26,8 @@ from typing import Optional
 
 import pyqtgraph as pg
 
+from ._axes import Axis
+
 
 def invertQTransform(tr):
     """Return a QTransform that is the inverse of *tr*.
@@ -52,6 +54,10 @@ def _do_overrides() -> None:
     pg.functions.invertQTransform = invertQTransform
     pg.PlotItem = PlotItem
 
+    # enable "QPainterPathPrivate for faster arrayToQPath" from
+    # https://github.com/pyqtgraph/pyqtgraph/pull/2324
+    pg.setConfigOption('enableExperimental', True)
+
 
 # NOTE: the below customized type contains all our changes on a method
 # by method basis as per the diff:
@@ -62,6 +68,20 @@ class PlotItem(pg.PlotItem):
     Overrides for the core plot object mostly pertaining to overlayed
     multi-view management as it relates to multi-axis managment.
 
+    This object is the combination of a ``ViewBox`` and multiple
+    ``AxisItem``s and so far we've added additional functionality and
+    APIs for:
+    - removal of axes
+
+    ---
+
+    From ``pyqtgraph`` super type docs:
+    - Manage placement of ViewBox, AxisItems, and LabelItems
+    - Create and manage a list of PlotDataItems displayed inside the
+      ViewBox
+    - Implement a context menu with commonly used display and analysis
+      options
+
     '''
     def __init__(
         self,
@@ -86,6 +106,8 @@ class PlotItem(pg.PlotItem):
             enableMenu=enableMenu,
             kargs=kargs,
         )
+        self.name = name
+        self.chart_widget = None
         # self.setAxisItems(
         #     axisItems,
         #     default_axes=default_axes,
@@ -209,7 +231,12 @@ class PlotItem(pg.PlotItem):
                 # adding this is without it there's some weird
                 # ``ViewBox`` geometry bug.. where a gap for the
                 # 'bottom' axis is somehow left in?
-                axis = pg.AxisItem(orientation=name, parent=self)
+                # axis = pg.AxisItem(orientation=name, parent=self)
+                axis = Axis(
+                    self,
+                    orientation=name,
+                    parent=self,
+                )
 
             axis.linkToView(self.vb)
 

piker/ui/_position.py

@@ -41,7 +41,11 @@ from ._anchors import (
     pp_tight_and_right,  # wanna keep it straight in the long run
     gpath_pin,
 )
-from ..calc import humanize, pnl, puterize
+from ..calc import (
+    humanize,
+    pnl,
+    puterize,
+)
 from ..clearing._allocate import Allocator
 from ..pp import Position
 from ..data._normalize import iterticks
@@ -80,9 +84,9 @@ async def update_pnl_from_feed(
     '''
     global _pnl_tasks
 
-    pp = order_mode.current_pp
-    live = pp.live_pp
-    key = live.symbol.front_fqsn()
+    pp: PositionTracker = order_mode.current_pp
+    live: Position = pp.live_pp
+    key: str = live.symbol.front_fqsn()
 
     log.info(f'Starting pnl display for {pp.alloc.account}')
 
@@ -101,11 +105,22 @@ async def update_pnl_from_feed(
         async with flume.stream.subscribe() as bstream:
             # last_tick = time.time()
             async for quotes in bstream:
-
                 # now = time.time()
                 # period = now - last_tick
 
                 for sym, quote in quotes.items():
+                    # print(f'{key} PnL: sym:{sym}')
+
+                    # TODO: uggggh we probably want a better state
+                    # management then this sincce we want to enable
+                    # updating whatever the current symbol is in
+                    # real-time right?
+                    if sym != key:
+                        continue
+
+                    # watch out for wrong quote msg-data if you muck
+                    # with backend feed subs code..
+                    # assert sym == quote['fqsn']
 
                     for tick in iterticks(quote, types):
                         # print(f'{1/period} Hz')
@@ -119,13 +134,17 @@ async def update_pnl_from_feed(
 
                         else:
                             # compute and display pnl status
-                            order_mode.pane.pnl_label.format(
-                                pnl=copysign(1, size) * pnl(
+                            pnl_val = (
+                                 copysign(1, size)
+                                 *
+                                 pnl(
                                      # live.ppu,
                                      order_mode.current_pp.live_pp.ppu,
                                      tick['price'],
-                                ),
                                  )
+                             )
+                            # print(f'formatting PNL {sym} => {pnl_val}')
+                            order_mode.pane.pnl_label.format(pnl=pnl_val)
 
                         # last_tick = time.time()
     finally:

piker/ui/_render.py

@@ -0,0 +1,320 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+High level streaming graphics primitives.
+
+This is an intermediate layer which associates real-time low latency
+graphics primitives with underlying stream/flow related data structures
+for fast incremental update.
+
+'''
+from __future__ import annotations
+from typing import (
+    TYPE_CHECKING,
+)
+
+import msgspec
+import numpy as np
+import pyqtgraph as pg
+from PyQt5.QtGui import QPainterPath
+
+from ..data._formatters import (
+    IncrementalFormatter,
+)
+from ..data._pathops import (
+    xy_downsample,
+)
+from ..log import get_logger
+from .._profile import (
+    Profiler,
+)
+
+if TYPE_CHECKING:
+    from ._dataviz import Viz
+
+
+log = get_logger(__name__)
+
+
+class Renderer(msgspec.Struct):
+
+    viz: Viz
+    fmtr: IncrementalFormatter
+
+    # output graphics rendering, the main object
+    # processed in ``QGraphicsObject.paint()``
+    path: QPainterPath | None = None
+    fast_path: QPainterPath | None = None
+
+    # downsampling state
+    _last_uppx: float = 0
+    _in_ds: bool = False
+
+    def draw_path(
+        self,
+        x: np.ndarray,
+        y: np.ndarray,
+        connect: str | np.ndarray = 'all',
+        path: QPainterPath | None = None,
+        redraw: bool = False,
+
+    ) -> QPainterPath:
+
+        path_was_none = path is None
+
+        if redraw and path:
+            path.clear()
+
+            # TODO: avoid this?
+            if self.fast_path:
+                self.fast_path.clear()
+
+        path = pg.functions.arrayToQPath(
+            x,
+            y,
+            connect=connect,
+            finiteCheck=False,
+
+            # reserve mem allocs see:
+            # - https://doc.qt.io/qt-5/qpainterpath.html#reserve
+            # - https://doc.qt.io/qt-5/qpainterpath.html#capacity
+            # - https://doc.qt.io/qt-5/qpainterpath.html#clear
+            # XXX: right now this is based on ad-hoc checks on a
+            # hidpi 3840x2160 4k monitor but we should optimize for
+            # the target display(s) on the sys.
+            # if no_path_yet:
+            #     graphics.path.reserve(int(500e3))
+            # path=path,  # path re-use / reserving
+        )
+
+        # avoid mem allocs if possible
+        if path_was_none:
+            path.reserve(path.capacity())
+
+        return path
+
+    def render(
+        self,
+
+        new_read,
+        array_key: str,
+        profiler: Profiler,
+        uppx: float = 1,
+
+        # redraw and ds flags
+        should_redraw: bool = False,
+        new_sample_rate: bool = False,
+        should_ds: bool = False,
+        showing_src_data: bool = True,
+
+        do_append: bool = True,
+        use_fpath: bool = True,
+
+        # only render datums "in view" of the ``ChartView``
+        use_vr: bool = True,
+
+    ) -> tuple[QPainterPath, bool]:
+        '''
+        Render the current graphics path(s)
+
+        There are (at least) 3 stages from source data to graphics data:
+        - a data transform (which can be stored in additional shm)
+        - a graphics transform which converts discrete basis data to
+          a `float`-basis view-coords graphics basis. (eg. ``ohlc_flatten()``,
+          ``step_path_arrays_from_1d()``, etc.)
+
+        - blah blah blah (from notes)
+
+        '''
+        # TODO: can the renderer just call ``Viz.read()`` directly?
+        # unpack latest source data read
+        fmtr = self.fmtr
+
+        (
+            _,
+            _,
+            array,
+            ivl,
+            ivr,
+            in_view,
+        ) = new_read
+
+        # xy-path data transform: convert source data to a format
+        # able to be passed to a `QPainterPath` rendering routine.
+        fmt_out = fmtr.format_to_1d(
+            new_read,
+            array_key,
+            profiler,
+
+            slice_to_inview=use_vr,
+        )
+
+        # no history in view case
+        if not fmt_out:
+            # XXX: this might be why the profiler only has exits?
+            return
+
+        (
+            x_1d,
+            y_1d,
+            connect,
+            prepend_length,
+            append_length,
+            view_changed,
+            # append_tres,
+
+        ) = fmt_out
+
+        # redraw conditions
+        if (
+            prepend_length > 0
+            or new_sample_rate
+            or view_changed
+
+            # NOTE: comment this to try and make "append paths"
+            # work below..
+            or append_length > 0
+        ):
+            should_redraw = True
+
+        path: QPainterPath = self.path
+        fast_path: QPainterPath = self.fast_path
+        reset: bool = False
+
+        self.viz.yrange = None
+
+        # redraw the entire source data if we have either of:
+        # - no prior path graphic rendered or,
+        # - we always intend to re-render the data only in view
+        if (
+            path is None
+            or should_redraw
+        ):
+            # print(f"{self.viz.name} -> REDRAWING BRUH")
+            if new_sample_rate and showing_src_data:
+                log.info(f'DE-downsampling -> {array_key}')
+                self._in_ds = False
+
+            elif should_ds and uppx > 1:
+
+                ds_out = xy_downsample(
+                    x_1d,
+                    y_1d,
+                    uppx,
+                )
+                if ds_out is not None:
+                    x_1d, y_1d, ymn, ymx = ds_out
+                    self.viz.yrange = ymn, ymx
+                    # print(f'{self.viz.name} post ds: ymn, ymx: {ymn},{ymx}')
+
+                    reset = True
+                    profiler(f'FULL PATH downsample redraw={should_ds}')
+                    self._in_ds = True
+
+            path = self.draw_path(
+                x=x_1d,
+                y=y_1d,
+                connect=connect,
+                path=path,
+                redraw=True,
+            )
+
+            profiler(
+                'generated fresh path. '
+                f'(should_redraw: {should_redraw} '
+                f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})'
+            )
+
+        # TODO: get this piecewise prepend working - right now it's
+        # giving heck on vwap...
+        # elif prepend_length:
+
+        #     prepend_path = pg.functions.arrayToQPath(
+        #         x[0:prepend_length],
+        #         y[0:prepend_length],
+        #         connect='all'
+        #     )
+
+        #     # swap prepend path in "front"
+        #     old_path = graphics.path
+        #     graphics.path = prepend_path
+        #     # graphics.path.moveTo(new_x[0], new_y[0])
+        #     graphics.path.connectPath(old_path)
+
+        elif (
+            append_length > 0
+            and do_append
+        ):
+            profiler(f'sliced append path {append_length}')
+            # (
+            #     x_1d,
+            #     y_1d,
+            #     connect,
+            # ) = append_tres
+
+            profiler(
+                f'diffed array input, append_length={append_length}'
+            )
+
+            # if should_ds and uppx > 1:
+            #     new_x, new_y = xy_downsample(
+            #         new_x,
+            #         new_y,
+            #         uppx,
+            #     )
+            #     profiler(f'fast path downsample redraw={should_ds}')
+
+            append_path = self.draw_path(
+                x=x_1d,
+                y=y_1d,
+                connect=connect,
+                path=fast_path,
+            )
+            profiler('generated append qpath')
+
+            if use_fpath:
+                # an attempt at trying to make append-updates faster..
+                if fast_path is None:
+                    fast_path = append_path
+                    # fast_path.reserve(int(6e3))
+                else:
+                    # print(
+                    #     f'{self.viz.name}: FAST PATH\n'
+                    #     f"append_path br: {append_path.boundingRect()}\n"
+                    #     f"path size: {size}\n"
+                    #     f"append_path len: {append_path.length()}\n"
+                    #     f"fast_path len: {fast_path.length()}\n"
+                    # )
+
+                    fast_path.connectPath(append_path)
+                    size = fast_path.capacity()
+                    profiler(f'connected fast path w size: {size}')
+
+                    # graphics.path.moveTo(new_x[0], new_y[0])
+                    # path.connectPath(append_path)
+
+                    # XXX: lol this causes a hang..
+                    # graphics.path = graphics.path.simplified()
+            else:
+                size = path.capacity()
+                profiler(f'connected history path w size: {size}')
+                path.connectPath(append_path)
+
+        self.path = path
+        self.fast_path = fast_path
+
+        return self.path, reset

piker/ui/_search.py

@@ -144,15 +144,29 @@ class CompleterView(QTreeView):
         self._font_size: int = 0  # pixels
         self._init: bool = False
 
-    async def on_pressed(self, idx: QModelIndex) -> None:
+    async def on_pressed(
+        self,
+        idx: QModelIndex,
+    ) -> None:
         '''
         Mouse pressed on view handler.
 
         '''
         search = self.parent()
-        await search.chart_current_item()
+
+        await search.chart_current_item(
+            clear_to_cache=True,
+        )
+
+        # XXX: this causes Qt to hang and segfault..lovely
+        # self.show_cache_entries(
+        #     only=True,
+        #     keep_current_item_selected=True,
+        # )
+
         search.focus()
 
+
     def set_font_size(self, size: int = 18):
         # print(size)
         if size < 0:
@@ -288,7 +302,7 @@ class CompleterView(QTreeView):
     def select_first(self) -> QStandardItem:
         '''
         Select the first depth >= 2 entry from the completer tree and
-        return it's item.
+        return its item.
 
         '''
         # ensure we're **not** selecting the first level parent node and
@@ -416,12 +430,26 @@ class CompleterView(QTreeView):
         section: str,
         values: Sequence[str],
         clear_all: bool = False,
+        reverse: bool = False,
 
     ) -> None:
         '''
         Set result-rows for depth = 1 tree section ``section``.
 
         '''
+        if (
+            values
+            and not isinstance(values[0], str)
+        ):
+            flattened: list[str] = []
+            for val in values:
+                flattened.extend(val)
+
+            values = flattened
+
+        if reverse:
+            values = reversed(values)
+
         model = self.model()
         if clear_all:
             # XXX: rewrite the model from scratch if caller requests it
@@ -598,22 +626,50 @@ class SearchWidget(QtWidgets.QWidget):
         self.show()
         self.bar.focus()
 
-    def show_only_cache_entries(self) -> None:
+    def show_cache_entries(
+        self,
+        only: bool = False,
+        keep_current_item_selected: bool = False,
+
+    ) -> None:
         '''
         Clear the search results view and show only cached (aka recently
         loaded with active data) feeds in the results section.
 
         '''
         godw = self.godwidget
+
+        # first entry in the cache is the current symbol(s)
+        fqsns = set()
+        for multi_fqsns in list(godw._chart_cache):
+            for fqsn in set(multi_fqsns):
+                fqsns.add(fqsn)
+
+        if keep_current_item_selected:
+            sel = self.view.selectionModel()
+            cidx = sel.currentIndex()
+
         self.view.set_section_entries(
             'cache',
-            list(reversed(godw._chart_cache)),
+            list(fqsns),
             # remove all other completion results except for cache
-            clear_all=True,
+            clear_all=only,
+            reverse=True,
         )
 
-    def get_current_item(self) -> Optional[tuple[str, str]]:
-        '''Return the current completer tree selection as
+        if (
+            keep_current_item_selected
+            and cidx.isValid()
+        ):
+            # set current selection back to what it was before filling out
+            # the view results.
+            self.view.select_from_idx(cidx)
+        else:
+            self.view.select_first()
+
+    def get_current_item(self) -> tuple[QModelIndex, str, str] | None:
+        '''
+        Return the current completer tree selection as
         a tuple ``(parent: str, child: str)`` if valid, else ``None``.
 
         '''
@@ -639,7 +695,11 @@ class SearchWidget(QtWidgets.QWidget):
             if provider == 'cache':
                 symbol, _, provider = symbol.rpartition('.')
 
-            return provider, symbol
+            return (
+                cidx,
+                provider,
+                symbol,
+            )
 
         else:
             return None
@@ -660,15 +720,16 @@ class SearchWidget(QtWidgets.QWidget):
         if value is None:
             return None
 
-        provider, symbol = value
+        cidx, provider, symbol = value
         godw = self.godwidget
 
-        log.info(f'Requesting symbol: {symbol}.{provider}')
+        fqsn = f'{symbol}.{provider}'
+        log.info(f'Requesting symbol: {fqsn}')
 
+        # assert provider in symbol
         await godw.load_symbols(
-            provider,
-            [symbol],
-            'info',
+            fqsns=[fqsn],
+            loglevel='info',
         )
 
         # fully qualified symbol name (SNS i guess is what we're
@@ -682,13 +743,15 @@ class SearchWidget(QtWidgets.QWidget):
             # Re-order the symbol cache on the chart to display in
             # LIFO order. this is normally only done internally by
             # the chart on new symbols being loaded into memory
-            godw.set_chart_symbol(
-                fqsn, (
+            godw.set_chart_symbols(
+                (fqsn,), (
                     godw.hist_linked,
                     godw.rt_linked,
                 )
             )
-            self.show_only_cache_entries()
+            self.show_cache_entries(
+                only=True,
+            )
 
         self.bar.focus()
         return fqsn
@@ -757,9 +820,10 @@ async def pack_matches(
     with trio.CancelScope() as cs:
         task_status.started(cs)
         # ensure ^ status is updated
-        results = await search(pattern)
+        results = list(await search(pattern))
 
-    if provider != 'cache':  # XXX: don't cache the cache results xD
+    # XXX: don't cache the cache results xD
+    if provider != 'cache':
         matches[(provider, pattern)] = results
 
         # print(f'results from {provider}: {results}')
@@ -806,7 +870,7 @@ async def fill_results(
     has_results: defaultdict[str, set[str]] = defaultdict(set)
 
     # show cached feed list at startup
-    search.show_only_cache_entries()
+    search.show_cache_entries()
     search.on_resize()
 
     while True:
@@ -860,8 +924,9 @@ async def fill_results(
                     # it hasn't already been searched with the current
                     # input pattern (in which case just look up the old
                     # results).
-                    if (period >= pause) and (
-                        provider not in already_has_results
+                    if (
+                        period >= pause
+                        and provider not in already_has_results
                     ):
 
                         # TODO: it may make more sense TO NOT search the
@@ -869,7 +934,9 @@ async def fill_results(
                         # cpu-bound.
                         if provider != 'cache':
                             view.clear_section(
-                                provider, status_field='-> searchin..')
+                                provider,
+                                status_field='-> searchin..',
+                            )
 
                         await n.start(
                             pack_matches,
@@ -890,11 +957,20 @@ async def fill_results(
                         # re-searching it's ``dict`` since it's easier
                         # but it also causes it to be slower then cached
                         # results from other providers on occasion.
-                        if results and provider != 'cache':
+                        if (
+                            results
+                        ):
+                            if provider != 'cache':
                                 view.set_section_entries(
                                     section=provider,
                                     values=results,
                                 )
+                            else:
+                                # if provider == 'cache':
+                                # for the cache just show what we got
+                                # that matches
+                                search.show_cache_entries()
+
                         else:
                             view.clear_section(provider)
 
@@ -916,11 +992,10 @@ async def handle_keyboard_input(
     global _search_active, _search_enabled
 
     # startup
-    bar = searchbar
-    search = searchbar.parent()
-    godwidget = search.godwidget
-    view = bar.view
-    view.set_font_size(bar.dpi_font.px_size)
+    searchw = searchbar.parent()
+    godwidget = searchw.godwidget
+    view = searchbar.view
+    view.set_font_size(searchbar.dpi_font.px_size)
     send, recv = trio.open_memory_channel(616)
 
     async with trio.open_nursery() as n:
@@ -931,13 +1006,13 @@ async def handle_keyboard_input(
         n.start_soon(
             partial(
                 fill_results,
-                search,
+                searchw,
                 recv,
             )
         )
 
-        bar.focus()
-        search.show_only_cache_entries()
+        searchbar.focus()
+        searchw.show_cache_entries()
         await trio.sleep(0)
 
         async for kbmsg in recv_chan:
@@ -949,20 +1024,29 @@ async def handle_keyboard_input(
             if mods == Qt.ControlModifier:
                 ctl = True
 
-            if key in (Qt.Key_Enter, Qt.Key_Return):
+            if key in (
+                Qt.Key_Enter,
+                Qt.Key_Return
+            ):
                 _search_enabled = False
-                await search.chart_current_item(clear_to_cache=True)
-                search.show_only_cache_entries()
-                view.show_matches()
-                search.focus()
+                await searchw.chart_current_item(clear_to_cache=True)
 
-            elif not ctl and not bar.text():
-                # if nothing in search text show the cache
-                view.set_section_entries(
-                    'cache',
-                    list(reversed(godwidget._chart_cache)),
-                    clear_all=True,
-                )
+                # XXX: causes hang and segfault..
+                # searchw.show_cache_entries(
+                #     only=True,
+                #     keep_current_item_selected=True,
+                # )
+
+                view.show_matches()
+                searchw.focus()
+
+            elif (
+                not ctl
+                and not searchbar.text()
+            ):
+                # TODO: really should factor this somewhere..bc
+                # we're doin it in another spot as well..
+                searchw.show_cache_entries(only=True)
                 continue
 
             # cancel and close
@@ -971,7 +1055,7 @@ async def handle_keyboard_input(
                 Qt.Key_Space,   # i feel like this is the "native" one
                 Qt.Key_Alt,
             }:
-                bar.unfocus()
+                searchbar.unfocus()
 
                 # kill the search and focus back on main chart
                 if godwidget:
@@ -979,41 +1063,54 @@ async def handle_keyboard_input(
 
                 continue
 
-            if ctl and key in {
-                Qt.Key_L,
-            }:
+            if (
+                ctl
+                and key in {Qt.Key_L}
+            ):
                 # like url (link) highlight in a web browser
-                bar.focus()
+                searchbar.focus()
 
             # selection navigation controls
-            elif ctl and key in {
-                Qt.Key_D,
-            }:
+            elif (
+                ctl
+                and key in {Qt.Key_D}
+            ):
                 view.next_section(direction='down')
                 _search_enabled = False
 
-            elif ctl and key in {
-                Qt.Key_U,
-            }:
+            elif (
+                ctl
+                and key in {Qt.Key_U}
+            ):
                 view.next_section(direction='up')
                 _search_enabled = False
 
             # selection navigation controls
-            elif (ctl and key in {
-
+            elif (
+                ctl and (
+                    key in {
                         Qt.Key_K,
                         Qt.Key_J,
+                    }
 
-            }) or key in {
-
+                    or key in {
                         Qt.Key_Up,
                         Qt.Key_Down,
-            }:
+                    }
+                )
+            ):
                 _search_enabled = False
-                if key in {Qt.Key_K, Qt.Key_Up}:
+
+                if key in {
+                    Qt.Key_K,
+                    Qt.Key_Up
+                }:
                     item = view.select_previous()
 
-                elif key in {Qt.Key_J, Qt.Key_Down}:
+                elif key in {
+                    Qt.Key_J,
+                    Qt.Key_Down,
+                }:
                     item = view.select_next()
 
                 if item:
@@ -1022,26 +1119,39 @@ async def handle_keyboard_input(
                     # if we're in the cache section and thus the next
                     # selection is a cache item, switch and show it
                     # immediately since it should be very fast.
-                    if parent_item and parent_item.text() == 'cache':
-                        await search.chart_current_item(clear_to_cache=False)
+                    if (
+                        parent_item
+                        and parent_item.text() == 'cache'
+                    ):
+                        await searchw.chart_current_item(clear_to_cache=False)
 
+            # ACTUAL SEARCH BLOCK #
+            # where we fuzzy complete and fill out sections.
             elif not ctl:
                 # relay to completer task
                 _search_enabled = True
-                send.send_nowait(search.bar.text())
+                send.send_nowait(searchw.bar.text())
                 _search_active.set()
 
 
 async def search_simple_dict(
     text: str,
     source: dict,
+
 ) -> dict[str, Any]:
 
+    tokens = []
+    for key in source:
+        if not isinstance(key, str):
+            tokens.extend(key)
+        else:
+            tokens.append(key)
+
     # search routine can be specified as a function such
     # as in the case of the current app's local symbol cache
     matches = fuzzy.extractBests(
         text,
-        source.keys(),
+        tokens,
         score_cutoff=90,
     )
 

piker/ui/_style.py

@@ -240,12 +240,12 @@ def hcolor(name: str) -> str:
         'gunmetal': '#91A3B0',
         'battleship': '#848482',
 
+        # default ohlc-bars/curve gray
+        'bracket': '#666666',  # like the logo
+
         # bluish
         'charcoal': '#36454F',
 
-        # default bars
-        'bracket': '#666666',  # like the logo
-
         # work well for filled polygons which want a 'bracket' feel
         # going light to dark
         'davies': '#555555',

piker/ui/order_mode.py

@@ -88,7 +88,7 @@ class Dialog(Struct):
     # TODO: use ``pydantic.UUID4`` field
     uuid: str
     order: Order
-    symbol: Symbol
+    symbol: str
     lines: list[LevelLine]
     last_status_close: Callable = lambda: None
     msgs: dict[str, dict] = {}
@@ -349,7 +349,7 @@ class OrderMode:
 
         '''
         if not order:
-            staged = self._staged_order
+            staged: Order = self._staged_order
             # apply order fields for ems
             oid = str(uuid.uuid4())
             order = staged.copy()
@@ -379,7 +379,7 @@ class OrderMode:
         dialog = Dialog(
             uuid=order.oid,
             order=order,
-            symbol=order.symbol,
+            symbol=order.symbol,  # XXX: always a str?
             lines=lines,
             last_status_close=self.multistatus.open_status(
                 f'submitting {order.exec_mode}-{order.action}',
@@ -494,7 +494,7 @@ class OrderMode:
 
         uuid: str,
         price: float,
-        arrow_index: float,
+        time_s: float,
 
         pointing: Optional[str] = None,
 
@@ -513,22 +513,32 @@ class OrderMode:
         '''
         dialog = self.dialogs[uuid]
         lines = dialog.lines
+        chart = self.chart
+
         # XXX: seems to fail on certain types of races?
         # assert len(lines) == 2
         if lines:
-            flume: Flume = self.feed.flumes[self.chart.linked.symbol.fqsn]
+            flume: Flume = self.feed.flumes[chart.linked.symbol.fqsn]
             _, _, ratio = flume.get_ds_info()
-            for i, chart in [
-                (arrow_index, self.chart),
-                (flume.izero_hist
-                 +
-                 round((arrow_index - flume.izero_rt)/ratio),
-                 self.hist_chart)
+
+            for chart, shm in [
+                (self.chart, flume.rt_shm),
+                (self.hist_chart, flume.hist_shm),
             ]:
+                viz = chart.get_viz(chart.name)
+                index_field = viz.index_field
+                arr = shm.array
+
+                # TODO: borked for int index based..
+                index = flume.get_index(time_s, arr)
+
+                # get absolute index for arrow placement
+                arrow_index = arr[index_field][index]
+
                 self.arrows.add(
                     chart.plotItem,
                     uuid,
-                    i,
+                    arrow_index,
                     price,
                     pointing=pointing,
                     color=lines[0].color
@@ -693,7 +703,6 @@ async def open_order_mode(
 
         # symbol id
         symbol = chart.linked.symbol
-        symkey = symbol.front_fqsn()
 
         # map of per-provider account keys to position tracker instances
         trackers: dict[str, PositionTracker] = {}
@@ -854,7 +863,7 @@ async def open_order_mode(
         # the expected symbol key in its positions msg.
         for (broker, acctid), msgs in position_msgs.items():
             for msg in msgs:
-                log.info(f'Loading pp for {symkey}:\n{pformat(msg)}')
+                log.info(f'Loading pp for {acctid}@{broker}:\n{pformat(msg)}')
                 await process_trade_msg(
                     mode,
                     book,
@@ -930,7 +939,6 @@ async def process_trade_msg(
 
 ) -> tuple[Dialog, Status]:
 
-    get_index = mode.chart.get_index
     fmsg = pformat(msg)
     log.debug(f'Received order msg:\n{fmsg}')
     name = msg['name']
@@ -965,6 +973,9 @@ async def process_trade_msg(
     oid = msg.oid
     dialog: Dialog = mode.dialogs.get(oid)
 
+    if dialog:
+        fqsn = dialog.symbol
+
     match msg:
         case Status(
             resp='dark_open' | 'open',
@@ -1034,10 +1045,11 @@ async def process_trade_msg(
             # should only be one "fill" for an alert
             # add a triangle and remove the level line
             req = Order(**req)
+            tm = time.time()
             mode.on_fill(
                 oid,
                 price=req.price,
-                arrow_index=get_index(time.time()),
+                time_s=tm,
             )
             mode.lines.remove_line(uuid=oid)
             msg.req = req
@@ -1065,15 +1077,10 @@ async def process_trade_msg(
             action = order.action
             details = msg.brokerd_msg
 
-            # TODO: some kinda progress system
-            mode.on_fill(
-                oid,
-                price=details['price'],
-                pointing='up' if action == 'buy' else 'down',
+            # TODO: put the actual exchange timestamp?
+            # TODO: some kinda progress system?
 
-                # TODO: put the actual exchange timestamp
-                arrow_index=get_index(
-                    # TODO: note currently the ``kraken`` openOrders sub
+            # NOTE: currently the ``kraken`` openOrders sub
             # doesn't deliver their engine timestamp as part of
             # it's schema, so this value is **not** from them
             # (see our backend code). We should probably either
@@ -1083,8 +1090,12 @@ async def process_trade_msg(
             # a true backend one? This will require finagling
             # with how each backend tracks/summarizes time
             # stamps for the downstream API.
-                    details['broker_time']
-                ),
+            tm = details['broker_time']
+            mode.on_fill(
+                oid,
+                price=details['price'],
+                time_s=tm,
+                pointing='up' if action == 'buy' else 'down',
             )
 
             # TODO: append these fill events to the position's clear

piker/ui/qt/__init__.py

@@ -1,3 +0,0 @@
-"""
-Super hawt Qt UI components
-"""

piker/ui/qt/stackof_candle.py

@@ -1,67 +0,0 @@
-import sys
-
-from PySide2.QtCharts import QtCharts
-from PySide2.QtWidgets import QApplication, QMainWindow
-from PySide2.QtCore import Qt, QPointF
-from PySide2 import QtGui
-import qdarkstyle
-
-data = ((1, 7380, 7520, 7380, 7510, 7324),
-    (2, 7520, 7580, 7410, 7440, 7372),
-    (3, 7440, 7650, 7310, 7520, 7434),
-    (4, 7450, 7640, 7450, 7550, 7480),
-    (5, 7510, 7590, 7460, 7490, 7502),
-    (6, 7500, 7590, 7480, 7560, 7512),
-    (7, 7560, 7830, 7540, 7800, 7584))
-
-
-app = QApplication([])
-# set dark stylesheet
-# import pdb; pdb.set_trace()
-app.setStyleSheet(qdarkstyle.load_stylesheet_pyside())
-
-series = QtCharts.QCandlestickSeries()
-series.setDecreasingColor(Qt.darkRed)
-series.setIncreasingColor(Qt.darkGreen)
-
-ma5 = QtCharts.QLineSeries()  # 5-days average data line
-tm = []  # stores str type data
-
-# in a loop,  series and ma5 append corresponding data
-for num, o, h, l, c, m in data:
-    candle = QtCharts.QCandlestickSet(o, h, l, c)
-    series.append(candle)
-    ma5.append(QPointF(num, m))
-    tm.append(str(num))
-
-pen = candle.pen()
-# import pdb; pdb.set_trace()
-
-chart = QtCharts.QChart()
-
-# import pdb; pdb.set_trace()
-series.setBodyOutlineVisible(False)
-series.setCapsVisible(False)
-# brush = QtGui.QBrush()
-# brush.setColor(Qt.green)
-# series.setBrush(brush)
-chart.addSeries(series)  # candle
-chart.addSeries(ma5)  # ma5 line
-
-chart.setAnimationOptions(QtCharts.QChart.SeriesAnimations)
-chart.createDefaultAxes()
-chart.legend().hide()
-
-chart.axisX(series).setCategories(tm)
-chart.axisX(ma5).setVisible(False)
-
-view = QtCharts.QChartView(chart)
-view.chart().setTheme(QtCharts.QChart.ChartTheme.ChartThemeDark)
-view.setRubberBand(QtCharts.QChartView.HorizontalRubberBand)
-# chartview.chart().setTheme(QtCharts.QChart.ChartTheme.ChartThemeBlueCerulean)
-
-ui = QMainWindow()
-# ui.setGeometry(50, 50, 500, 300)
-ui.setCentralWidget(view)
-ui.show()
-sys.exit(app.exec_())

snippets/qt_screen_info.py

@@ -1,22 +1,26 @@
 """
-Resource list for mucking with DPIs on multiple screens:
-
-- https://stackoverflow.com/questions/42141354/convert-pixel-size-to-point-size-for-fonts-on-multiple-platforms
-- https://stackoverflow.com/questions/25761556/qt5-font-rendering-different-on-various-platforms/25929628#25929628
-- https://doc.qt.io/qt-5/highdpi.html
-- https://stackoverflow.com/questions/20464814/changing-dpi-scaling-size-of-display-make-qt-applications-font-size-get-rendere
-- https://stackoverflow.com/a/20465247
-- https://doc.qt.io/archives/qt-4.8/qfontmetrics.html#width
-- https://forum.qt.io/topic/54136/how-do-i-get-the-qscreen-my-widget-is-on-qapplication-desktop-screen-returns-a-qwidget-and-qobject_cast-qscreen-returns-null/3
-- https://forum.qt.io/topic/43625/point-sizes-are-they-reliable/4
-- https://stackoverflow.com/questions/16561879/what-is-the-difference-between-logicaldpix-and-physicaldpix-in-qt
-- https://doc.qt.io/qt-5/qguiapplication.html#screenAt
-
+DPI and info helper script for display metrics.
 """
 
-from pyqtgraph import QtGui
+# Resource list for mucking with DPIs on multiple screens:
+# https://stackoverflow.com/questions/42141354/convert-pixel-size-to-point-size-for-fonts-on-multiple-platforms
+# https://stackoverflow.com/questions/25761556/qt5-font-rendering-different-on-various-platforms/25929628#25929628
+# https://doc.qt.io/qt-5/highdpi.html
+# https://stackoverflow.com/questions/20464814/changing-dpi-scaling-size-of-display-make-qt-applications-font-size-get-rendere
+# https://stackoverflow.com/a/20465247
+# https://doc.qt.io/archives/qt-4.8/qfontmetrics.html#width
+# https://forum.qt.io/topic/54136/how-do-i-get-the-qscreen-my-widget-is-on-qapplication-desktop-screen-returns-a-qwidget-and-qobject_cast-qscreen-returns-null/3
+# https://forum.qt.io/topic/43625/point-sizes-are-they-reliable/4
+# https://stackoverflow.com/questions/16561879/what-is-the-difference-between-logicaldpix-and-physicaldpix-in-qt
+# https://doc.qt.io/qt-5/qguiapplication.html#screenAt
+
+from pyqtgraph import (
+    QtGui,
+    QtWidgets,
+)
 from PyQt5.QtCore import (
-     Qt, QCoreApplication
+     Qt,
+     QCoreApplication,
 )
 
 # Proper high DPI scaling is available in Qt >= 5.6.0. This attibute
@@ -28,55 +32,47 @@ if hasattr(Qt, 'AA_UseHighDpiPixmaps'):
     QCoreApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True)
 
 
-app = QtGui.QApplication([])
-window = QtGui.QMainWindow()
-main_widget = QtGui.QWidget()
+app = QtWidgets.QApplication([])
+window = QtWidgets.QMainWindow()
+main_widget = QtWidgets.QWidget()
 window.setCentralWidget(main_widget)
 window.show()
 
+# TODO: move widget through multiple displays and auto-detect the pixel
+# ratio? (probably is gonna require calls to i3ipc on linux)..
 pxr = main_widget.devicePixelRatioF()
 
-# screen_num = app.desktop().screenNumber()
+# TODO: how to detect list of displays from API?
 # screen = app.screens()[screen_num]
 
-screen = app.screenAt(main_widget.geometry().center())
 
+def ppscreeninfo(screen: 'QScreen') -> None:
+    # screen_num = app.desktop().screenNumber()
     name = screen.name()
     size = screen.size()
     geo = screen.availableGeometry()
     phydpi = screen.physicalDotsPerInch()
     logdpi = screen.logicalDotsPerInch()
+    rr = screen.refreshRate()
 
     print(
         # f'screen number: {screen_num}\n',
-    f'screen name: {name}\n'
-    f'screen size: {size}\n'
-    f'screen geometry: {geo}\n\n'
+        f'screen: {name}\n'
+        f'  size: {size}\n'
+        f'  geometry: {geo}\n'
+        f'  logical dpi: {logdpi}\n'
         f'  devicePixelRationF(): {pxr}\n'
         f'  physical dpi: {phydpi}\n'
-    f'logical dpi: {logdpi}\n'
+        f'  refresh rate: {rr}\n'
     )
 
-print('-'*50)
+    print('-'*50 + '\n')
+
+screen = app.screenAt(main_widget.geometry().center())
+ppscreeninfo(screen)
 
 screen = app.primaryScreen()
-
-name = screen.name()
-size = screen.size()
-geo = screen.availableGeometry()
-phydpi = screen.physicalDotsPerInch()
-logdpi = screen.logicalDotsPerInch()
-
-print(
-    # f'screen number: {screen_num}\n',
-    f'screen name: {name}\n'
-    f'screen size: {size}\n'
-    f'screen geometry: {geo}\n\n'
-    f'devicePixelRationF(): {pxr}\n'
-    f'physical dpi: {phydpi}\n'
-    f'logical dpi: {logdpi}\n'
-)
-
+ppscreeninfo(screen)
 
 # app-wide font
 font = QtGui.QFont("Hack")