Update profile msgs to new apis

Was broken since the `_adhoc_futes_set` rework a while back. Removes the
cmdty symbols from that set into a new one and fixes the contract
case block to catch `Contract(secType='CMDTY')` case. Also makes
`Client.search_symbols()` return details `dict`s so that `piker search`
will work again..

piker/_daemon.py

@@ -257,7 +257,7 @@ async def open_piker_runtime(
     # and spawn the service tree distributed per that.
     start_method: str = 'trio',
 
-    tractor_kwargs: dict = {},
+    **tractor_kwargs,
 
 ) -> tuple[
     tractor.Actor,

piker/_profile.py

@@ -152,9 +152,14 @@ class Profiler(object):
             # don't do anything
             return cls._disabledProfiler
 
-        # create an actual profiling object
         cls._depth += 1
         obj = super(Profiler, cls).__new__(cls)
+        obj._msgs = []
+
+        # create an actual profiling object
+        if cls._depth < 1:
+            cls._msgs = []
+
         obj._name = msg or func_qualname
         obj._delayed = delayed
         obj._markCount = 0
@@ -174,8 +179,12 @@ class Profiler(object):
 
         self._markCount += 1
         newTime = perf_counter()
+        tot_ms = (newTime - self._firstTime) * 1000
         ms = (newTime - self._lastTime) * 1000
-        self._newMsg("  %s: %0.4f ms", msg, ms)
+        self._newMsg(
+            f"  {msg}: {ms:0.4f}, tot:{tot_ms:0.4f}"
+        )
+
         self._lastTime = newTime
 
     def mark(self, msg=None):

piker/brokers/ib/api.py

@@ -161,10 +161,17 @@ _futes_venues = (
     'CME',
     'CMECRYPTO',
     'COMEX',
-    'CMDTY',  # special name case..
+    # 'CMDTY',  # special name case..
     'CBOT',  # (treasury) yield futures
 )
 
+_adhoc_cmdty_set = {
+    # metals
+    # https://misc.interactivebrokers.com/cstools/contract_info/v3.10/index.php?action=Conid%20Info&wlId=IB&conid=69067924
+    'xauusd.cmdty',  # london gold spot ^
+    'xagusd.cmdty',  # silver spot
+}
+
 _adhoc_futes_set = {
 
     # equities
@@ -186,16 +193,12 @@ _adhoc_futes_set = {
     # raw
     'lb.comex',  # random len lumber
 
-    # metals
-    # https://misc.interactivebrokers.com/cstools/contract_info/v3.10/index.php?action=Conid%20Info&wlId=IB&conid=69067924
-    'xauusd.cmdty',  # london gold spot ^
     'gc.comex',
     'mgc.comex',  # micro
 
     # oil & gas
     'cl.comex',
 
-    'xagusd.cmdty',  # silver spot
     'ni.comex',  # silver futes
     'qi.comex',  # mini-silver futes
 
@@ -259,6 +262,7 @@ _exch_skip_list = {
     'FUNDSERV',
     'SWB2',
     'PSE',
+    'PHLX',
 }
 
 _enters = 0
@@ -514,15 +518,18 @@ class Client:
         except ConnectionError:
             return {}
 
+        dict_results: dict[str, dict] = {}
         for key, deats in results.copy().items():
 
             tract = deats.contract
             sym = tract.symbol
             sectype = tract.secType
+            deats_dict = asdict(deats)
 
             if sectype == 'IND':
-                results[f'{sym}.IND'] = tract
                 results.pop(key)
+                key = f'{sym}.IND'
+                results[key] = tract
                 # exch = tract.exchange
 
                 # XXX: add back one of these to get the weird deadlock
@@ -559,20 +566,25 @@ class Client:
                     # if cons:
                     all_deats = await self.con_deats([con])
                     results |= all_deats
+                    for key in all_deats:
+                        dict_results[key] = asdict(all_deats[key])
 
             # forex pairs
             elif sectype == 'CASH':
+                results.pop(key)
                 dst, src = tract.localSymbol.split('.')
                 pair_key = "/".join([dst, src])
                 exch = tract.exchange.lower()
-                results[f'{pair_key}.{exch}'] = tract
-                results.pop(key)
+                key = f'{pair_key}.{exch}'
+                results[key] = tract
 
                 # XXX: again seems to trigger the weird tractor
                 # bug with the debugger..
                 # assert 0
 
-        return results
+            dict_results[key] = deats_dict
+
+        return dict_results
 
     async def get_fute(
         self,
@@ -1036,7 +1048,11 @@ def con2fqsn(
             # TODO: option symbol parsing and sane display:
             symbol = con.localSymbol.replace(' ', '')
 
-        case ibis.Commodity():
+        case (
+            ibis.Commodity()
+            # search API endpoint returns std con box..
+            | ibis.Contract(secType='CMDTY')
+        ):
             # commodities and forex don't have an exchange name and
             # no real volume so we have to calculate the price
             suffix = con.secType

piker/clearing/_ems.py

@@ -46,6 +46,7 @@ from ..data._normalize import iterticks
 from ..data._source import (
     unpack_fqsn,
     mk_fqsn,
+    float_digits,
 )
 from ..data.feed import (
     Feed,
@@ -1250,6 +1251,7 @@ async def process_client_order_cmds(
 
                 spread_slap: float = 5
                 min_tick = flume.symbol.tick_size
+                min_tick_digits = float_digits(min_tick)
 
                 if action == 'buy':
                     tickfilter = ('ask', 'last', 'trade')
@@ -1258,12 +1260,18 @@ async def process_client_order_cmds(
                     # TODO: we probably need to scale this based
                     # on some near term historical spread
                     # measure?
-                    abs_diff_away = spread_slap * min_tick
+                    abs_diff_away = round(
+                        spread_slap * min_tick,
+                        ndigits=min_tick_digits,
+                    )
 
                 elif action == 'sell':
                     tickfilter = ('bid', 'last', 'trade')
                     percent_away = -0.005
-                    abs_diff_away = -spread_slap * min_tick
+                    abs_diff_away = round(
+                        -spread_slap * min_tick,
+                        ndigits=min_tick_digits,
+                    )
 
                 else:  # alert
                     tickfilter = ('trade', 'utrade', 'last')

piker/data/_formatters.py

@@ -0,0 +1,824 @@
+# piker: trading gear for hackers
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Pre-(path)-graphics formatted x/y nd/1d rendering subsystem.
+
+"""
+from __future__ import annotations
+from typing import (
+    Optional,
+    TYPE_CHECKING,
+)
+
+import msgspec
+import numpy as np
+from numpy.lib import recfunctions as rfn
+
+from ._sharedmem import (
+    ShmArray,
+)
+from ._pathops import (
+    path_arrays_from_ohlc,
+)
+
+if TYPE_CHECKING:
+    from ._dataviz import (
+        Viz,
+    )
+    from .._profile import Profiler
+
+
+class IncrementalFormatter(msgspec.Struct):
+    '''
+    Incrementally updating, pre-path-graphics tracking, formatter.
+
+    Allows tracking source data state in an updateable pre-graphics
+    ``np.ndarray`` format (in local process memory) as well as
+    incrementally rendering from that format **to** 1d x/y for path
+    generation using ``pg.functions.arrayToQPath()``.
+
+    '''
+    shm: ShmArray
+    viz: Viz
+
+    # the value to be multiplied any any index into the x/y_1d arrays
+    # given the input index is based on the original source data array.
+    flat_index_ratio: float = 1
+
+    @property
+    def index_field(self) -> 'str':
+        '''
+        Value (``str``) used to look up the "index series" from the
+        underlying source ``numpy`` struct-array; delegate directly to
+        the managing ``Viz``.
+
+        '''
+        return self.viz.index_field
+
+    # Incrementally updated xy ndarray formatted data, a pre-1d
+    # format which is updated and cached independently of the final
+    # pre-graphics-path 1d format.
+    x_nd: Optional[np.ndarray] = None
+    y_nd: Optional[np.ndarray] = None
+
+    @property
+    def xy_nd(self) -> tuple[np.ndarray, np.ndarray]:
+        return (
+            self.x_nd[self.xy_slice],
+            self.y_nd[self.xy_slice],
+        )
+
+    @property
+    def xy_slice(self) -> slice:
+        return slice(
+            self.xy_nd_start,
+            self.xy_nd_stop,
+        )
+
+    # indexes which slice into the above arrays (which are allocated
+    # based on source data shm input size) and allow retrieving
+    # incrementally updated data.
+    xy_nd_start: int | None = None
+    xy_nd_stop: int | None = None
+
+    # TODO: eventually incrementally update 1d-pre-graphics path data?
+    x_1d: np.ndarray | None = None
+    y_1d: np.ndarray | None = None
+
+    # incremental view-change state(s) tracking
+    _last_vr: tuple[float, float] | None = None
+    _last_ivdr: tuple[float, float] | None = None
+
+    @property
+    def index_step_size(self) -> float:
+        '''
+        Readonly value computed on first ``.diff()`` call.
+
+        '''
+        return self.viz.index_step()
+
+    def diff(
+        self,
+        new_read: tuple[np.ndarray],
+
+    ) -> tuple[
+        np.ndarray,
+        np.ndarray,
+    ]:
+        # TODO:
+        # - can the renderer just call ``Viz.read()`` directly? unpack
+        #   latest source data read
+        # - eventually maybe we can implement some kind of
+        #   transform on the ``QPainterPath`` that will more or less
+        #   detect the diff in "elements" terms? update diff state since
+        #   we've now rendered paths.
+        (
+            xfirst,
+            xlast,
+            array,
+            ivl,
+            ivr,
+            in_view,
+        ) = new_read
+
+        index = array['index']
+
+        # if the first index in the read array is 0 then
+        # it means the source buffer has bee completely backfilled to
+        # available space.
+        src_start = index[0]
+        src_stop = index[-1] + 1
+
+        # these are the "formatted output data" indices
+        # for the pre-graphics arrays.
+        nd_start = self.xy_nd_start
+        nd_stop = self.xy_nd_stop
+
+        if (
+            nd_start is None
+        ):
+            assert nd_stop is None
+
+            # setup to do a prepend of all existing src history
+            nd_start = self.xy_nd_start = src_stop
+            # set us in a zero-to-append state
+            nd_stop = self.xy_nd_stop = src_stop
+
+        # compute the length diffs between the first/last index entry in
+        # the input data and the last indexes we have on record from the
+        # last time we updated the curve index.
+        prepend_length = int(nd_start - src_start)
+        append_length = int(src_stop - nd_stop)
+
+        # blah blah blah
+        # do diffing for prepend, append and last entry
+        return (
+            slice(src_start, nd_start),
+            prepend_length,
+            append_length,
+            slice(nd_stop, src_stop),
+        )
+
+    def _track_inview_range(
+        self,
+        view_range: tuple[int, int],
+
+    ) -> bool:
+        # if a view range is passed, plan to draw the
+        # source ouput that's "in view" of the chart.
+        vl, vr = view_range
+        zoom_or_append = False
+        last_vr = self._last_vr
+
+        # incremental in-view data update.
+        if last_vr:
+            lvl, lvr = last_vr  # relative slice indices
+
+            # TODO: detecting more specifically the interaction changes
+            # last_ivr = self._last_ivdr or (vl, vr)
+            # al, ar = last_ivr  # abs slice indices
+            # left_change = abs(x_iv[0] - al) >= 1
+            # right_change = abs(x_iv[-1] - ar) >= 1
+
+            # likely a zoom/pan view change or data append update
+            if (
+                (vr - lvr) > 2
+                or vl < lvl
+
+                # append / prepend update
+                # we had an append update where the view range
+                # didn't change but the data-viewed (shifted)
+                # underneath, so we need to redraw.
+                # or left_change and right_change and last_vr == view_range
+
+                # not (left_change and right_change) and ivr
+                # (
+                # or abs(x_iv[ivr] - livr) > 1
+            ):
+                zoom_or_append = True
+
+        self._last_vr = view_range
+
+        return zoom_or_append
+
+    def format_to_1d(
+        self,
+        new_read: tuple,
+        array_key: str,
+        profiler: Profiler,
+
+        slice_to_inview: bool = True,
+
+    ) -> tuple[
+        np.ndarray,
+        np.ndarray,
+    ]:
+        shm = self.shm
+
+        (
+            _,
+            _,
+            array,
+            ivl,
+            ivr,
+            in_view,
+
+        ) = new_read
+
+        (
+            pre_slice,
+            prepend_len,
+            append_len,
+            post_slice,
+        ) = self.diff(new_read)
+
+        # we first need to allocate xy data arrays
+        # from the source data.
+        if self.y_nd is None:
+            self.xy_nd_start = shm._first.value
+            self.xy_nd_stop = shm._last.value
+            self.x_nd, self.y_nd = self.allocate_xy_nd(
+                shm,
+                array_key,
+            )
+            profiler('allocated xy history')
+
+        # once allocated we do incremental pre/append
+        # updates from the diff with the source buffer.
+        else:
+            if prepend_len:
+
+                self.incr_update_xy_nd(
+                    shm,
+                    array_key,
+
+                    # this is the pre-sliced, "normally expected"
+                    # new data that an updater would normally be
+                    # expected to process, however in some cases (like
+                    # step curves) the updater routine may want to do
+                    # the source history-data reading itself, so we pass
+                    # both here.
+                    shm._array[pre_slice],
+                    pre_slice,
+                    prepend_len,
+
+                    self.xy_nd_start,
+                    self.xy_nd_stop,
+                    is_append=False,
+                )
+
+                self.xy_nd_start -= prepend_len
+                profiler('prepended xy history: {prepend_length}')
+
+            if append_len:
+                self.incr_update_xy_nd(
+                    shm,
+                    array_key,
+
+                    shm._array[post_slice],
+                    post_slice,
+                    append_len,
+
+                    self.xy_nd_start,
+                    self.xy_nd_stop,
+                    is_append=True,
+                )
+                self.xy_nd_stop += append_len
+                profiler('appened xy history: {append_length}')
+                # sanity
+                # slice_ln = post_slice.stop - post_slice.start
+                # assert append_len == slice_ln
+
+        view_changed: bool = False
+        view_range: tuple[int, int] = (ivl, ivr)
+        if slice_to_inview:
+            view_changed = self._track_inview_range(view_range)
+            array = in_view
+            profiler(f'{self.viz.name} view range slice {view_range}')
+
+        # TODO: we need to check if the last-datum-in-view is true and
+        # if so only slice to the 2nd last datumonly slice to the 2nd
+        # last datum.
+        # hist = array[:slice_to_head]
+
+        # XXX: WOA WTF TRACTOR DEBUGGING BUGGG
+        # assert 0
+
+        # xy-path data transform: convert source data to a format
+        # able to be passed to a `QPainterPath` rendering routine.
+        if not len(array):
+            # XXX: this might be why the profiler only has exits?
+            return
+
+        # TODO: hist here should be the pre-sliced
+        # x/y_data in the case where allocate_xy is
+        # defined?
+        x_1d, y_1d, connect = self.format_xy_nd_to_1d(
+            array,
+            array_key,
+            view_range,
+        )
+        # cache/save last 1d outputs for use by other
+        # readers (eg. `Viz.draw_last_datum()` in the
+        # only-draw-last-uppx case).
+        self.x_1d = x_1d
+        self.y_1d = y_1d
+
+        # app_tres = None
+        # if append_len:
+        #     appended = array[-append_len-1:slice_to_head]
+        #     app_tres = self.format_xy_nd_to_1d(
+        #         appended,
+        #         array_key,
+        #         (
+        #             view_range[1] - append_len + slice_to_head,
+        #             view_range[1]
+        #         ),
+        #     )
+        #     # assert (len(appended) - 1) == append_len
+        #     # assert len(appended) == append_len
+        #     print(
+        #         f'{self.viz.name} APPEND LEN: {append_len}\n'
+        #         f'{self.viz.name} APPENDED: {appended}\n'
+        #         f'{self.viz.name} app_tres: {app_tres}\n'
+        #     )
+
+        # update the last "in view data range"
+        if len(x_1d):
+            self._last_ivdr = x_1d[0], x_1d[-1]
+
+        profiler('.format_to_1d()')
+
+        return (
+            x_1d,
+            y_1d,
+            connect,
+            prepend_len,
+            append_len,
+            view_changed,
+            # app_tres,
+        )
+
+    ###############################
+    # Sub-type override interface #
+    ###############################
+
+    x_offset: np.ndarray = np.array([0])
+
+    # optional pre-graphics xy formatted data which
+    # is incrementally updated in sync with the source data.
+    # XXX: was ``.allocate_xy()``
+    def allocate_xy_nd(
+        self,
+        src_shm: ShmArray,
+        data_field: str,
+
+    ) -> tuple[
+        np.ndarray,  # x
+        np.nd.array  # y
+    ]:
+        '''
+        Convert the structured-array ``src_shm`` format to
+        a equivalently shaped (and field-less) ``np.ndarray``.
+
+        Eg. a 4 field x N struct-array => (N, 4)
+
+        '''
+        y_nd = src_shm._array[data_field].copy()
+        x_nd = (
+            src_shm._array[self.index_field].copy()
+            +
+            self.x_offset
+        )
+        return x_nd, y_nd
+
+    # XXX: was ``.update_xy()``
+    def incr_update_xy_nd(
+        self,
+
+        src_shm: ShmArray,
+        data_field: str,
+
+        new_from_src: np.ndarray,  # portion of source that was updated
+
+        read_slc: slice,
+        ln: int,  # len of updated
+
+        nd_start: int,
+        nd_stop: int,
+
+        is_append: bool,
+
+    ) -> None:
+        # write pushed data to flattened copy
+        y_nd_new = new_from_src[data_field]
+        self.y_nd[read_slc] = y_nd_new
+
+        x_nd_new = self.x_nd[read_slc]
+        x_nd_new[:] = (
+            new_from_src[self.index_field]
+            +
+            self.x_offset
+        )
+
+        # x_nd = self.x_nd[self.xy_slice]
+        # y_nd = self.y_nd[self.xy_slice]
+        # name = self.viz.name
+        # if 'trade_rate' == name:
+        #     s = 4
+        #     print(
+        #         f'{name.upper()}:\n'
+        #         'NEW_FROM_SRC:\n'
+        #         f'new_from_src: {new_from_src}\n\n'
+
+        #         f'PRE self.x_nd:'
+        #         f'\n{list(x_nd[-s:])}\n'
+
+        #         f'PRE self.y_nd:\n'
+        #         f'{list(y_nd[-s:])}\n\n'
+
+        #         f'TO WRITE:\n'
+
+        #         f'x_nd_new:\n'
+        #         f'{x_nd_new[0]}\n'
+
+        #         f'y_nd_new:\n'
+        #         f'{y_nd_new}\n'
+        #     )
+
+    # XXX: was ``.format_xy()``
+    def format_xy_nd_to_1d(
+        self,
+
+        array: np.ndarray,
+        array_key: str,
+        vr: tuple[int, int],
+
+    ) -> tuple[
+        np.ndarray,  # 1d x
+        np.ndarray,  # 1d y
+        np.ndarray | str,  # connection array/style
+    ]:
+        '''
+        Default xy-nd array to 1d pre-graphics-path render routine.
+
+        Return single field column data verbatim
+
+        '''
+        # NOTE: we don't include the very last datum which is filled in
+        # normally by another graphics object.
+        x_1d = array[self.index_field][:-1]
+        y_1d = array[array_key][:-1]
+
+        # name = self.viz.name
+        # if 'trade_rate' == name:
+        #     s = 4
+        #     x_nd = list(self.x_nd[self.xy_slice][-s:-1])
+        #     y_nd = list(self.y_nd[self.xy_slice][-s:-1])
+        #     print(
+        #         f'{name}:\n'
+        #         f'XY data:\n'
+        #         f'x: {x_nd}\n'
+        #         f'y: {y_nd}\n\n'
+        #         f'x_1d: {list(x_1d[-s:])}\n'
+        #         f'y_1d: {list(y_1d[-s:])}\n\n'
+
+        #     )
+        return (
+            x_1d,
+            y_1d,
+
+            # 1d connection array or style-key to
+            # ``pg.functions.arrayToQPath()``
+            'all',
+        )
+
+
+class OHLCBarsFmtr(IncrementalFormatter):
+    x_offset: np.ndarray = np.array([
+        -0.5,
+        0,
+        0,
+        0.5,
+    ])
+
+    fields: list[str] = ['open', 'high', 'low', 'close']
+    flat_index_ratio: float = 4
+
+    def allocate_xy_nd(
+        self,
+
+        ohlc_shm: ShmArray,
+        data_field: str,
+
+    ) -> tuple[
+        np.ndarray,  # x
+        np.nd.array  # y
+    ]:
+        '''
+        Convert an input struct-array holding OHLC samples into a pair of
+        flattened x, y arrays with the same size (datums wise) as the source
+        data.
+
+        '''
+        y_nd = ohlc_shm.ustruct(self.fields)
+
+        # generate an flat-interpolated x-domain
+        x_nd = (
+            np.broadcast_to(
+                ohlc_shm._array[self.index_field][:, None],
+                (
+                    ohlc_shm._array.size,
+                    # 4,  # only ohlc
+                    y_nd.shape[1],
+                ),
+            )
+            +
+            self.x_offset
+        )
+        assert y_nd.any()
+
+        # write pushed data to flattened copy
+        return (
+            x_nd,
+            y_nd,
+        )
+
+    def incr_update_xy_nd(
+        self,
+
+        src_shm: ShmArray,
+        data_field: str,
+
+        new_from_src: np.ndarray,  # portion of source that was updated
+
+        read_slc: slice,
+        ln: int,  # len of updated
+
+        nd_start: int,
+        nd_stop: int,
+
+        is_append: bool,
+
+    ) -> None:
+        # write newly pushed data to flattened copy
+        # a struct-arr is always passed in.
+        new_y_nd = rfn.structured_to_unstructured(
+            new_from_src[self.fields]
+        )
+        self.y_nd[read_slc] = new_y_nd
+
+        # generate same-valued-per-row x support based on y shape
+        x_nd_new = self.x_nd[read_slc]
+        x_nd_new[:] = np.broadcast_to(
+            new_from_src[self.index_field][:, None],
+            new_y_nd.shape,
+        ) + self.x_offset
+
+    # TODO: can we drop this frame and just use the above?
+    def format_xy_nd_to_1d(
+        self,
+
+        array: np.ndarray,
+        array_key: str,
+        vr: tuple[int, int],
+
+        start: int = 0,  # XXX: do we need this?
+        # 0.5 is no overlap between arms, 1.0 is full overlap
+        w: float = 0.16,
+
+    ) -> tuple[
+        np.ndarray,
+        np.ndarray,
+        np.ndarray,
+    ]:
+        '''
+        More or less direct proxy to the ``numba``-fied
+        ``path_arrays_from_ohlc()`` (above) but with closed in kwargs
+        for line spacing.
+
+        '''
+        x, y, c = path_arrays_from_ohlc(
+            array[:-1],
+            start,
+            bar_w=self.index_step_size,
+            bar_gap=w * self.index_step_size,
+
+            # XXX: don't ask, due to a ``numba`` bug..
+            use_time_index=(self.index_field == 'time'),
+        )
+        return x, y, c
+
+
+class OHLCBarsAsCurveFmtr(OHLCBarsFmtr):
+
+    def format_xy_nd_to_1d(
+        self,
+
+        array: np.ndarray,
+        array_key: str,
+        vr: tuple[int, int],
+
+    ) -> tuple[
+        np.ndarray,
+        np.ndarray,
+        str,
+    ]:
+        # TODO: in the case of an existing ``.update_xy()``
+        # should we be passing in array as an xy arrays tuple?
+
+        # 2 more datum-indexes to capture zero at end
+        x_flat = self.x_nd[self.xy_nd_start:self.xy_nd_stop-1]
+        y_flat = self.y_nd[self.xy_nd_start:self.xy_nd_stop-1]
+
+        # slice to view
+        ivl, ivr = vr
+        x_iv_flat = x_flat[ivl:ivr]
+        y_iv_flat = y_flat[ivl:ivr]
+
+        # reshape to 1d for graphics rendering
+        y_iv = y_iv_flat.reshape(-1)
+        x_iv = x_iv_flat.reshape(-1)
+
+        return x_iv, y_iv, 'all'
+
+
+class StepCurveFmtr(IncrementalFormatter):
+
+    x_offset: np.ndarray = np.array([
+        0,
+        1,
+    ])
+
+    def allocate_xy_nd(
+        self,
+
+        shm: ShmArray,
+        data_field: str,
+
+    ) -> tuple[
+        np.ndarray,  # x
+        np.nd.array  # y
+    ]:
+        '''
+        Convert an input 1d shm array to a "step array" format
+        for use by path graphics generation.
+
+        '''
+        i = shm._array[self.index_field].copy()
+        out = shm._array[data_field].copy()
+
+        x_out = (
+            np.broadcast_to(
+                i[:, None],
+                (i.size, 2),
+            )
+            +
+            self.x_offset
+        )
+
+        # fill out Nx2 array to hold each step's left + right vertices.
+        y_out = np.empty(
+            x_out.shape,
+            dtype=out.dtype,
+        )
+        # fill in (current) values from source shm buffer
+        y_out[:] = out[:, np.newaxis]
+
+        # TODO: pretty sure we can drop this?
+        # start y at origin level
+        # y_out[0, 0] = 0
+        # y_out[self.xy_nd_start] = 0
+        return x_out, y_out
+
+    def incr_update_xy_nd(
+        self,
+
+        src_shm: ShmArray,
+        array_key: str,
+
+        new_from_src: np.ndarray,  # portion of source that was updated
+        read_slc: slice,
+        ln: int,  # len of updated
+
+        nd_start: int,
+        nd_stop: int,
+
+        is_append: bool,
+
+    ) -> tuple[
+        np.ndarray,
+        slice,
+    ]:
+        # NOTE: for a step curve we slice from one datum prior
+        # to the current "update slice" to get the previous
+        # "level".
+        #
+        # why this is needed,
+        # - the current new append slice will often have a zero
+        #   value in the latest datum-step (at least for zero-on-new
+        #   cases like vlm in the) as per configuration of the FSP
+        #   engine.
+        # - we need to look back a datum to get the last level which
+        #   will be used to terminate/complete the last step x-width
+        #   which will be set to pair with the last x-index THIS MEANS
+        #
+        # XXX: this means WE CAN'T USE the append slice since we need to
+        # "look backward" one step to get the needed back-to-zero level
+        # and the update data in ``new_from_src`` will only contain the
+        # latest new data.
+        back_1 = slice(
+            read_slc.start - 1,
+            read_slc.stop,
+        )
+
+        to_write = src_shm._array[back_1]
+        y_nd_new = self.y_nd[back_1]
+        y_nd_new[:] = to_write[array_key][:, None]
+
+        x_nd_new = self.x_nd[read_slc]
+        x_nd_new[:] = (
+            new_from_src[self.index_field][:, None]
+            +
+            self.x_offset
+        )
+
+        # XXX: uncomment for debugging
+        # x_nd = self.x_nd[self.xy_slice]
+        # y_nd = self.y_nd[self.xy_slice]
+        # name = self.viz.name
+        # if 'dolla_vlm' in name:
+        #     s = 4
+        #     print(
+        #         f'{name}:\n'
+        #         'NEW_FROM_SRC:\n'
+        #         f'new_from_src: {new_from_src}\n\n'
+
+        #         f'PRE self.x_nd:'
+        #         f'\n{x_nd[-s:]}\n'
+        #         f'PRE self.y_nd:\n'
+        #         f'{y_nd[-s:]}\n\n'
+
+        #         f'TO WRITE:\n'
+        #         f'x_nd_new:\n'
+        #         f'{x_nd_new}\n'
+        #         f'y_nd_new:\n'
+        #         f'{y_nd_new}\n'
+        #     )
+
+    def format_xy_nd_to_1d(
+        self,
+
+        array: np.ndarray,
+        array_key: str,
+        vr: tuple[int, int],
+
+    ) -> tuple[
+        np.ndarray,
+        np.ndarray,
+        str,
+    ]:
+        last_t, last = array[-1][[self.index_field, array_key]]
+
+        start = self.xy_nd_start
+        stop = self.xy_nd_stop
+
+        x_step = self.x_nd[start:stop]
+        y_step = self.y_nd[start:stop]
+
+        # slice out in-view data
+        ivl, ivr = vr
+
+        # NOTE: add an extra step to get the vertical-line-down-to-zero
+        # adjacent to the last-datum graphic (filled rect).
+        x_step_iv = x_step[ivl:ivr+1]
+        y_step_iv = y_step[ivl:ivr+1]
+
+        # flatten to 1d
+        x_1d = x_step_iv.reshape(x_step_iv.size)
+        y_1d = y_step_iv.reshape(y_step_iv.size)
+
+        # debugging
+        # if y_1d.any():
+        #     s = 6
+        #     print(
+        #         f'x_step_iv:\n{x_step_iv[-s:]}\n'
+        #         f'y_step_iv:\n{y_step_iv[-s:]}\n\n'
+        #         f'x_1d:\n{x_1d[-s:]}\n'
+        #         f'y_1d:\n{y_1d[-s:]}\n'
+        #     )
+
+        return x_1d, y_1d, 'all'

piker/data/_m4.py

@@ -15,17 +15,30 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 '''
-Graphics related downsampling routines for compressing to pixel
-limits on the display device.
+Graphics downsampling using the infamous M4 algorithm.
+
+This is one of ``piker``'s secret weapons allowing us to boss all other
+charting platforms B)
+
+(AND DON'T YOU DARE TAKE THIS CODE WITHOUT CREDIT OR WE'LL SUE UR F#&@* ASS).
+
+NOTES: this method is a so called "visualization driven data
+aggregation" approach. It gives error-free line chart
+downsampling, see
+further scientific paper resources:
+- http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
+- http://www.vldb.org/2014/program/papers/demo/p997-jugel.pdf
+
+Details on implementation of this algo are based in,
+https://github.com/pikers/piker/issues/109
 
 '''
 import math
 from typing import Optional
 
 import numpy as np
-from numpy.lib import recfunctions as rfn
 from numba import (
-    jit,
+    njit,
     # float64, optional, int64,
 )
 
@@ -35,109 +48,6 @@ from ..log import get_logger
 log = get_logger(__name__)
 
 
-def hl2mxmn(ohlc: np.ndarray) -> np.ndarray:
-    '''
-    Convert a OHLC struct-array containing 'high'/'low' columns
-    to a "joined" max/min 1-d array.
-
-    '''
-    index = ohlc['index']
-    hls = ohlc[[
-        'low',
-        'high',
-    ]]
-
-    mxmn = np.empty(2*hls.size, dtype=np.float64)
-    x = np.empty(2*hls.size, dtype=np.float64)
-    trace_hl(hls, mxmn, x, index[0])
-    x = x + index[0]
-
-    return mxmn, x
-
-
-@jit(
-    # TODO: the type annots..
-    # float64[:](float64[:],),
-    nopython=True,
-)
-def trace_hl(
-    hl: 'np.ndarray',
-    out: np.ndarray,
-    x: np.ndarray,
-    start: int,
-
-    # the "offset" values in the x-domain which
-    # place the 2 output points around each ``int``
-    # master index.
-    margin: float = 0.43,
-
-) -> None:
-    '''
-    "Trace" the outline of the high-low values of an ohlc sequence
-    as a line such that the maximum deviation (aka disperaion) between
-    bars if preserved.
-
-    This routine is expected to modify input arrays in-place.
-
-    '''
-    last_l = hl['low'][0]
-    last_h = hl['high'][0]
-
-    for i in range(hl.size):
-        row = hl[i]
-        l, h = row['low'], row['high']
-
-        up_diff = h - last_l
-        down_diff = last_h - l
-
-        if up_diff > down_diff:
-            out[2*i + 1] = h
-            out[2*i] = last_l
-        else:
-            out[2*i + 1] = l
-            out[2*i] = last_h
-
-        last_l = l
-        last_h = h
-
-        x[2*i] = int(i) - margin
-        x[2*i + 1] = int(i) + margin
-
-    return out
-
-
-def ohlc_flatten(
-    ohlc: np.ndarray,
-    use_mxmn: bool = True,
-
-) -> tuple[np.ndarray, np.ndarray]:
-    '''
-    Convert an OHLCV struct-array into a flat ready-for-line-plotting
-    1-d array that is 4 times the size with x-domain values distributed
-    evenly (by 0.5 steps) over each index.
-
-    '''
-    index = ohlc['index']
-
-    if use_mxmn:
-        # traces a line optimally over highs to lows
-        # using numba. NOTE: pretty sure this is faster
-        # and looks about the same as the below output.
-        flat, x = hl2mxmn(ohlc)
-
-    else:
-        flat = rfn.structured_to_unstructured(
-            ohlc[['open', 'high', 'low', 'close']]
-        ).flatten()
-
-        x = np.linspace(
-            start=index[0] - 0.5,
-            stop=index[-1] + 0.5,
-            num=len(flat),
-        )
-    return x, flat
-
-
 def ds_m4(
     x: np.ndarray,
     y: np.ndarray,
@@ -160,16 +70,6 @@ def ds_m4(
     This is more or less an OHLC style sampling of a line-style series.
 
     '''
-    # NOTE: this method is a so called "visualization driven data
-    # aggregation" approach. It gives error-free line chart
-    # downsampling, see
-    # further scientific paper resources:
-    # - http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
-    # - http://www.vldb.org/2014/program/papers/demo/p997-jugel.pdf
-
-    # Details on implementation of this algo are based in,
-    # https://github.com/pikers/piker/issues/109
-
     # XXX: from infinite on downsampling viewable graphics:
     # "one thing i remembered about the binning - if you are
     # picking a range within your timeseries the start and end bin
@@ -191,6 +91,14 @@ def ds_m4(
         x_end = x[-1]  # x end value/highest in domain
         xrange = (x_end - x_start)
 
+    if xrange < 0:
+        log.error(f'-VE M4 X-RANGE: {x_start} -> {x_end}')
+        # XXX: broken x-range calc-case, likely the x-end points
+        # are wrong and have some default value set (such as
+        # x_end -> <some epoch float> while x_start -> 0.5).
+        # breakpoint()
+        return None
+
     # XXX: always round up on the input pixels
     # lnx = len(x)
     # uppx *= max(4 / (1 + math.log(uppx, 2)), 1)
@@ -256,8 +164,7 @@ def ds_m4(
     return nb, x_out, y_out, ymn, ymx
 
 
-@jit(
-    nopython=True,
+@njit(
     nogil=True,
 )
 def _m4(

piker/data/_pathops.py

@@ -0,0 +1,452 @@
+# piker: trading gear for hackers
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Super fast ``QPainterPath`` generation related operator routines.
+
+"""
+from math import (
+    ceil,
+    floor,
+)
+
+import numpy as np
+from numpy.lib import recfunctions as rfn
+from numba import (
+    # types,
+    njit,
+    float64,
+    int64,
+    # optional,
+)
+
+# TODO: for ``numba`` typing..
+# from ._source import numba_ohlc_dtype
+from ._m4 import ds_m4
+from .._profile import (
+    Profiler,
+    pg_profile_enabled,
+    ms_slower_then,
+)
+
+
+def xy_downsample(
+    x,
+    y,
+    uppx,
+
+    x_spacer: float = 0.5,
+
+) -> tuple[
+    np.ndarray,
+    np.ndarray,
+    float,
+    float,
+]:
+    '''
+    Downsample 1D (flat ``numpy.ndarray``) arrays using M4 given an input
+    ``uppx`` (units-per-pixel) and add space between discreet datums.
+
+    '''
+    # downsample whenever more then 1 pixels per datum can be shown.
+    # always refresh data bounds until we get diffing
+    # working properly, see above..
+    m4_out = ds_m4(
+        x,
+        y,
+        uppx,
+    )
+
+    if m4_out is not None:
+        bins, x, y, ymn, ymx = m4_out
+        # flatten output to 1d arrays suitable for path-graphics generation.
+        x = np.broadcast_to(x[:, None], y.shape)
+        x = (x + np.array(
+            [-x_spacer, 0, 0, x_spacer]
+        )).flatten()
+        y = y.flatten()
+
+        return x, y, ymn, ymx
+
+    # XXX: we accept a None output for the case where the input range
+    # to ``ds_m4()`` is bad (-ve) and we want to catch and debug
+    # that (seemingly super rare) circumstance..
+    return None
+
+
+@njit(
+    # NOTE: need to construct this manually for readonly
+    # arrays, see https://github.com/numba/numba/issues/4511
+    # (
+    #     types.Array(
+    #         numba_ohlc_dtype,
+    #         1,
+    #         'C',
+    #         readonly=True,
+    #     ),
+    #     int64,
+    #     types.unicode_type,
+    #     optional(float64),
+    # ),
+    nogil=True
+)
+def path_arrays_from_ohlc(
+    data: np.ndarray,
+    start: int64,
+    bar_w: float64,
+    bar_gap: float64 = 0.16,
+    use_time_index: bool = True,
+
+    # XXX: ``numba`` issue: https://github.com/numba/numba/issues/8622
+    # index_field: str,
+
+) -> tuple[
+    np.ndarray,
+    np.ndarray,
+    np.ndarray,
+]:
+    '''
+    Generate an array of lines objects from input ohlc data.
+
+    '''
+    size = int(data.shape[0] * 6)
+
+    # XXX: see this for why the dtype might have to be defined outside
+    # the routine.
+    # https://github.com/numba/numba/issues/4098#issuecomment-493914533
+    x = np.zeros(
+        shape=size,
+        dtype=float64,
+    )
+    y, c = x.copy(), x.copy()
+
+    half_w: float = bar_w/2
+
+    # TODO: report bug for assert @
+    # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
+    for i, q in enumerate(data[start:], start):
+
+        open = q['open']
+        high = q['high']
+        low = q['low']
+        close = q['close']
+
+        if use_time_index:
+            index = float64(q['time'])
+        else:
+            index = float64(q['index'])
+
+        # XXX: ``numba`` issue: https://github.com/numba/numba/issues/8622
+        # index = float64(q[index_field])
+        # AND this (probably)
+        # open, high, low, close, index = q[
+        #     ['open', 'high', 'low', 'close', 'index']]
+
+        istart = i * 6
+        istop = istart + 6
+
+        # x,y detail the 6 points which connect all vertexes of a ohlc bar
+        mid: float = index + half_w
+        x[istart:istop] = (
+            index + bar_gap,
+            mid,
+            mid,
+            mid,
+            mid,
+            index + bar_w - bar_gap,
+        )
+        y[istart:istop] = (
+            open,
+            open,
+            low,
+            high,
+            close,
+            close,
+        )
+
+        # specifies that the first edge is never connected to the
+        # prior bars last edge thus providing a small "gap"/"space"
+        # between bars determined by ``bar_gap``.
+        c[istart:istop] = (1, 1, 1, 1, 1, 0)
+
+    return x, y, c
+
+
+def hl2mxmn(
+    ohlc: np.ndarray,
+    index_field: str = 'index',
+
+) -> np.ndarray:
+    '''
+    Convert a OHLC struct-array containing 'high'/'low' columns
+    to a "joined" max/min 1-d array.
+
+    '''
+    index = ohlc[index_field]
+    hls = ohlc[[
+        'low',
+        'high',
+    ]]
+
+    mxmn = np.empty(2*hls.size, dtype=np.float64)
+    x = np.empty(2*hls.size, dtype=np.float64)
+    trace_hl(hls, mxmn, x, index[0])
+    x = x + index[0]
+
+    return mxmn, x
+
+
+@njit(
+    # TODO: the type annots..
+    # float64[:](float64[:],),
+)
+def trace_hl(
+    hl: 'np.ndarray',
+    out: np.ndarray,
+    x: np.ndarray,
+    start: int,
+
+    # the "offset" values in the x-domain which
+    # place the 2 output points around each ``int``
+    # master index.
+    margin: float = 0.43,
+
+) -> None:
+    '''
+    "Trace" the outline of the high-low values of an ohlc sequence
+    as a line such that the maximum deviation (aka disperaion) between
+    bars if preserved.
+
+    This routine is expected to modify input arrays in-place.
+
+    '''
+    last_l = hl['low'][0]
+    last_h = hl['high'][0]
+
+    for i in range(hl.size):
+        row = hl[i]
+        l, h = row['low'], row['high']
+
+        up_diff = h - last_l
+        down_diff = last_h - l
+
+        if up_diff > down_diff:
+            out[2*i + 1] = h
+            out[2*i] = last_l
+        else:
+            out[2*i + 1] = l
+            out[2*i] = last_h
+
+        last_l = l
+        last_h = h
+
+        x[2*i] = int(i) - margin
+        x[2*i + 1] = int(i) + margin
+
+    return out
+
+
+def ohlc_flatten(
+    ohlc: np.ndarray,
+    use_mxmn: bool = True,
+    index_field: str = 'index',
+
+) -> tuple[np.ndarray, np.ndarray]:
+    '''
+    Convert an OHLCV struct-array into a flat ready-for-line-plotting
+    1-d array that is 4 times the size with x-domain values distributed
+    evenly (by 0.5 steps) over each index.
+
+    '''
+    index = ohlc[index_field]
+
+    if use_mxmn:
+        # traces a line optimally over highs to lows
+        # using numba. NOTE: pretty sure this is faster
+        # and looks about the same as the below output.
+        flat, x = hl2mxmn(ohlc)
+
+    else:
+        flat = rfn.structured_to_unstructured(
+            ohlc[['open', 'high', 'low', 'close']]
+        ).flatten()
+
+        x = np.linspace(
+            start=index[0] - 0.5,
+            stop=index[-1] + 0.5,
+            num=len(flat),
+        )
+    return x, flat
+
+
+def slice_from_time(
+    arr: np.ndarray,
+    start_t: float,
+    stop_t: float,
+    step: int | None = None,
+
+) -> slice:
+    '''
+    Calculate array indices mapped from a time range and return them in
+    a slice.
+
+    Given an input array with an epoch `'time'` series entry, calculate
+    the indices which span the time range and return in a slice. Presume
+    each `'time'` step increment is uniform and when the time stamp
+    series contains gaps (the uniform presumption is untrue) use
+    ``np.searchsorted()`` binary search to look up the appropriate
+    index.
+
+    '''
+    profiler = Profiler(
+        msg='slice_from_time()',
+        disabled=not pg_profile_enabled(),
+        ms_threshold=ms_slower_then,
+    )
+
+    times = arr['time']
+    t_first = floor(times[0])
+    t_last = ceil(times[-1])
+
+    # the greatest index we can return which slices to the
+    # end of the input array.
+    read_i_max = arr.shape[0]
+
+    # TODO: require this is always passed in?
+    if step is None:
+        step = round(t_last - times[-2])
+        if step == 0:
+            step = 1
+
+    # compute (presumed) uniform-time-step index offsets
+    i_start_t = floor(start_t)
+    read_i_start = floor(((i_start_t - t_first) // step)) - 1
+
+    i_stop_t = ceil(stop_t)
+
+    # XXX: edge case -> always set stop index to last in array whenever
+    # the input stop time is detected to be greater then the equiv time
+    # stamp at that last entry.
+    if i_stop_t >= t_last:
+        read_i_stop = read_i_max
+    else:
+        read_i_stop = ceil((i_stop_t - t_first) // step) + 1
+
+    # always clip outputs to array support
+    # for read start:
+    # - never allow a start < the 0 index
+    # - never allow an end index > the read array len
+    read_i_start = min(
+        max(0, read_i_start),
+        read_i_max - 1,
+    )
+    read_i_stop = max(
+        0,
+        min(read_i_stop, read_i_max),
+    )
+
+    # check for larger-then-latest calculated index for given start
+    # time, in which case we do a binary search for the correct index.
+    # NOTE: this is usually the result of a time series with time gaps
+    # where it is expected that each index step maps to a uniform step
+    # in the time stamp series.
+    t_iv_start = times[read_i_start]
+    if (
+        t_iv_start > i_start_t
+    ):
+        # do a binary search for the best index mapping to ``start_t``
+        # given we measured an overshoot using the uniform-time-step
+        # calculation from above.
+
+        # TODO: once we start caching these per source-array,
+        # we can just overwrite ``read_i_start`` directly.
+        new_read_i_start = np.searchsorted(
+            times,
+            i_start_t,
+            side='left',
+        )
+
+        # TODO: minimize binary search work as much as possible:
+        # - cache these remap values which compensate for gaps in the
+        #   uniform time step basis where we calc a later start
+        #   index for the given input ``start_t``.
+        # - can we shorten the input search sequence by heuristic?
+        #   up_to_arith_start = index[:read_i_start]
+
+        if (
+            new_read_i_start <= read_i_start
+        ):
+            # t_diff = t_iv_start - start_t
+            # print(
+            #     f"WE'RE CUTTING OUT TIME - STEP:{step}\n"
+            #     f'start_t:{start_t} -> 0index start_t:{t_iv_start}\n'
+            #     f'diff: {t_diff}\n'
+            #     f'REMAPPED START i: {read_i_start} -> {new_read_i_start}\n'
+            # )
+            read_i_start = new_read_i_start - 1
+
+    t_iv_stop = times[read_i_stop - 1]
+    if (
+        t_iv_stop > i_stop_t
+    ):
+        # t_diff = stop_t - t_iv_stop
+        # print(
+        #     f"WE'RE CUTTING OUT TIME - STEP:{step}\n"
+        #     f'calced iv stop:{t_iv_stop} -> stop_t:{stop_t}\n'
+        #     f'diff: {t_diff}\n'
+        #     # f'SHOULD REMAP STOP: {read_i_start} -> {new_read_i_start}\n'
+        # )
+        new_read_i_stop = np.searchsorted(
+            times[read_i_start:],
+            # times,
+            i_stop_t,
+            side='left',
+        )
+
+        if (
+            new_read_i_stop <= read_i_stop
+        ):
+            read_i_stop = read_i_start + new_read_i_stop + 1
+
+    # sanity checks for range size
+    # samples = (i_stop_t - i_start_t) // step
+    # index_diff = read_i_stop - read_i_start + 1
+    # if index_diff > (samples + 3):
+    #     breakpoint()
+
+    # read-relative indexes: gives a slice where `shm.array[read_slc]`
+    # will be the data spanning the input time range `start_t` ->
+    # `stop_t`
+    read_slc = slice(
+        int(read_i_start),
+        int(read_i_stop),
+    )
+
+    profiler(
+        'slicing complete'
+        # f'{start_t} -> {abs_slc.start} | {read_slc.start}\n'
+        # f'{stop_t} -> {abs_slc.stop} | {read_slc.stop}\n'
+    )
+
+    # NOTE: if caller needs absolute buffer indices they can
+    # slice the buffer abs index like so:
+    # index = arr['index']
+    # abs_indx = index[read_slc]
+    # abs_slc = slice(
+    #     int(abs_indx[0]),
+    #     int(abs_indx[-1]),
+    # )
+
+    return read_slc

piker/data/_sampling.py

@@ -253,20 +253,30 @@ class Sampler:
             # f'consumers: {subs}'
         )
         borked: set[tractor.MsgStream] = set()
-        for stream in subs:
+        sent: set[tractor.MsgStream] = set()
+        while True:
             try:
-                await stream.send({
-                    'index': time_stamp or last_ts,
-                    'period': period_s,
-                })
-            except (
-                trio.BrokenResourceError,
-                trio.ClosedResourceError
-            ):
-                log.error(
-                    f'{stream._ctx.chan.uid} dropped connection'
-                )
-                borked.add(stream)
+                for stream in (subs - sent):
+                    try:
+                        await stream.send({
+                            'index': time_stamp or last_ts,
+                            'period': period_s,
+                        })
+                        sent.add(stream)
+
+                    except (
+                        trio.BrokenResourceError,
+                        trio.ClosedResourceError
+                    ):
+                        log.error(
+                            f'{stream._ctx.chan.uid} dropped connection'
+                        )
+                        borked.add(stream)
+                else:
+                    break
+            except RuntimeError:
+                log.warning(f'Client subs {subs} changed while broadcasting')
+                continue
 
         for stream in borked:
             try:
@@ -848,6 +858,16 @@ async def uniform_rate_send(
         # rate timing exactly lul
         try:
             await stream.send({sym: first_quote})
+        except tractor.RemoteActorError as rme:
+            if rme.type is not tractor._exceptions.StreamOverrun:
+                raise
+            ctx = stream._ctx
+            chan = ctx.chan
+            log.warning(
+                'Throttled quote-stream overrun!\n'
+                f'{sym}:{ctx.cid}@{chan.uid}'
+            )
+
         except (
             # NOTE: any of these can be raised by ``tractor``'s IPC
             # transport-layer and we want to be highly resilient

piker/data/feed.py

@@ -207,7 +207,7 @@ def get_feed_bus(
 
 ) -> _FeedsBus:
     '''
-    Retreive broker-daemon-local data feeds bus from process global
+    Retrieve broker-daemon-local data feeds bus from process global
     scope. Serialize task access to lock.
 
     '''
@@ -250,6 +250,7 @@ async def start_backfill(
     shm: ShmArray,
     timeframe: float,
     sampler_stream: tractor.MsgStream,
+    feed_is_live: trio.Event,
 
     last_tsdb_dt: Optional[datetime] = None,
     storage: Optional[Storage] = None,
@@ -281,7 +282,14 @@ async def start_backfill(
             - pendulum.from_timestamp(times[-2])
         ).seconds
 
-        if step_size_s == 60:
+        # if the market is open (aka we have a live feed) but the
+        # history sample step index seems off we report the surrounding
+        # data and drop into a bp. this case shouldn't really ever
+        # happen if we're doing history retrieval correctly.
+        if (
+            step_size_s == 60
+            and feed_is_live.is_set()
+        ):
             inow = round(time.time())
             diff = inow - times[-1]
             if abs(diff) > 60:
@@ -499,6 +507,7 @@ async def basic_backfill(
     bfqsn: str,
     shms: dict[int, ShmArray],
     sampler_stream: tractor.MsgStream,
+    feed_is_live: trio.Event,
 
 ) -> None:
 
@@ -518,6 +527,7 @@ async def basic_backfill(
                     shm,
                     timeframe,
                     sampler_stream,
+                    feed_is_live,
                 )
             )
         except DataUnavailable:
@@ -534,6 +544,7 @@ async def tsdb_backfill(
     bfqsn: str,
     shms: dict[int, ShmArray],
     sampler_stream: tractor.MsgStream,
+    feed_is_live: trio.Event,
 
     task_status: TaskStatus[
         tuple[ShmArray, ShmArray]
@@ -568,6 +579,8 @@ async def tsdb_backfill(
                     shm,
                     timeframe,
                     sampler_stream,
+                    feed_is_live,
+
                     last_tsdb_dt=last_tsdb_dt,
                     tsdb_is_up=True,
                     storage=storage,
@@ -870,6 +883,7 @@ async def manage_history(
                         60: hist_shm,
                     },
                     sample_stream,
+                    feed_is_live,
                 )
 
                 # yield back after client connect with filled shm
@@ -904,6 +918,7 @@ async def manage_history(
                     60: hist_shm,
                 },
                 sample_stream,
+                feed_is_live,
             )
             task_status.started((
                 hist_zero_index,
@@ -1037,12 +1052,11 @@ async def allocate_persistent_feed(
 
     flume = Flume(
         symbol=symbol,
-        _hist_shm_token=hist_shm.token,
-        _rt_shm_token=rt_shm.token,
         first_quote=first_quote,
+        _rt_shm_token=rt_shm.token,
+        _hist_shm_token=hist_shm.token,
         izero_hist=izero_hist,
         izero_rt=izero_rt,
-        # throttle_rate=tick_throttle,
     )
 
     # for ambiguous names we simply apply the retreived
@@ -1066,7 +1080,10 @@ async def allocate_persistent_feed(
     # seed the buffer with a history datum - this is most handy
     # for many backends which don't sample @ 1s OHLC but do have
     # slower data such as 1m OHLC.
-    if not len(rt_shm.array):
+    if (
+        not len(rt_shm.array)
+        and hist_shm.array.size
+    ):
         rt_shm.push(hist_shm.array[-3:-1])
         ohlckeys = ['open', 'high', 'low', 'close']
         rt_shm.array[ohlckeys][-2:] = hist_shm.array['close'][-1]
@@ -1077,6 +1094,9 @@ async def allocate_persistent_feed(
         rt_shm.array['time'][0] = ts
         rt_shm.array['time'][1] = ts + 1
 
+    elif hist_shm.array.size == 0:
+        await tractor.breakpoint()
+
     # wait the spawning parent task to register its subscriber
     # send-stream entry before we start the sample loop.
     await sub_registered.wait()
@@ -1569,6 +1589,9 @@ async def open_feed(
                 (brokermod, bfqsns),
             ) in zip(ctxs, providers.items()):
 
+                # NOTE: do it asap to avoid overruns during multi-feed setup?
+                ctx._backpressure = backpressure
+
                 for fqsn, flume_msg in flumes_msg_dict.items():
                     flume = Flume.from_msg(flume_msg)
                     assert flume.symbol.fqsn == fqsn

piker/data/flows.py

@@ -22,17 +22,11 @@ real-time data processing data-structures.
 
 """
 from __future__ import annotations
-from contextlib import asynccontextmanager as acm
-from functools import partial
 from typing import (
-    AsyncIterator,
     TYPE_CHECKING,
 )
 
 import tractor
-from tractor.trionics import (
-    maybe_open_context,
-)
 import pendulum
 import numpy as np
 
@@ -45,12 +39,13 @@ from ._sharedmem import (
     ShmArray,
     _Token,
 )
-from ._sampling import (
-    open_sample_stream,
-)
+# from .._profile import (
+#     Profiler,
+#     pg_profile_enabled,
+# )
 
 if TYPE_CHECKING:
-    from pyqtgraph import PlotItem
+    # from pyqtgraph import PlotItem
     from .feed import Feed
 
 
@@ -147,26 +142,6 @@ class Flume(Struct):
     async def receive(self) -> dict:
         return await self.stream.receive()
 
-    @acm
-    async def index_stream(
-        self,
-        delay_s: float = 1,
-
-    ) -> AsyncIterator[int]:
-
-        if not self.feed:
-            raise RuntimeError('This flume is not part of any ``Feed``?')
-
-        # TODO: maybe a public (property) API for this in ``tractor``?
-        portal = self.stream._ctx._portal
-        assert portal
-
-        # XXX: this should be singleton on a host,
-        # a lone broker-daemon per provider should be
-        # created for all practical purposes
-        async with open_sample_stream(float(delay_s)) as stream:
-            yield stream
-
     def get_ds_info(
         self,
     ) -> tuple[float, float, float]:
@@ -218,104 +193,18 @@ class Flume(Struct):
     def get_index(
         self,
         time_s: float,
+        array: np.ndarray,
 
-    ) -> int:
+    ) -> int | float:
         '''
         Return array shm-buffer index for for epoch time.
 
         '''
-        array = self.rt_shm.array
         times = array['time']
-        mask = (times >= time_s)
-
-        if any(mask):
-            return array['index'][mask][0]
-
-        # just the latest index
-        array['index'][-1]
-
-    def slice_from_time(
-        self,
-        array: np.ndarray,
-        start_t: float,
-        stop_t: float,
-        timeframe_s: int = 1,
-        return_data: bool = False,
-
-    ) -> np.ndarray:
-        '''
-        Slice an input struct array providing only datums
-        "in view" of this chart.
-
-        '''
-        arr = {
-            1: self.rt_shm.array,
-            60: self.hist_shm.arry,
-        }[timeframe_s]
-
-        times = arr['time']
-        index = array['index']
-
-        # use advanced indexing to map the
-        # time range to the index range.
-        mask = (
-            (times >= start_t)
-            &
-            (times < stop_t)
+        first = np.searchsorted(
+            times,
+            time_s,
+            side='left',
         )
-
-        # TODO: if we can ensure each time field has a uniform
-        # step we can instead do some arithmetic to determine
-        # the equivalent index like we used to?
-        # return array[
-        #     lbar - ifirst:
-        #     (rbar - ifirst) + 1
-        # ]
-
-        i_by_t = index[mask]
-        i_0 = i_by_t[0]
-
-        abs_slc = slice(
-            i_0,
-            i_by_t[-1],
-        )
-        # slice data by offset from the first index
-        # available in the passed datum set.
-        read_slc = slice(
-            0,
-            i_by_t[-1] - i_0,
-        )
-        if not return_data:
-            return (
-                abs_slc,
-                read_slc,
-            )
-
-        # also return the readable data from the timerange
-        return (
-            abs_slc,
-            read_slc,
-            arr[mask],
-        )
-
-    def view_data(
-        self,
-        plot: PlotItem,
-        timeframe_s: int = 1,
-
-    ) -> np.ndarray:
-
-        # get far-side x-indices plot view
-        vr = plot.viewRect()
-
-        (
-            abs_slc,
-            buf_slc,
-            iv_arr,
-        ) = self.slice_from_time(
-            start_t=vr.left(),
-            stop_t=vr.right(),
-            timeframe_s=timeframe_s,
-            return_data=True,
-        )
-        return iv_arr
+        imx = times.shape[0] - 1
+        return min(first, imx)

piker/fsp/_engine.py

@@ -188,6 +188,8 @@ async def fsp_compute(
 
     history_by_field['time'] = src_time[-len(history_by_field):]
 
+    history_output['time'] = src.array['time']
+
     # TODO: XXX:
     # THERE'S A BIG BUG HERE WITH THE `index` field since we're
     # prepending a copy of the first value a few times to make

piker/pp.py

@@ -29,6 +29,7 @@ import re
 import time
 from typing import (
     Any,
+    Iterator,
     Optional,
     Union,
 )
@@ -53,7 +54,7 @@ def open_trade_ledger(
     broker: str,
     account: str,
 
-) -> str:
+) -> dict:
     '''
     Indempotently create and read in a trade log file from the
     ``<configuration_dir>/ledgers/`` directory.
@@ -116,6 +117,21 @@ class Transaction(Struct, frozen=True):
     # from: Optional[str] = None
 
 
+def iter_by_dt(
+    clears: dict[str, Any],
+) -> Iterator[tuple[str, dict]]:
+    '''
+    Iterate entries of a ``clears: dict`` table sorted by entry recorded
+    datetime presumably set at the ``'dt'`` field in each entry.
+
+    '''
+    for tid, data in sorted(
+        list(clears.items()),
+        key=lambda item: item[1]['dt'],
+    ):
+        yield tid, data
+
+
 class Position(Struct):
     '''
     Basic pp (personal/piker position) model with attached clearing
@@ -183,12 +199,7 @@ class Position(Struct):
         toml_clears_list = []
 
         # reverse sort so latest clears are at top of section?
-        for tid, data in sorted(
-            list(clears.items()),
-
-            # sort by datetime
-            key=lambda item: item[1]['dt'],
-        ):
+        for tid, data in iter_by_dt(clears):
             inline_table = toml.TomlDecoder().get_empty_inline_table()
 
             # serialize datetime to parsable `str`
@@ -301,6 +312,14 @@ class Position(Struct):
     # def lifo_price() -> float:
     #     ...
 
+    def iter_clears(self) -> Iterator[tuple[str, dict]]:
+        '''
+        Iterate the internally managed ``.clears: dict`` table in
+        datetime-stamped order.
+
+        '''
+        return iter_by_dt(self.clears)
+
     def calc_ppu(
         self,
         # include transaction cost in breakeven price
@@ -331,10 +350,9 @@ class Position(Struct):
         asize_h: list[float] = []  # historical accumulative size
         ppu_h: list[float] = []  # historical price-per-unit
 
-        clears = list(self.clears.items())
-
-        for i, (tid, entry) in enumerate(clears):
-
+        tid: str
+        entry: dict[str, Any]
+        for (tid, entry) in self.iter_clears():
             clear_size = entry['size']
             clear_price = entry['price']
 
@@ -344,6 +362,11 @@ class Position(Struct):
 
             sign_change: bool = False
 
+            if accum_size == 0:
+                ppu_h.append(0)
+                asize_h.append(0)
+                continue
+
             if accum_size == 0:
                 ppu_h.append(0)
                 asize_h.append(0)

piker/ui/_app.py

@@ -118,17 +118,10 @@ async def _async_main(
         # godwidget.hbox.addWidget(search)
         godwidget.search = search
 
-        symbols: list[str] = []
-
-        for sym in syms:
-            symbol, _, provider = sym.rpartition('.')
-            symbols.append(symbol)
-
         # this internally starts a ``display_symbol_data()`` task above
         order_mode_ready = await godwidget.load_symbols(
-            provider,
-            symbols,
-            loglevel
+            fqsns=syms,
+            loglevel=loglevel,
         )
 
         # spin up a search engine for the local cached symbol set
@@ -185,8 +178,7 @@ def _main(
     tractor_kwargs,
 ) -> None:
     '''
-    Sync entry point to start a chart: a ``tractor`` + Qt runtime
-    entry point
+    Sync entry point to start a chart: a ``tractor`` + Qt runtime.
 
     '''
     run_qtractor(

piker/ui/_axes.py

@@ -18,6 +18,7 @@
 Chart axes graphics and behavior.
 
 """
+from __future__ import annotations
 from functools import lru_cache
 from typing import Optional, Callable
 from math import floor
@@ -27,6 +28,7 @@ import pyqtgraph as pg
 from PyQt5 import QtCore, QtGui, QtWidgets
 from PyQt5.QtCore import QPointF
 
+from . import _pg_overrides as pgo
 from ..data._source import float_digits
 from ._label import Label
 from ._style import DpiAwareFont, hcolor, _font
@@ -46,8 +48,8 @@ class Axis(pg.AxisItem):
     '''
     def __init__(
         self,
-        linkedsplits,
-        typical_max_str: str = '100 000.000',
+        plotitem: pgo.PlotItem,
+        typical_max_str: str = '100 000.000  ',
         text_color: str = 'bracket',
         lru_cache_tick_strings: bool = True,
         **kwargs
@@ -61,36 +63,42 @@ class Axis(pg.AxisItem):
         # XXX: pretty sure this makes things slower
         # self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
 
-        self.linkedsplits = linkedsplits
+        self.pi = plotitem
         self._dpi_font = _font
 
         self.setTickFont(_font.font)
         font_size = self._dpi_font.font.pixelSize()
 
+        style_conf = {
+            'textFillLimits': [(0, 0.5)],
+            'tickFont': self._dpi_font.font,
+
+        }
+        text_offset = None
         if self.orientation in ('bottom',):
             text_offset = floor(0.25 * font_size)
 
         elif self.orientation in ('left', 'right'):
             text_offset = floor(font_size / 2)
 
-        self.setStyle(**{
-            'textFillLimits': [(0, 0.5)],
-            'tickFont': self._dpi_font.font,
-
-            # offset of text *away from* axis line in px
-            # use approx. half the font pixel size (height)
-            'tickTextOffset': text_offset,
-        })
+        if text_offset:
+            style_conf.update({
+                # offset of text *away from* axis line in px
+                # use approx. half the font pixel size (height)
+                'tickTextOffset': text_offset,
+            })
 
+        self.setStyle(**style_conf)
         self.setTickFont(_font.font)
 
         # NOTE: this is for surrounding "border"
         self.setPen(_axis_pen)
 
         # this is the text color
-        # self.setTextPen(pg.mkPen(hcolor(text_color)))
         self.text_color = text_color
 
+        # generate a bounding rect based on sizing to a "typical"
+        # maximum length-ed string defined as init default.
         self.typical_br = _font._qfm.boundingRect(typical_max_str)
 
         # size the pertinent axis dimension to a "typical value"
@@ -102,6 +110,9 @@ class Axis(pg.AxisItem):
                 maxsize=2**20
             )(self.tickStrings)
 
+        # axis "sticky" labels
+        self._stickies: dict[str, YAxisLabel] = {}
+
     # NOTE: only overriden to cast tick values entries into tuples
     # for use with the lru caching.
     def tickValues(
@@ -139,6 +150,38 @@ class Axis(pg.AxisItem):
     def txt_offsets(self) -> tuple[int, int]:
         return tuple(self.style['tickTextOffset'])
 
+    def add_sticky(
+        self,
+        pi: pgo.PlotItem,
+        name: None | str = None,
+        digits: None | int = 2,
+        bg_color='default',
+        fg_color='black',
+
+    ) -> YAxisLabel:
+
+        # if the sticky is for our symbol
+        # use the tick size precision for display
+        name = name or pi.name
+        digits = digits or 2
+
+        # TODO: ``._ysticks`` should really be an attr on each
+        # ``PlotItem`` now instead of the containing widget (because of
+        # overlays) ?
+
+        # add y-axis "last" value label
+        sticky = self._stickies[name] = YAxisLabel(
+            pi=pi,
+            parent=self,
+            digits=digits,  # TODO: pass this from symbol data
+            opacity=0.9,  # slight see-through
+            bg_color=bg_color,
+            fg_color=fg_color,
+        )
+
+        pi.sigRangeChanged.connect(sticky.update_on_resize)
+        return sticky
+
 
 class PriceAxis(Axis):
 
@@ -200,7 +243,6 @@ class PriceAxis(Axis):
         self._min_tick = size
 
     def size_to_values(self) -> None:
-        # self.typical_br = _font._qfm.boundingRect(typical_max_str)
         self.setWidth(self.typical_br.width())
 
     # XXX: drop for now since it just eats up h space
@@ -255,28 +297,50 @@ class DynamicDateAxis(Axis):
 
     ) -> list[str]:
 
-        chart = self.linkedsplits.chart
-        flow = chart._flows[chart.name]
-        shm = flow.shm
-        bars = shm.array
-        first = shm._first.value
+        # XX: ARGGGGG AG:LKSKDJF:LKJSDFD
+        chart = self.pi.chart_widget
 
-        bars_len = len(bars)
-        times = bars['time']
+        viz = chart._vizs[chart.name]
+        shm = viz.shm
+        array = shm.array
+        times = array['time']
+        i_0, i_l = times[0], times[-1]
 
-        epochs = times[list(
-            map(
-                int,
-                filter(
-                    lambda i: i > 0 and i < bars_len,
-                    (i-first for i in indexes)
+        # edge cases
+        if (
+            not indexes
+            or
+            (indexes[0] < i_0
+             and indexes[-1] < i_l)
+            or
+            (indexes[0] > i_0
+             and indexes[-1] > i_l)
+        ):
+            return []
+
+        if viz.index_field == 'index':
+            arr_len = times.shape[0]
+            first = shm._first.value
+            epochs = times[
+                list(
+                    map(
+                        int,
+                        filter(
+                            lambda i: i > 0 and i < arr_len,
+                            (i - first for i in indexes)
+                        )
+                    )
                 )
-            )
-        )]
+            ]
+        else:
+            epochs = list(map(int, indexes))
 
         # TODO: **don't** have this hard coded shift to EST
         # delay = times[-1] - times[-2]
-        dts = np.array(epochs, dtype='datetime64[s]')
+        dts = np.array(
+            epochs,
+            dtype='datetime64[s]',
+        )
 
         # see units listing:
         # https://numpy.org/devdocs/reference/arrays.datetime.html#datetime-units
@@ -294,24 +358,39 @@ class DynamicDateAxis(Axis):
         spacing: float,
 
     ) -> list[str]:
+
+        return self._indexes_to_timestrs(values)
+
+        # NOTE: handy for debugging the lru cache
         # info = self.tickStrings.cache_info()
         # print(info)
-        return self._indexes_to_timestrs(values)
 
 
 class AxisLabel(pg.GraphicsObject):
 
-    _x_margin = 0
-    _y_margin = 0
+    # relative offsets *OF* the bounding rect relative
+    # to parent graphics object.
+    # eg.  <parent>| => <_x_br_offset> => | <text> |
+    _x_br_offset: float = 0
+    _y_br_offset: float = 0
+
+    # relative offsets of text *within* bounding rect
+    # eg. | <_x_margin> => <text> |
+    _x_margin: float = 0
+    _y_margin: float = 0
+
+    # multiplier of the text content's height in order
+    # to force a larger (y-dimension) bounding rect.
+    _y_txt_h_scaling: float = 1
 
     def __init__(
         self,
         parent: pg.GraphicsItem,
         digits: int = 2,
 
-        bg_color: str = 'bracket',
+        bg_color: str = 'default',
         fg_color: str = 'black',
-        opacity: int = 1,  # XXX: seriously don't set this to 0
+        opacity: int = .8,  # XXX: seriously don't set this to 0
         font_size: str = 'default',
 
         use_arrow: bool = True,
@@ -322,6 +401,7 @@ class AxisLabel(pg.GraphicsObject):
         self.setParentItem(parent)
 
         self.setFlag(self.ItemIgnoresTransformations)
+        self.setZValue(100)
 
         # XXX: pretty sure this is faster
         self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
@@ -353,14 +433,14 @@ class AxisLabel(pg.GraphicsObject):
         p: QtGui.QPainter,
         opt: QtWidgets.QStyleOptionGraphicsItem,
         w: QtWidgets.QWidget
+
     ) -> None:
-        """Draw a filled rectangle based on the size of ``.label_str`` text.
+        '''
+        Draw a filled rectangle based on the size of ``.label_str`` text.
 
         Subtypes can customize further by overloading ``.draw()``.
 
-        """
-        # p.setCompositionMode(QtWidgets.QPainter.CompositionMode_SourceOver)
-
+        '''
         if self.label_str:
 
             # if not self.rect:
@@ -371,7 +451,11 @@ class AxisLabel(pg.GraphicsObject):
 
             p.setFont(self._dpifont.font)
             p.setPen(self.fg_color)
-            p.drawText(self.rect, self.text_flags, self.label_str)
+            p.drawText(
+                self.rect,
+                self.text_flags,
+                self.label_str,
+            )
 
     def draw(
         self,
@@ -379,6 +463,8 @@ class AxisLabel(pg.GraphicsObject):
         rect: QtCore.QRectF
     ) -> None:
 
+        p.setOpacity(self.opacity)
+
         if self._use_arrow:
             if not self.path:
                 self._draw_arrow_path()
@@ -386,15 +472,13 @@ class AxisLabel(pg.GraphicsObject):
             p.drawPath(self.path)
             p.fillPath(self.path, pg.mkBrush(self.bg_color))
 
-        # this adds a nice black outline around the label for some odd
-        # reason; ok by us
-        p.setOpacity(self.opacity)
-
         # this cause the L1 labels to glitch out if used in the subtype
         # and it will leave a small black strip with the arrow path if
         # done before the above
-        p.fillRect(self.rect, self.bg_color)
-
+        p.fillRect(
+            self.rect,
+            self.bg_color,
+        )
 
     def boundingRect(self):  # noqa
         '''
@@ -438,15 +522,18 @@ class AxisLabel(pg.GraphicsObject):
         txt_h, txt_w = txt_br.height(), txt_br.width()
         # print(f'wsw: {self._dpifont.boundingRect(" ")}')
 
-        # allow subtypes to specify a static width and height
+        # allow subtypes to override width and height
         h, w = self.size_hint()
-        # print(f'axis size: {self._parent.size()}')
-        # print(f'axis geo: {self._parent.geometry()}')
 
         self.rect = QtCore.QRectF(
-            0, 0,
+
+            # relative bounds offsets
+            self._x_br_offset,
+            self._y_br_offset,
+
             (w or txt_w) + self._x_margin / 2,
-            (h or txt_h) + self._y_margin / 2,
+
+            (h or txt_h) * self._y_txt_h_scaling + (self._y_margin / 2),
         )
         # print(self.rect)
         # hb = self.path.controlPointRect()
@@ -522,7 +609,7 @@ class XAxisLabel(AxisLabel):
 
 
 class YAxisLabel(AxisLabel):
-    _y_margin = 4
+    _y_margin: int = 4
 
     text_flags = (
         QtCore.Qt.AlignLeft
@@ -533,19 +620,19 @@ class YAxisLabel(AxisLabel):
 
     def __init__(
         self,
-        chart,
+        pi: pgo.PlotItem,
         *args,
         **kwargs
     ) -> None:
 
         super().__init__(*args, **kwargs)
 
-        self._chart = chart
-
-        chart.sigRangeChanged.connect(self.update_on_resize)
+        self._pi = pi
+        pi.sigRangeChanged.connect(self.update_on_resize)
 
         self._last_datum = (None, None)
 
+        self.x_offset = 0
         # pull text offset from axis from parent axis
         if getattr(self._parent, 'txt_offsets', False):
             self.x_offset, y_offset = self._parent.txt_offsets()
@@ -564,7 +651,8 @@ class YAxisLabel(AxisLabel):
         value: float,  # data for text
 
         # on odd dimension and/or adds nice black line
-        x_offset: Optional[int] = None
+        x_offset: int = 0,
+
     ) -> None:
 
         # this is read inside ``.paint()``
@@ -610,7 +698,7 @@ class YAxisLabel(AxisLabel):
             self._last_datum = (index, value)
 
         self.update_label(
-            self._chart.mapFromView(QPointF(index, value)),
+            self._pi.mapFromView(QPointF(index, value)),
             value
         )
 

piker/ui/_cursor.py

@@ -71,7 +71,7 @@ class LineDot(pg.CurvePoint):
 
         plot: ChartPlotWidget,  # type: ingore # noqa
         pos=None,
-        color: str = 'default_light',
+        color: str = 'bracket',
 
     ) -> None:
         # scale from dpi aware font size
@@ -198,12 +198,11 @@ class ContentsLabel(pg.LabelItem):
         self,
 
         name: str,
-        index: int,
+        ix: int,
         array: np.ndarray,
 
     ) -> None:
         # this being "html" is the dumbest shit :eyeroll:
-        first = array[0]['index']
 
         self.setText(
             "<b>i</b>:{index}<br/>"
@@ -216,7 +215,7 @@ class ContentsLabel(pg.LabelItem):
             "<b>C</b>:{}<br/>"
             "<b>V</b>:{}<br/>"
             "<b>wap</b>:{}".format(
-                *array[index - first][
+                *array[ix][
                     [
                         'time',
                         'open',
@@ -228,7 +227,7 @@ class ContentsLabel(pg.LabelItem):
                     ]
                 ],
                 name=name,
-                index=index,
+                index=ix,
             )
         )
 
@@ -236,15 +235,12 @@ class ContentsLabel(pg.LabelItem):
         self,
 
         name: str,
-        index: int,
+        ix: int,
         array: np.ndarray,
 
     ) -> None:
-
-        first = array[0]['index']
-        if index < array[-1]['index'] and index > first:
-            data = array[index - first][name]
-            self.setText(f"{name}: {data:.2f}")
+        data = array[ix][name]
+        self.setText(f"{name}: {data:.2f}")
 
 
 class ContentsLabels:
@@ -269,17 +265,20 @@ class ContentsLabels:
 
     def update_labels(
         self,
-        index: int,
+        x_in: int,
 
     ) -> None:
         for chart, name, label, update in self._labels:
 
-            flow = chart._flows[name]
-            array = flow.shm.array
+            viz = chart.get_viz(name)
+            array = viz.shm.array
+            index = array[viz.index_field]
+            start = index[0]
+            stop = index[-1]
 
             if not (
-                index >= 0
-                and index < array[-1]['index']
+                x_in >= start
+                and x_in <= stop
             ):
                 # out of range
                 print('WTF out of range?')
@@ -288,7 +287,10 @@ class ContentsLabels:
             # call provided update func with data point
             try:
                 label.show()
-                update(index, array)
+                ix = np.searchsorted(index, x_in)
+                if ix > len(array):
+                    breakpoint()
+                update(ix, array)
 
             except IndexError:
                 log.exception(f"Failed to update label: {name}")
@@ -349,7 +351,7 @@ class Cursor(pg.GraphicsObject):
         # XXX: not sure why these are instance variables?
         # It's not like we can change them on the fly..?
         self.pen = pg.mkPen(
-            color=hcolor('default'),
+            color=hcolor('bracket'),
             style=QtCore.Qt.DashLine,
         )
         self.lines_pen = pg.mkPen(
@@ -365,7 +367,7 @@ class Cursor(pg.GraphicsObject):
         self._lw = self.pixelWidth() * self.lines_pen.width()
 
         # xhair label's color name
-        self.label_color: str = 'default'
+        self.label_color: str = 'bracket'
 
         self._y_label_update: bool = True
 
@@ -418,7 +420,7 @@ class Cursor(pg.GraphicsObject):
         hl.hide()
 
         yl = YAxisLabel(
-            chart=plot,
+            pi=plot.plotItem,
             # parent=plot.getAxis('right'),
             parent=plot.pi_overlay.get_axis(plot.plotItem, 'right'),
             digits=digits or self.digits,
@@ -482,25 +484,32 @@ class Cursor(pg.GraphicsObject):
 
     def add_curve_cursor(
         self,
-        plot: ChartPlotWidget,  # noqa
+        chart: ChartPlotWidget,  # noqa
         curve: 'PlotCurveItem',  # noqa
 
     ) -> LineDot:
-        # if this plot contains curves add line dot "cursors" to denote
+        # if this chart contains curves add line dot "cursors" to denote
         # the current sample under the mouse
-        main_flow = plot._flows[plot.name]
+        main_viz = chart.get_viz(chart.name)
+
         # read out last index
-        i = main_flow.shm.array[-1]['index']
+        i = main_viz.shm.array[-1]['index']
         cursor = LineDot(
             curve,
             index=i,
-            plot=plot
+            plot=chart
         )
-        plot.addItem(cursor)
-        self.graphics[plot].setdefault('cursors', []).append(cursor)
+        chart.addItem(cursor)
+        self.graphics[chart].setdefault('cursors', []).append(cursor)
         return cursor
 
-    def mouseAction(self, action, plot):  # noqa
+    def mouseAction(
+        self,
+        action: str,
+        plot: ChartPlotWidget,
+
+    ) -> None:  # noqa
+
         log.debug(f"{(action, plot.name)}")
         if action == 'Enter':
             self.active_plot = plot

piker/ui/_curve.py

@@ -28,10 +28,7 @@ from PyQt5.QtWidgets import QGraphicsItem
 from PyQt5.QtCore import (
     Qt,
     QLineF,
-    QSizeF,
     QRectF,
-    # QRect,
-    QPointF,
 )
 from PyQt5.QtGui import (
     QPainter,
@@ -39,10 +36,6 @@ from PyQt5.QtGui import (
 )
 from .._profile import pg_profile_enabled, ms_slower_then
 from ._style import hcolor
-# from ._compression import (
-#     # ohlc_to_m4_line,
-#     ds_m4,
-# )
 from ..log import get_logger
 from .._profile import Profiler
 
@@ -58,7 +51,117 @@ _line_styles: dict[str, int] = {
 }
 
 
-class Curve(pg.GraphicsObject):
+class FlowGraphic(pg.GraphicsObject):
+    '''
+    Base class with minimal interface for `QPainterPath` implemented,
+    real-time updated "data flow" graphics.
+
+    See subtypes below.
+
+    '''
+    # sub-type customization methods
+    declare_paintables: Callable | None = None
+    sub_paint: Callable | None = None
+
+    # XXX-NOTE-XXX: graphics caching B)
+    # see explanation for different caching modes:
+    # https://stackoverflow.com/a/39410081
+    cache_mode: int = QGraphicsItem.DeviceCoordinateCache
+    # XXX: WARNING item caching seems to only be useful
+    # if we don't re-generate the entire QPainterPath every time
+    # don't ever use this - it's a colossal nightmare of artefacts
+    # and is disastrous for performance.
+    # QGraphicsItem.ItemCoordinateCache
+    # TODO: still questions todo with coord-cacheing that we should
+    # probably talk to a core dev about:
+    # - if this makes trasform interactions slower (such as zooming)
+    #   and if so maybe if/when we implement a "history" mode for the
+    #   view we disable this in that mode?
+
+    def __init__(
+        self,
+        *args,
+        name: str | None = None,
+
+        # line styling
+        color: str = 'bracket',
+        last_step_color: str | None = None,
+        fill_color: Optional[str] = None,
+        style: str = 'solid',
+
+        **kwargs
+
+    ) -> None:
+
+        self._name = name
+
+        # primary graphics item used for history
+        self.path: QPainterPath = QPainterPath()
+
+        # additional path that can be optionally used for appends which
+        # tries to avoid triggering an update/redraw of the presumably
+        # larger historical ``.path`` above. the flag to enable
+        # this behaviour is found in `Renderer.render()`.
+        self.fast_path: QPainterPath | None = None
+
+        # TODO: evaluating the path capacity stuff and see
+        # if it really makes much diff pre-allocating it.
+        # self._last_cap: int = 0
+        # cap = path.capacity()
+        # if cap != self._last_cap:
+        #     print(f'NEW CAPACITY: {self._last_cap} -> {cap}')
+        #     self._last_cap = cap
+
+        # all history of curve is drawn in single px thickness
+        self._color: str = color
+        pen = pg.mkPen(hcolor(color), width=1)
+        pen.setStyle(_line_styles[style])
+
+        if 'dash' in style:
+            pen.setDashPattern([8, 3])
+
+        self._pen = pen
+        self._brush = pg.functions.mkBrush(
+            hcolor(fill_color or color)
+        )
+
+        # last segment is drawn in 2px thickness for emphasis
+        if last_step_color:
+            self.last_step_pen = pg.mkPen(
+                hcolor(last_step_color),
+                width=2,
+            )
+        else:
+            self.last_step_pen = pg.mkPen(
+                self._pen,
+                width=2,
+            )
+
+        self._last_line: QLineF = QLineF()
+
+        super().__init__(*args, **kwargs)
+
+        # apply cache mode
+        self.setCacheMode(self.cache_mode)
+
+    def x_uppx(self) -> int:
+
+        px_vecs = self.pixelVectors()[0]
+        if px_vecs:
+            return px_vecs.x()
+        else:
+            return 0
+
+    def x_last(self) -> float | None:
+        '''
+        Return the last most x value of the last line segment or if not
+        drawn yet, ``None``.
+
+        '''
+        return self._last_line.x1() if self._last_line else None
+
+
+class Curve(FlowGraphic):
     '''
     A faster, simpler, append friendly version of
     ``pyqtgraph.PlotCurveItem`` built for highly customizable real-time
@@ -75,7 +178,7 @@ class Curve(pg.GraphicsObject):
     lower level graphics data can be rendered in different threads and
     then read and drawn in this main thread without having to worry
     about dealing with Qt's concurrency primitives. See
-    ``piker.ui._flows.Renderer`` for details and logic related to lower
+    ``piker.ui._render.Renderer`` for details and logic related to lower
     level path generation and incremental update. The main differences in
     the path generation code include:
 
@@ -87,127 +190,38 @@ class Curve(pg.GraphicsObject):
       updates don't trigger a full path redraw.
 
     '''
-
-    # sub-type customization methods
-    sub_br: Optional[Callable] = None
-    sub_paint: Optional[Callable] = None
-    declare_paintables: Optional[Callable] = None
+    # TODO: can we remove this?
+    # sub_br: Optional[Callable] = None
 
     def __init__(
         self,
         *args,
 
-        step_mode: bool = False,
-        color: str = 'default_lightest',
-        fill_color: Optional[str] = None,
-        style: str = 'solid',
-        name: Optional[str] = None,
-        use_fpath: bool = True,
+        # color: str = 'default_lightest',
+        # fill_color: Optional[str] = None,
+        # style: str = 'solid',
 
         **kwargs
 
     ) -> None:
 
-        self._name = name
-
         # brutaaalll, see comments within..
         self.yData = None
         self.xData = None
 
-        # self._last_cap: int = 0
-        self.path: Optional[QPainterPath] = None
-
-        # additional path used for appends which tries to avoid
-        # triggering an update/redraw of the presumably larger
-        # historical ``.path`` above.
-        self.use_fpath = use_fpath
-        self.fast_path: Optional[QPainterPath] = None
-
         # TODO: we can probably just dispense with the parent since
         # we're basically only using the pen setting now...
         super().__init__(*args, **kwargs)
 
-        # all history of curve is drawn in single px thickness
-        pen = pg.mkPen(hcolor(color))
-        pen.setStyle(_line_styles[style])
-
-        if 'dash' in style:
-            pen.setDashPattern([8, 3])
-
-        self._pen = pen
-
-        # last segment is drawn in 2px thickness for emphasis
-        # self.last_step_pen = pg.mkPen(hcolor(color), width=2)
-        self.last_step_pen = pg.mkPen(pen, width=2)
-
-        # self._last_line: Optional[QLineF] = None
-        self._last_line = QLineF()
-        self._last_w: float = 1
-
-        # flat-top style histogram-like discrete curve
-        # self._step_mode: bool = step_mode
+        self._last_line: QLineF = QLineF()
 
         # self._fill = True
-        self._brush = pg.functions.mkBrush(hcolor(fill_color or color))
-
-        # NOTE: this setting seems to mostly prevent redraws on mouse
-        # interaction which is a huge boon for avg interaction latency.
-
-        # TODO: one question still remaining is if this makes trasform
-        # interactions slower (such as zooming) and if so maybe if/when
-        # we implement a "history" mode for the view we disable this in
-        # that mode?
-        # don't enable caching by default for the case where the
-        # only thing drawn is the "last" line segment which can
-        # have a weird artifact where it won't be fully drawn to its
-        # endpoint (something we saw on trade rate curves)
-        self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
-
-        # XXX: see explanation for different caching modes:
-        # https://stackoverflow.com/a/39410081
-        # seems to only be useful if we don't re-generate the entire
-        # QPainterPath every time
-        # curve.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
-
-        # don't ever use this - it's a colossal nightmare of artefacts
-        # and is disastrous for performance.
-        # curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache)
 
         # allow sub-type customization
         declare = self.declare_paintables
         if declare:
             declare()
 
-    # TODO: probably stick this in a new parent
-    # type which will contain our own version of
-    # what ``PlotCurveItem`` had in terms of base
-    # functionality? A `FlowGraphic` maybe?
-    def x_uppx(self) -> int:
-
-        px_vecs = self.pixelVectors()[0]
-        if px_vecs:
-            xs_in_px = px_vecs.x()
-            return round(xs_in_px)
-        else:
-            return 0
-
-    def px_width(self) -> float:
-
-        vb = self.getViewBox()
-        if not vb:
-            return 0
-
-        vr = self.viewRect()
-        l, r = int(vr.left()), int(vr.right())
-
-        start, stop = self._xrange
-        lbar = max(l, start)
-        rbar = min(r, stop)
-
-        return vb.mapViewToDevice(
-            QLineF(lbar, 0, rbar, 0)
-        ).length()
-
     # XXX: lol brutal, the internals of `CurvePoint` (inherited by
     # our `LineDot`) required ``.getData()`` to work..
     def getData(self):
@@ -231,8 +245,8 @@ class Curve(pg.GraphicsObject):
             self.path.clear()
 
             if self.fast_path:
-                # self.fast_path.clear()
-                self.fast_path = None
+                self.fast_path.clear()
+                # self.fast_path = None
 
     @cm
     def reset_cache(self) -> None:
@@ -252,77 +266,65 @@ class Curve(pg.GraphicsObject):
             self.boundingRect = self._path_br
             return self._path_br()
 
+    # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect
     def _path_br(self):
         '''
         Post init ``.boundingRect()```.
 
         '''
-        # hb = self.path.boundingRect()
-        hb = self.path.controlPointRect()
-        hb_size = hb.size()
-
-        fp = self.fast_path
-        if fp:
-            fhb = fp.controlPointRect()
-            hb_size = fhb.size() + hb_size
-
-        # print(f'hb_size: {hb_size}')
-
-        # if self._last_step_rect:
-        #     hb_size += self._last_step_rect.size()
-
-        # if self._line:
-        #     br = self._last_step_rect.bottomRight()
-
-        # tl = QPointF(
-        #     # self._vr[0],
-        #     # hb.topLeft().y(),
-        #     # 0,
-        #     # hb_size.height() + 1
+        # profiler = Profiler(
+        #     msg=f'Curve.boundingRect(): `{self._name}`',
+        #     disabled=not pg_profile_enabled(),
+        #     ms_threshold=ms_slower_then,
         # )
-
-        #     br = self._last_step_rect.bottomRight()
-
-        w = hb_size.width()
-        h = hb_size.height()
-
-        sbr = self.sub_br
-        if sbr:
-            w, h = self.sub_br(w, h)
-        else:
-            # assume plain line graphic and use
-            # default unit step in each direction.
-
-            # only on a plane line do we include
-            # and extra index step's worth of width
-            # since in the step case the end of the curve
-            # actually terminates earlier so we don't need
-            # this for the last step.
-            w += self._last_w
-            # ll = self._last_line
-            h += 1  # ll.y2() - ll.y1()
-
-        # br = QPointF(
-        #     self._vr[-1],
-        #     # tl.x() + w,
-        #     tl.y() + h,
-        # )
-
-        br = QRectF(
-
-            # top left
-            # hb.topLeft()
-            # tl,
-            QPointF(hb.topLeft()),
-
-            # br,
-            # total size
-            # QSizeF(hb_size)
-            # hb_size,
-            QSizeF(w, h)
+        pr = self.path.controlPointRect()
+        hb_tl, hb_br = (
+            pr.topLeft(),
+            pr.bottomRight(),
+        )
+        mn_y = hb_tl.y()
+        mx_y = hb_br.y()
+        most_left = hb_tl.x()
+        most_right = hb_br.x()
+        # profiler('calc path vertices')
+
+        # TODO: if/when we get fast path appends working in the
+        # `Renderer`, then we might need to actually use this..
+        # fp = self.fast_path
+        # if fp:
+        #     fhb = fp.controlPointRect()
+        #     # hb_size = fhb.size() + hb_size
+        #     br = pr.united(fhb)
+
+        # XXX: *was* a way to allow sub-types to extend the
+        # boundingrect calc, but in the one use case for a step curve
+        # doesn't seem like we need it as long as the last line segment
+        # is drawn as it is?
+
+        # sbr = self.sub_br
+        # if sbr:
+        #     # w, h = self.sub_br(w, h)
+        #     sub_br = sbr()
+        #     br = br.united(sub_br)
+
+        # assume plain line graphic and use
+        # default unit step in each direction.
+        ll = self._last_line
+        y1, y2 = ll.y1(), ll.y2()
+        x1, x2 = ll.x1(), ll.x2()
+
+        ymn = min(y1, y2, mn_y)
+        ymx = max(y1, y2, mx_y)
+        most_left = min(x1, x2, most_left)
+        most_right = max(x1, x2, most_right)
+        # profiler('calc last line vertices')
+
+        return QRectF(
+            most_left,
+            ymn,
+            most_right - most_left + 1,
+            ymx,
         )
-        # print(f'bounding rect: {br}')
-        return br
 
     def paint(
         self,
@@ -340,18 +342,14 @@ class Curve(pg.GraphicsObject):
 
         sub_paint = self.sub_paint
         if sub_paint:
-            sub_paint(p, profiler)
+            sub_paint(p)
 
         p.setPen(self.last_step_pen)
         p.drawLine(self._last_line)
-        profiler('.drawLine()')
-        p.setPen(self._pen)
+        profiler('last datum `.drawLine()`')
 
+        p.setPen(self._pen)
         path = self.path
-        # cap = path.capacity()
-        # if cap != self._last_cap:
-        #     print(f'NEW CAPACITY: {self._last_cap} -> {cap}')
-        #     self._last_cap = cap
 
         if path:
             p.drawPath(path)
@@ -374,22 +372,30 @@ class Curve(pg.GraphicsObject):
         self,
         path: QPainterPath,
         src_data: np.ndarray,
-        render_data: np.ndarray,
         reset: bool,
         array_key: str,
+        index_field: str,
 
     ) -> None:
         # default line draw last call
         # with self.reset_cache():
-        x = render_data['index']
-        y = render_data[array_key]
+        x = src_data[index_field]
+        y = src_data[array_key]
+
+        x_last = x[-1]
+        x_2last = x[-2]
 
         # draw the "current" step graphic segment so it
         # lines up with the "middle" of the current
         # (OHLC) sample.
         self._last_line = QLineF(
-            x[-2], y[-2],
-            x[-1], y[-1],
+
+            # NOTE: currently we draw in x-domain
+            # from last datum to current such that
+            # the end of line touches the "beginning"
+            # of the current datum step span.
+            x_2last, y[-2],
+            x_last, y[-1],
         )
 
         return x, y
@@ -401,17 +407,20 @@ class Curve(pg.GraphicsObject):
 # (via it's max / min) even when highly zoomed out.
 class FlattenedOHLC(Curve):
 
+    # avoids strange dragging/smearing artifacts when panning..
+    cache_mode: int = QGraphicsItem.NoCache
+
     def draw_last_datum(
         self,
         path: QPainterPath,
         src_data: np.ndarray,
-        render_data: np.ndarray,
         reset: bool,
         array_key: str,
+        index_field: str,
 
     ) -> None:
         lasts = src_data[-2:]
-        x = lasts['index']
+        x = lasts[index_field]
         y = lasts['close']
 
         # draw the "current" step graphic segment so it
@@ -435,9 +444,9 @@ class StepCurve(Curve):
         self,
         path: QPainterPath,
         src_data: np.ndarray,
-        render_data: np.ndarray,
         reset: bool,
         array_key: str,
+        index_field: str,
 
         w: float = 0.5,
 
@@ -446,40 +455,31 @@ class StepCurve(Curve):
         # TODO: remove this and instead place all step curve
         # updating into pre-path data render callbacks.
         # full input data
-        x = src_data['index']
+        x = src_data[index_field]
         y = src_data[array_key]
 
         x_last = x[-1]
+        x_2last = x[-2]
         y_last = y[-1]
+        step_size = x_last - x_2last
 
         # lol, commenting this makes step curves
         # all "black" for me :eyeroll:..
         self._last_line = QLineF(
-            x_last - w, 0,
-            x_last + w, 0,
+            x_2last, 0,
+            x_last, 0,
         )
         self._last_step_rect = QRectF(
-            x_last - w, 0,
-            x_last + w, y_last,
+            x_last, 0,
+            step_size, y_last,
         )
         return x, y
 
     def sub_paint(
         self,
         p: QPainter,
-        profiler: Profiler,
 
     ) -> None:
         # p.drawLines(*tuple(filter(bool, self._last_step_lines)))
         # p.drawRect(self._last_step_rect)
         p.fillRect(self._last_step_rect, self._brush)
-        profiler('.fillRect()')
-
-    def sub_br(
-        self,
-        path_w: float,
-        path_h: float,
-
-    ) -> (float, float):
-        # passthrough
-        return path_w, path_h

piker/ui/_editors.py

@@ -377,7 +377,7 @@ class SelectRect(QtWidgets.QGraphicsRectItem):
         nbars = ixmx - ixmn + 1
 
         chart = self._chart
-        data = chart._flows[chart.name].shm.array[ixmn:ixmx]
+        data = chart.get_viz(chart.name).shm.array[ixmn:ixmx]
 
         if len(data):
             std = data['close'].std()

piker/ui/_fsp.py

@@ -42,6 +42,8 @@ from ..data._sharedmem import (
     _Token,
     try_read,
 )
+from ..data.feed import Flume
+from ..data._source import Symbol
 from ._chart import (
     ChartPlotWidget,
     LinkedSplits,
@@ -76,15 +78,14 @@ def has_vlm(ohlcv: ShmArray) -> bool:
 
 
 def update_fsp_chart(
-    chart: ChartPlotWidget,
-    flow,
+    viz,
     graphics_name: str,
     array_key: Optional[str],
     **kwargs,
 
 ) -> None:
 
-    shm = flow.shm
+    shm = viz.shm
     if not shm:
         return
 
@@ -99,18 +100,15 @@ def update_fsp_chart(
     # update graphics
     # NOTE: this does a length check internally which allows it
     # staying above the last row check below..
-    chart.update_graphics_from_flow(
-        graphics_name,
-        array_key=array_key or graphics_name,
-        **kwargs,
-    )
+    viz.update_graphics()
 
     # XXX: re: ``array_key``: fsp func names must be unique meaning we
     # can't have duplicates of the underlying data even if multiple
     # sub-charts reference it under different 'named charts'.
 
     # read from last calculated value and update any label
-    last_val_sticky = chart._ysticks.get(graphics_name)
+    last_val_sticky = viz.plot.getAxis(
+        'right')._stickies.get(graphics_name)
     if last_val_sticky:
         last = last_row[array_key]
         last_val_sticky.update_from_data(-1, last)
@@ -211,7 +209,7 @@ async def open_fsp_actor_cluster(
 async def run_fsp_ui(
 
     linkedsplits: LinkedSplits,
-    shm: ShmArray,
+    flume: Flume,
     started: trio.Event,
     target: Fsp,
     conf: dict[str, dict],
@@ -248,9 +246,11 @@ async def run_fsp_ui(
             else:
                 chart = linkedsplits.subplots[overlay_with]
 
+            shm = flume.rt_shm
             chart.draw_curve(
-                name=name,
-                shm=shm,
+                name,
+                shm,
+                flume,
                 overlay=True,
                 color='default_light',
                 array_key=name,
@@ -260,8 +260,9 @@ async def run_fsp_ui(
         else:
             # create a new sub-chart widget for this fsp
             chart = linkedsplits.add_plot(
-                name=name,
-                shm=shm,
+                name,
+                shm,
+                flume,
 
                 array_key=name,
                 sidepane=sidepane,
@@ -281,9 +282,10 @@ async def run_fsp_ui(
         # profiler(f'fsp:{name} chart created')
 
         # first UI update, usually from shm pushed history
+        viz = chart.get_viz(array_key)
         update_fsp_chart(
             chart,
-            chart._flows[array_key],
+            viz,
             name,
             array_key=array_key,
         )
@@ -310,7 +312,7 @@ async def run_fsp_ui(
         #     level_line(chart, 70, orient_v='bottom')
         #     level_line(chart, 80, orient_v='top')
 
-        chart.view._set_yrange()
+        chart.view._set_yrange(viz=viz)
         # done()  # status updates
 
         # profiler(f'fsp:{func_name} starting update loop')
@@ -351,6 +353,9 @@ async def run_fsp_ui(
         #         last = time.time()
 
 
+# TODO: maybe this should be our ``Viz`` type since it maps
+# one flume to the next? The machinery for task/actor mgmt should
+# be part of the instantiation API?
 class FspAdmin:
     '''
     Client API for orchestrating FSP actors and displaying
@@ -362,7 +367,7 @@ class FspAdmin:
         tn: trio.Nursery,
         cluster: dict[str, tractor.Portal],
         linked: LinkedSplits,
-        src_shm: ShmArray,
+        flume: Flume,
 
     ) -> None:
         self.tn = tn
@@ -374,7 +379,11 @@ class FspAdmin:
             tuple[tractor.MsgStream, ShmArray]
         ] = {}
         self._flow_registry: dict[_Token, str] = {}
-        self.src_shm = src_shm
+
+        # TODO: make this a `.src_flume` and add
+        # a `dst_flume`?
+        # (=> but then wouldn't this be the most basic `Viz`?)
+        self.flume = flume
 
     def rr_next_portal(self) -> tractor.Portal:
         name, portal = next(self._rr_next_actor)
@@ -387,7 +396,7 @@ class FspAdmin:
         complete: trio.Event,
         started: trio.Event,
         fqsn: str,
-        dst_shm: ShmArray,
+        dst_fsp_flume: Flume,
         conf: dict,
         target: Fsp,
         loglevel: str,
@@ -408,9 +417,10 @@ class FspAdmin:
                 # data feed key
                 fqsn=fqsn,
 
+                # TODO: pass `Flume.to_msg()`s here?
                 # mems
-                src_shm_token=self.src_shm.token,
-                dst_shm_token=dst_shm.token,
+                src_shm_token=self.flume.rt_shm.token,
+                dst_shm_token=dst_fsp_flume.rt_shm.token,
 
                 # target
                 ns_path=ns_path,
@@ -427,12 +437,14 @@ class FspAdmin:
             ctx.open_stream() as stream,
         ):
 
+            dst_fsp_flume.stream: tractor.MsgStream = stream
+
             # register output data
             self._registry[
                 (fqsn, ns_path)
             ] = (
                 stream,
-                dst_shm,
+                dst_fsp_flume.rt_shm,
                 complete
             )
 
@@ -467,9 +479,9 @@ class FspAdmin:
         worker_name: Optional[str] = None,
         loglevel: str = 'info',
 
-    ) -> (ShmArray, trio.Event):
+    ) -> (Flume, trio.Event):
 
-        fqsn = self.linked.symbol.front_fqsn()
+        fqsn = self.flume.symbol.fqsn
 
         # allocate an output shm array
         key, dst_shm, opened = maybe_mk_fsp_shm(
@@ -477,8 +489,28 @@ class FspAdmin:
             target=target,
             readonly=True,
         )
+
+        portal = self.cluster.get(worker_name) or self.rr_next_portal()
+        provider_tag = portal.channel.uid
+
+        symbol = Symbol(
+            key=key,
+            broker_info={
+                provider_tag: {'asset_type': 'fsp'},
+            },
+        )
+        dst_fsp_flume = Flume(
+            symbol=symbol,
+            _rt_shm_token=dst_shm.token,
+            first_quote={},
+
+            # set to 0 presuming for now that we can't load
+            # FSP history (though we should eventually).
+            izero_hist=0,
+            izero_rt=0,
+        )
         self._flow_registry[(
-            self.src_shm._token,
+            self.flume.rt_shm._token,
             target.name
         )] = dst_shm._token
 
@@ -487,7 +519,6 @@ class FspAdmin:
         #         f'Already started FSP `{fqsn}:{func_name}`'
         #     )
 
-        portal = self.cluster.get(worker_name) or self.rr_next_portal()
         complete = trio.Event()
         started = trio.Event()
         self.tn.start_soon(
@@ -496,13 +527,13 @@ class FspAdmin:
             complete,
             started,
             fqsn,
-            dst_shm,
+            dst_fsp_flume,
             conf,
             target,
             loglevel,
         )
 
-        return dst_shm, started
+        return dst_fsp_flume, started
 
     async def open_fsp_chart(
         self,
@@ -514,7 +545,7 @@ class FspAdmin:
 
     ) -> (trio.Event, ChartPlotWidget):
 
-        shm, started = await self.start_engine_task(
+        flume, started = await self.start_engine_task(
             target,
             conf,
             loglevel,
@@ -526,7 +557,7 @@ class FspAdmin:
                 run_fsp_ui,
 
                 self.linked,
-                shm,
+                flume,
                 started,
                 target,
 
@@ -540,7 +571,7 @@ class FspAdmin:
 @acm
 async def open_fsp_admin(
     linked: LinkedSplits,
-    src_shm: ShmArray,
+    flume: Flume,
     **kwargs,
 
 ) -> AsyncGenerator[dict, dict[str, tractor.Portal]]:
@@ -561,7 +592,7 @@ async def open_fsp_admin(
             tn,
             cluster_map,
             linked,
-            src_shm,
+            flume,
         )
         try:
             yield admin
@@ -575,8 +606,9 @@ async def open_fsp_admin(
 async def open_vlm_displays(
 
     linked: LinkedSplits,
-    ohlcv: ShmArray,
+    flume: Flume,
     dvlm: bool = True,
+    loglevel: str = 'info',
 
     task_status: TaskStatus[ChartPlotWidget] = trio.TASK_STATUS_IGNORED,
 
@@ -597,6 +629,8 @@ async def open_vlm_displays(
     sig = inspect.signature(flow_rates.func)
     params = sig.parameters
 
+    ohlcv: ShmArray = flume.rt_shm
+
     async with (
         open_fsp_sidepane(
             linked, {
@@ -616,7 +650,7 @@ async def open_vlm_displays(
                 }
             },
         ) as sidepane,
-        open_fsp_admin(linked, ohlcv) as admin,
+        open_fsp_admin(linked, flume) as admin,
     ):
         # TODO: support updates
         # period_field = sidepane.fields['period']
@@ -624,14 +658,21 @@ async def open_vlm_displays(
         #     str(period_param.default)
         # )
 
+        # use slightly less light (then bracket) gray
+        # for volume from "main exchange" and a more "bluey"
+        # gray for "dark" vlm.
+        vlm_color = 'i3'
+        dark_vlm_color = 'charcoal'
+
         # built-in vlm which we plot ASAP since it's
         # usually data provided directly with OHLC history.
         shm = ohlcv
-        ohlc_chart = linked.chart
+        # ohlc_chart = linked.chart
 
-        chart = linked.add_plot(
+        vlm_chart = linked.add_plot(
             name='volume',
             shm=shm,
+            flume=flume,
 
             array_key='volume',
             sidepane=sidepane,
@@ -644,71 +685,45 @@ async def open_vlm_displays(
             # the curve item internals are pretty convoluted.
             style='step',
         )
-        ohlc_chart.view.enable_auto_yrange(
-            src_vb=chart.view,
-        )
-
-        # force 0 to always be in view
-        def multi_maxmin(
-            names: list[str],
-
-        ) -> tuple[float, float]:
-            '''
-            Flows "group" maxmin loop; assumes all named flows
-            are in the same co-domain and thus can be sorted
-            as one set.
-
-            Iterates all the named flows and calls the chart
-            api to find their range values and return.
-
-            TODO: really we should probably have a more built-in API
-            for this?
-
-            '''
-            mx = 0
-            for name in names:
-                ymn, ymx = chart.maxmin(name=name)
-                mx = max(mx, ymx)
-
-            return 0, mx
+        vlm_viz = vlm_chart._vizs['volume']
 
         # TODO: fix the x-axis label issue where if you put
         # the axis on the left it's totally not lined up...
         # show volume units value on LHS (for dinkus)
-        # chart.hideAxis('right')
-        # chart.showAxis('left')
+        # vlm_chart.hideAxis('right')
+        vlm_chart.hideAxis('left')
 
         # send back new chart to caller
-        task_status.started(chart)
+        task_status.started(vlm_chart)
 
         # should **not** be the same sub-chart widget
-        assert chart.name != linked.chart.name
+        assert vlm_chart.name != linked.chart.name
 
         # sticky only on sub-charts atm
-        last_val_sticky = chart._ysticks[chart.name]
+        last_val_sticky = vlm_chart.plotItem.getAxis(
+            'right')._stickies.get(vlm_chart.name)
 
         # read from last calculated value
         value = shm.array['volume'][-1]
 
         last_val_sticky.update_from_data(-1, value)
 
-        vlm_curve = chart.update_graphics_from_flow(
-            'volume',
-            # shm.array,
-        )
+        _, _, vlm_curve = vlm_viz.update_graphics()
 
         # size view to data once at outset
-        chart.view._set_yrange()
+        # vlm_chart.view._set_yrange(
+        #     viz=vlm_viz
+        # )
 
         # add axis title
-        axis = chart.getAxis('right')
+        axis = vlm_chart.getAxis('right')
         axis.set_title(' vlm')
 
         if dvlm:
 
             tasks_ready = []
             # spawn and overlay $ vlm on the same subchart
-            dvlm_shm, started = await admin.start_engine_task(
+            dvlm_flume, started = await admin.start_engine_task(
                 dolla_vlm,
 
                 {  # fsp engine conf
@@ -720,45 +735,32 @@ async def open_vlm_displays(
                         },
                     },
                 },
-                # loglevel,
+                loglevel,
             )
-            tasks_ready.append(started)
-
-            # FIXME: we should error on starting the same fsp right
-            # since it might collide with existing shm.. or wait we
-            # had this before??
-            # dolla_vlm,
-
-            tasks_ready.append(started)
-            # profiler(f'created shm for fsp actor: {display_name}')
-
-            # wait for all engine tasks to startup
-            async with trio.open_nursery() as n:
-                for event in tasks_ready:
-                    n.start_soon(event.wait)
 
             # dolla vlm overlay
             # XXX: the main chart already contains a vlm "units" axis
             # so here we add an overlay wth a y-range in
             # $ liquidity-value units (normally a fiat like USD).
-            dvlm_pi = chart.overlay_plotitem(
+            dvlm_pi = vlm_chart.overlay_plotitem(
                 'dolla_vlm',
                 index=0,  # place axis on inside (nearest to chart)
+
                 axis_title=' $vlm',
-                axis_side='right',
+                axis_side='left',
+
                 axis_kwargs={
                     'typical_max_str': ' 100.0 M ',
                     'formatter': partial(
                         humanize,
                         digits=2,
                     ),
+                    'text_color': vlm_color,
                 },
             )
 
-            dvlm_pi.hideAxis('left')
-            dvlm_pi.hideAxis('bottom')
             # all to be overlayed curve names
-            fields = [
+            dvlm_fields = [
                'dolla_vlm',
                'dark_vlm',
             ]
@@ -771,27 +773,12 @@ async def open_vlm_displays(
                 'dark_trade_rate',
             ]
 
-            group_mxmn = partial(
-                multi_maxmin,
-                # keep both regular and dark vlm in view
-                names=fields,
-                # names=fields + dvlm_rate_fields,
-            )
-
-            # add custom auto range handler
-            dvlm_pi.vb._maxmin = group_mxmn
-
-            # use slightly less light (then bracket) gray
-            # for volume from "main exchange" and a more "bluey"
-            # gray for "dark" vlm.
-            vlm_color = 'i3'
-            dark_vlm_color = 'charcoal'
-
             # add dvlm (step) curves to common view
             def chart_curves(
                 names: list[str],
                 pi: pg.PlotItem,
                 shm: ShmArray,
+                flume: Flume,
                 step_mode: bool = False,
                 style: str = 'solid',
 
@@ -805,9 +792,13 @@ async def open_vlm_displays(
                     else:
                         color = 'bracket'
 
-                    curve, _ = chart.draw_curve(
-                        name=name,
-                        shm=shm,
+                    assert isinstance(shm, ShmArray)
+                    assert isinstance(flume, Flume)
+
+                    viz = vlm_chart.draw_curve(
+                        name,
+                        shm,
+                        flume,
                         array_key=name,
                         overlay=pi,
                         color=color,
@@ -815,38 +806,32 @@ async def open_vlm_displays(
                         style=style,
                         pi=pi,
                     )
+                    assert viz.plot is pi
 
-                    # TODO: we need a better API to do this..
-                    # specially store ref to shm for lookup in display loop
-                    # since only a placeholder of `None` is entered in
-                    # ``.draw_curve()``.
-                    flow = chart._flows[name]
-                    assert flow.plot is pi
-
+            await started.wait()
             chart_curves(
-                fields,
+                dvlm_fields,
                 dvlm_pi,
-                dvlm_shm,
+                dvlm_flume.rt_shm,
+                dvlm_flume,
                 step_mode=True,
             )
 
-            # spawn flow rates fsp **ONLY AFTER** the 'dolla_vlm' fsp is
-            # up since this one depends on it.
-
-            fr_shm, started = await admin.start_engine_task(
+            # NOTE: spawn flow rates fsp **ONLY AFTER** the 'dolla_vlm' fsp is
+            # up since calculating vlm "rates" obvs first requires the
+            # underlying vlm event feed ;)
+            fr_flume, started = await admin.start_engine_task(
                 flow_rates,
                 {  # fsp engine conf
                     'func_name': 'flow_rates',
-                    'zero_on_step': False,
+                    'zero_on_step': True,
                 },
-                # loglevel,
+                loglevel,
             )
-            await started.wait()
-
             # chart_curves(
             #     dvlm_rate_fields,
             #     dvlm_pi,
-            #     fr_shm,
+            #     fr_flume.rt_shm,
             # )
 
             # TODO: is there a way to "sync" the dual axes such that only
@@ -855,24 +840,26 @@ async def open_vlm_displays(
             # displayed and the curves are effectively the same minus
             # liquidity events (well at least on low OHLC periods - 1s).
             vlm_curve.hide()
-            chart.removeItem(vlm_curve)
-            vflow = chart._flows['volume']
-            vflow.render = False
+            vlm_chart.removeItem(vlm_curve)
+            # vlm_chart.plotItem.layout.setMinimumWidth(0)
+            # vlm_chart.removeAxis('left')
+            vlm_viz = vlm_chart._vizs['volume']
+            vlm_viz.render = False
 
             # avoid range sorting on volume once disabled
-            chart.view.disable_auto_yrange()
+            vlm_chart.view.disable_auto_yrange()
 
             # Trade rate overlay
             # XXX: requires an additional overlay for
             # a trades-per-period (time) y-range.
-            tr_pi = chart.overlay_plotitem(
+            tr_pi = vlm_chart.overlay_plotitem(
                 'trade_rates',
 
                 # TODO: dynamically update period (and thus this axis?)
                 # title from user input.
                 axis_title='clears',
-
                 axis_side='left',
+
                 axis_kwargs={
                     'typical_max_str': ' 10.0 M ',
                     'formatter': partial(
@@ -883,18 +870,13 @@ async def open_vlm_displays(
                 },
 
             )
-            # add custom auto range handler
-            tr_pi.vb.maxmin = partial(
-                multi_maxmin,
-                # keep both regular and dark vlm in view
-                names=trade_rate_fields,
-            )
-            tr_pi.hideAxis('bottom')
 
+            await started.wait()
             chart_curves(
                 trade_rate_fields,
                 tr_pi,
-                fr_shm,
+                fr_flume.rt_shm,
+                fr_flume,
                 # step_mode=True,
 
                 # dashed line to represent "individual trades" being
@@ -928,7 +910,7 @@ async def open_vlm_displays(
 async def start_fsp_displays(
 
     linked: LinkedSplits,
-    ohlcv: ShmArray,
+    flume: Flume,
     group_status_key: str,
     loglevel: str,
 
@@ -971,7 +953,10 @@ async def start_fsp_displays(
     async with (
 
         # NOTE: this admin internally opens an actor cluster
-        open_fsp_admin(linked, ohlcv) as admin,
+        open_fsp_admin(
+            linked,
+            flume,
+        ) as admin,
     ):
         statuses = []
         for target, conf in fsp_conf.items():

piker/ui/_interaction.py

@@ -20,8 +20,15 @@ Chart view box primitives
 """
 from __future__ import annotations
 from contextlib import asynccontextmanager
+from math import (
+    isinf,
+)
 import time
-from typing import Optional, Callable
+from typing import (
+    Optional,
+    Callable,
+    TYPE_CHECKING,
+)
 
 import pyqtgraph as pg
 # from pyqtgraph.GraphicsScene import mouseEvents
@@ -35,10 +42,17 @@ import trio
 from ..log import get_logger
 from .._profile import Profiler
 from .._profile import pg_profile_enabled, ms_slower_then
+from ..data.types import Struct
+from ..data._pathops import slice_from_time
 # from ._style import _min_points_to_show
 from ._editors import SelectRect
 from . import _event
 
+if TYPE_CHECKING:
+    from ._chart import ChartPlotWidget
+    from ._dataviz import Viz
+    # from ._overlay import PlotItemOverlay
+
 
 log = get_logger(__name__)
 
@@ -76,7 +90,6 @@ async def handle_viewmode_kb_inputs(
     pressed: set[str] = set()
 
     last = time.time()
-    trigger_mode: str
     action: str
 
     on_next_release: Optional[Callable] = None
@@ -332,6 +345,49 @@ async def handle_viewmode_mouse(
             view.order_mode.submit_order()
 
 
+class OverlayT(Struct):
+    '''
+    An overlay co-domain range transformer.
+
+    Used to translate and apply a range from one y-range
+    to another based on a returns logarithm:
+
+    R(ymn, ymx, yref) = (ymx - yref)/yref
+
+    which gives the log-scale multiplier, and
+
+    ymx_t = yref * (1 + R)
+
+    which gives the inverse to translate to the same value
+    in the target co-domain.
+
+    '''
+    viz: Viz  # viz with largest measured dispersion
+
+    mx: float = 0
+    mn: float = float('inf')
+
+    up_swing: float = 0
+    down_swing: float = 0
+    disp: float = 0
+
+    def loglin_from_range(
+        self,
+
+        y_ref: float,  # reference value for dispersion metric
+        mn: float,  # min y in target log-lin range
+        mx: float,  # max y in target log-lin range
+        offset: float,  # y-offset to start log-scaling from
+
+    ) -> tuple[float, float]:
+        r_up = (mx - y_ref) / y_ref
+        r_down = (mn - y_ref) / y_ref
+        ymn = offset * (1 + r_down)
+        ymx = offset * (1 + r_up)
+
+        return ymn, ymx
+
+
 class ChartView(ViewBox):
     '''
     Price chart view box with interaction behaviors you'd expect from
@@ -366,7 +422,6 @@ class ChartView(ViewBox):
         )
         # for "known y-range style"
         self._static_yrange = static_yrange
-        self._maxmin = None
 
         # disable vertical scrolling
         self.setMouseEnabled(
@@ -375,7 +430,7 @@ class ChartView(ViewBox):
         )
 
         self.linked = None
-        self._chart: 'ChartPlotWidget' = None  # noqa
+        self._chart: ChartPlotWidget | None = None  # noqa
 
         # add our selection box annotator
         self.select_box = SelectRect(self)
@@ -387,6 +442,10 @@ class ChartView(ViewBox):
         self.setFocusPolicy(QtCore.Qt.StrongFocus)
         self._ic = None
 
+        # TODO: probably just assign this whenever a new `PlotItem` is
+        # allocated since they're 1to1 with views..
+        self._viz: Viz | None = None
+
     def start_ic(
         self,
     ) -> None:
@@ -446,29 +505,18 @@ class ChartView(ViewBox):
             yield self
 
     @property
-    def chart(self) -> 'ChartPlotWidget':  # type: ignore # noqa
+    def chart(self) -> ChartPlotWidget:  # type: ignore # noqa
         return self._chart
 
     @chart.setter
-    def chart(self, chart: 'ChartPlotWidget') -> None:  # type: ignore # noqa
+    def chart(self, chart: ChartPlotWidget) -> None:  # type: ignore # noqa
         self._chart = chart
         self.select_box.chart = chart
-        if self._maxmin is None:
-            self._maxmin = chart.maxmin
-
-    @property
-    def maxmin(self) -> Callable:
-        return self._maxmin
-
-    @maxmin.setter
-    def maxmin(self, callback: Callable) -> None:
-        self._maxmin = callback
 
     def wheelEvent(
         self,
         ev,
         axis=None,
-        # relayed_from: ChartView = None,
     ):
         '''
         Override "center-point" location for scrolling.
@@ -483,7 +531,6 @@ class ChartView(ViewBox):
         if (
             not linked
         ):
-            # print(f'{self.name} not linked but relay from {relayed_from.name}')
             return
 
         if axis in (0, 1):
@@ -495,22 +542,23 @@ class ChartView(ViewBox):
         chart = self.linked.chart
 
         # don't zoom more then the min points setting
-        l, lbar, rbar, r = chart.bars_range()
-        # vl = r - l
+        viz = chart.get_viz(chart.name)
+        vl, lbar, rbar, vr = viz.bars_range()
 
-        # if ev.delta() > 0 and vl <= _min_points_to_show:
-        #     log.debug("Max zoom bruh...")
+        # TODO: max/min zoom limits incorporating time step size.
+        # rl = vr - vl
+        # if ev.delta() > 0 and rl <= _min_points_to_show:
+        #     log.warning("Max zoom bruh...")
         #     return
-
         # if (
         #     ev.delta() < 0
-        #     and vl >= len(chart._flows[chart.name].shm.array) + 666
+        #     and rl >= len(chart._vizs[chart.name].shm.array) + 666
         # ):
-        #     log.debug("Min zoom bruh...")
+        #     log.warning("Min zoom bruh...")
         #     return
 
         # actual scaling factor
-        s = 1.015 ** (ev.delta() * -1 / 20)  # self.state['wheelScaleFactor'])
+        s = 1.016 ** (ev.delta() * -1 / 20)  # self.state['wheelScaleFactor'])
         s = [(None if m is False else s) for m in mask]
 
         if (
@@ -536,50 +584,19 @@ class ChartView(ViewBox):
             # scale_y = 1.3 ** (center.y() * -1 / 20)
             self.scaleBy(s, center)
 
+        # zoom in view-box area
         else:
-
-            # center = pg.Point(
-            #     fn.invertQTransform(self.childGroup.transform()).map(ev.pos())
-            # )
-
-            # XXX: scroll "around" the right most element in the view
-            # which stays "pinned" in place.
-
-            # furthest_right_coord = self.boundingRect().topRight()
-
-            # yaxis = pg.Point(
-            #     fn.invertQTransform(
-            #         self.childGroup.transform()
-            #     ).map(furthest_right_coord)
-            # )
-
-            # This seems like the most "intuitive option, a hybrid of
-            # tws and tv styles
-            last_bar = pg.Point(int(rbar)) + 1
-
-            ryaxis = chart.getAxis('right')
-            r_axis_x = ryaxis.pos().x()
-
-            end_of_l1 = pg.Point(
-                round(
-                    chart.cv.mapToView(
-                        pg.Point(r_axis_x - chart._max_l1_line_len)
-                        # QPointF(chart._max_l1_line_len, 0)
-                    ).x()
-                )
-            )  # .x()
-
-            # self.state['viewRange'][0][1] = end_of_l1
-            # focal = pg.Point((last_bar.x() + end_of_l1)/2)
-
+            # use right-most point of current curve graphic
+            xl = viz.graphics.x_last()
             focal = min(
-                last_bar,
-                end_of_l1,
-                key=lambda p: p.x()
+                xl,
+                vr,
             )
-            # focal = pg.Point(last_bar.x() + end_of_l1)
 
             self._resetTarget()
+
+            # NOTE: scroll "around" the right most datum-element in view
+            # gives the feeling of staying "pinned" in place.
             self.scaleBy(s, focal)
 
             # XXX: the order of the next 2 lines i'm pretty sure
@@ -587,7 +604,7 @@ class ChartView(ViewBox):
             # update, but i gotta feelin that because this one is signal
             # based (and thus not necessarily sync invoked right away)
             # that calling the resize method manually might work better.
-            self.sigRangeChangedManually.emit(mask)
+            # self.sigRangeChangedManually.emit(mask)
 
             # XXX: without this is seems as though sometimes
             # when zooming in from far out (and maybe vice versa?)
@@ -597,7 +614,8 @@ class ChartView(ViewBox):
             # that never seems to happen? Only question is how much this
             # "double work" is causing latency when these missing event
             # fires don't happen?
-            self.maybe_downsample_graphics()
+            self.interact_graphics_cycle()
+            self.interact_graphics_cycle()
 
             ev.accept()
 
@@ -605,21 +623,8 @@ class ChartView(ViewBox):
         self,
         ev,
         axis: Optional[int] = None,
-        # relayed_from: ChartView = None,
 
     ) -> None:
-        # if relayed_from:
-        #     print(f'PAN: {self.name} -> RELAYED FROM: {relayed_from.name}')
-
-        # NOTE since in the overlay case axes are already
-        # "linked" any x-range change will already be mirrored
-        # in all overlaid ``PlotItems``, so we need to simply
-        # ignore the signal here since otherwise we get N-calls
-        # from N-overlays resulting in an "accelerated" feeling
-        # panning motion instead of the expect linear shift.
-        # if relayed_from:
-        #     return
-
         pos = ev.pos()
         lastPos = ev.lastPos()
         dif = pos - lastPos
@@ -629,7 +634,10 @@ class ChartView(ViewBox):
         button = ev.button()
 
         # Ignore axes if mouse is disabled
-        mouseEnabled = np.array(self.state['mouseEnabled'], dtype=np.float)
+        mouseEnabled = np.array(
+            self.state['mouseEnabled'],
+            dtype=np.float,
+        )
         mask = mouseEnabled.copy()
         if axis is not None:
             mask[1-axis] = 0.0
@@ -689,9 +697,6 @@ class ChartView(ViewBox):
 
             # PANNING MODE
             else:
-                # XXX: WHY
-                ev.accept()
-
                 try:
                     self.start_ic()
                 except RuntimeError:
@@ -715,7 +720,10 @@ class ChartView(ViewBox):
                 if x is not None or y is not None:
                     self.translateBy(x=x, y=y)
 
-                self.sigRangeChangedManually.emit(self.state['mouseEnabled'])
+                # self.sigRangeChangedManually.emit(mask)
+                    # self.state['mouseEnabled']
+                # )
+                self.interact_graphics_cycle()
 
                 if ev.isFinish():
                     self.signal_ic()
@@ -723,6 +731,9 @@ class ChartView(ViewBox):
                     # self._ic = None
                     # self.chart.resume_all_feeds()
 
+                # # XXX: WHY
+                # ev.accept()
+
         # WEIRD "RIGHT-CLICK CENTER ZOOM" MODE
         elif button & QtCore.Qt.RightButton:
 
@@ -743,10 +754,12 @@ class ChartView(ViewBox):
             center = Point(tr.map(ev.buttonDownPos(QtCore.Qt.RightButton)))
             self._resetTarget()
             self.scaleBy(x=x, y=y, center=center)
-            self.sigRangeChangedManually.emit(self.state['mouseEnabled'])
 
-            # XXX: WHY
-            ev.accept()
+            # self.sigRangeChangedManually.emit(self.state['mouseEnabled'])
+            self.interact_graphics_cycle()
+
+        # XXX: WHY
+        ev.accept()
 
     # def mouseClickEvent(self, event: QtCore.QEvent) -> None:
     #      '''This routine is rerouted to an async handler.
@@ -768,7 +781,12 @@ class ChartView(ViewBox):
         *,
 
         yrange: Optional[tuple[float, float]] = None,
-        range_margin: float = 0.06,
+        viz: Viz | None = None,
+
+        # NOTE: this value pairs (more or less) with L1 label text
+        # height offset from from the bid/ask lines.
+        range_margin: float | None = 0.09,
+
         bars_range: Optional[tuple[int, int, int, int]] = None,
 
         # flag to prevent triggering sibling charts from the same linked
@@ -786,14 +804,13 @@ class ChartView(ViewBox):
 
         '''
         name = self.name
-        # print(f'YRANGE ON {name}')
+        # print(f'YRANGE ON {name} -> yrange{yrange}')
         profiler = Profiler(
             msg=f'`ChartView._set_yrange()`: `{name}`',
             disabled=not pg_profile_enabled(),
             ms_threshold=ms_slower_then,
             delayed=True,
         )
-        set_range = True
         chart = self._chart
 
         # view has been set in 'axis' mode
@@ -802,8 +819,8 @@ class ChartView(ViewBox):
         # - disable autoranging
         # - remove any y range limits
         if chart._static_yrange == 'axis':
-            set_range = False
             self.setLimits(yMin=None, yMax=None)
+            return
 
         # static y-range has been set likely by
         # a specialized FSP configuration.
@@ -816,42 +833,70 @@ class ChartView(ViewBox):
         elif yrange is not None:
             ylow, yhigh = yrange
 
-        if set_range:
+        # XXX: only compute the mxmn range
+        # if none is provided as input!
+        if not yrange:
 
-            # XXX: only compute the mxmn range
-            # if none is provided as input!
-            if not yrange:
-                # flow = chart._flows[name]
-                yrange = self._maxmin()
+            if not viz:
+                breakpoint()
 
-                if yrange is None:
-                    log.warning(f'No yrange provided for {name}!?')
-                    print(f"WTF NO YRANGE {name}")
-                    return
+            out = viz.maxmin()
+            if out is None:
+                log.warning(f'No yrange provided for {name}!?')
+                return
+            (
+                ixrng,
+                _,
+                yrange
+            ) = out
+
+            profiler(f'`{self.name}:Viz.maxmin()` -> {ixrng}=>{yrange}')
+
+            if yrange is None:
+                log.warning(f'No yrange provided for {name}!?')
+                return
 
             ylow, yhigh = yrange
 
-            profiler(f'callback ._maxmin(): {yrange}')
-
-            # view margins: stay within a % of the "true range"
+        # view margins: stay within a % of the "true range"
+        if range_margin is not None:
             diff = yhigh - ylow
-            ylow = ylow - (diff * range_margin)
-            yhigh = yhigh + (diff * range_margin)
-
-            # XXX: this often needs to be unset
-            # to get different view modes to operate
-            # correctly!
-            self.setLimits(
-                yMin=ylow,
-                yMax=yhigh,
+            ylow = max(
+                ylow - (diff * range_margin),
+                0,
+            )
+            yhigh = min(
+                yhigh + (diff * range_margin),
+                yhigh * (1 + range_margin),
             )
-            self.setYRange(ylow, yhigh)
-            profiler(f'set limits: {(ylow, yhigh)}')
 
+        # XXX: this often needs to be unset
+        # to get different view modes to operate
+        # correctly!
+
+        # print(
+        #     f'set limits {self.name}:\n'
+        #     f'ylow: {ylow}\n'
+        #     f'yhigh: {yhigh}\n'
+        # )
+        self.setYRange(
+            ylow,
+            yhigh,
+            padding=0,
+        )
+        self.setLimits(
+            yMin=ylow,
+            yMax=yhigh,
+        )
+        self.update()
+
+        # LOL: yet anothercucking pg buggg..
+        # can't use `msg=f'setYRange({ylow}, {yhigh}')`
         profiler.finish()
 
     def enable_auto_yrange(
         self,
+        viz: Viz,
         src_vb: Optional[ChartView] = None,
 
     ) -> None:
@@ -863,9 +908,6 @@ class ChartView(ViewBox):
         if src_vb is None:
             src_vb = self
 
-        # widget-UIs/splitter(s) resizing
-        src_vb.sigResized.connect(self._set_yrange)
-
         # re-sampling trigger:
         # TODO: a smarter way to avoid calling this needlessly?
         # 2 things i can think of:
@@ -873,37 +915,21 @@ class ChartView(ViewBox):
         #   iterate those.
         # - only register this when certain downsample-able graphics are
         #   "added to scene".
-        src_vb.sigRangeChangedManually.connect(
-            self.maybe_downsample_graphics
-        )
-        # mouse wheel doesn't emit XRangeChanged
-        src_vb.sigRangeChangedManually.connect(self._set_yrange)
+        # src_vb.sigRangeChangedManually.connect(
+        #     self.interact_graphics_cycle
+        # )
 
-        # XXX: enabling these will cause "jittery"-ness
-        # on zoom where sharp diffs in the y-range will
-        # not re-size right away until a new sample update?
-        # if src_vb is not self:
-        #     src_vb.sigXRangeChanged.connect(self._set_yrange)
-        #     src_vb.sigXRangeChanged.connect(
-        #         self.maybe_downsample_graphics
-        #     )
+        # widget-UIs/splitter(s) resizing
+        src_vb.sigResized.connect(
+            self.interact_graphics_cycle
+        )
 
     def disable_auto_yrange(self) -> None:
 
+        # XXX: not entirely sure why we can't de-reg this..
         self.sigResized.disconnect(
-            self._set_yrange,
+            self.interact_graphics_cycle
         )
-        self.sigRangeChangedManually.disconnect(
-            self.maybe_downsample_graphics
-        )
-        self.sigRangeChangedManually.disconnect(
-            self._set_yrange,
-        )
-
-        # self.sigXRangeChanged.disconnect(self._set_yrange)
-        # self.sigXRangeChanged.disconnect(
-        #     self.maybe_downsample_graphics
-        # )
 
     def x_uppx(self) -> float:
         '''
@@ -912,7 +938,7 @@ class ChartView(ViewBox):
         graphics items which are our children.
 
         '''
-        graphics = [f.graphics for f in self._chart._flows.values()]
+        graphics = [f.graphics for f in self._chart._vizs.values()]
         if not graphics:
             return 0
 
@@ -923,59 +949,439 @@ class ChartView(ViewBox):
         else:
             return 0
 
-    def maybe_downsample_graphics(
+    def interact_graphics_cycle(
         self,
-        autoscale_overlays: bool = True,
+        *args,  # capture signal-handler related shit
+
+        debug_print: bool = False,
+        do_overlay_scaling: bool = True,
+        do_linked_charts: bool = True,
     ):
         profiler = Profiler(
-            msg=f'ChartView.maybe_downsample_graphics() for {self.name}',
+            msg=f'ChartView.interact_graphics_cycle() for {self.name}',
             disabled=not pg_profile_enabled(),
+            ms_threshold=ms_slower_then,
 
             # XXX: important to avoid not seeing underlying
-            # ``.update_graphics_from_flow()`` nested profiling likely
+            # ``Viz.update_graphics()`` nested profiling likely
             # due to the way delaying works and garbage collection of
             # the profiler in the delegated method calls.
-            ms_threshold=6,
-            # ms_threshold=ms_slower_then,
+            delayed=True,
+
+            # for hardcore latency checking, comment these flags above.
+            # disabled=False,
+            # ms_threshold=4,
         )
 
-        # TODO: a faster single-loop-iterator way of doing this XD
         chart = self._chart
-        plots = {chart.name: chart}
-
         linked = self.linked
-        if linked:
+        if (
+            do_linked_charts
+            and linked
+        ):
+            plots = {linked.chart.name: linked.chart}
             plots |= linked.subplots
+        else:
+            plots = {chart.name: chart}
 
+        # TODO: a faster single-loop-iterator way of doing this?
         for chart_name, chart in plots.items():
-            for name, flow in chart._flows.items():
 
-                if (
-                    not flow.render
+            # Common `PlotItem` maxmin table; presumes that some path
+            # graphics (and thus their backing data sets) are in the
+            # same co-domain and view box (since the were added
+            # a separate graphics objects to a common plot) and thus can
+            # be sorted as one set per plot.
+            mxmns_by_common_pi: dict[
+                pg.PlotItem,
+                tuple[float, float],
+            ] = {}
 
-                    # XXX: super important to be aware of this.
-                    # or not flow.graphics.isVisible()
-                ):
+            # proportional group auto-scaling per overlay set.
+            # -> loop through overlays on each multi-chart widget
+            #    and scale all y-ranges based on autoscale config.
+            # -> for any "group" overlay we want to dispersion normalize
+            #    and scale minor charts onto the major chart: the chart
+            #    with the most dispersion in the set.
+            major_viz: Viz = None
+            major_mx: float = 0
+            major_mn: float = float('inf')
+            # mx_up_rng: float = 0
+            # mn_down_rng: float = 0
+            mx_disp: float = 0
+
+            # collect certain flows have grapics objects **in seperate
+            # plots/viewboxes** into groups and do a common calc to
+            # determine auto-ranging input for `._set_yrange()`.
+            # this is primarly used for our so called "log-linearized
+            # multi-plot" overlay technique.
+            start_datums: dict[
+                ViewBox,
+                tuple[
+                    Viz,
+                    float,  # y start
+                    float,  # y min
+                    float,  # y max
+                    float,  # y median
+                    slice,  # in-view array slice
+                    np.ndarray,  # in-view array
+                ],
+            ] = {}
+            major_in_view: np.ndarray = None
+
+            for name, viz in chart._vizs.items():
+
+                if not viz.render:
                     # print(f'skipping {flow.name}')
                     continue
 
                 # pass in no array which will read and render from the last
                 # passed array (normally provided by the display loop.)
-                chart.update_graphics_from_flow(
-                    name,
-                    use_vr=True,
+                in_view, i_read_range, _ = viz.update_graphics()
+
+                if not in_view:
+                    continue
+
+                profiler(f'{viz.name}@{chart_name} `Viz.update_graphics()`')
+
+                out = viz.maxmin(i_read_range=i_read_range)
+                if out is None:
+                    log.warning(f'No yrange provided for {name}!?')
+                    return
+                (
+                    ixrng,
+                    read_slc,
+                    yrange
+                ) = out
+                profiler(f'{viz.name}@{chart_name} `Viz.maxmin()`')
+
+                pi = viz.plot
+
+                # handle multiple graphics-objs per viewbox cases
+                mxmn = mxmns_by_common_pi.get(pi)
+                if mxmn:
+                    yrange = mxmns_by_common_pi[pi] = (
+                        min(yrange[0], mxmn[0]),
+                        max(yrange[1], mxmn[1]),
+                    )
+
+                else:
+                    mxmns_by_common_pi[pi] = yrange
+
+                profiler(f'{viz.name}@{chart_name} common pi sort')
+
+                # handle overlay log-linearized group scaling cases
+                # TODO: a better predicate here, likely something
+                # to do with overlays and their settings..
+                if (
+                    viz.is_ohlc
+                ):
+                    ymn, ymx = yrange
+                    # print(f'adding {viz.name} to overlay')
+                    # mxmn_groups[viz.name] = out
+                    # viz = chart._vizs[viz_name]
+
+                    # determine start datum in view
+                    arr = viz.shm.array
+                    in_view = arr[read_slc]
+                    row_start = arr[read_slc.start - 1]
+
+                    # y_med = (ymx - ymn) / 2
+                    # y_med = viz.median_from_range(
+                    #     read_slc.start,
+                    #     read_slc.stop,
+                    # )
+                    if viz.is_ohlc:
+                        y_start = row_start['open']
+                    else:
+                        y_start = row_start[viz.name]
+
+                    profiler(f'{viz.name}@{chart_name} MINOR curve median')
+
+                    start_datums[viz.plot.vb] = (
+                        viz,
+                        y_start,
+                        ymn,
+                        ymx,
+                        # y_med,
+                        read_slc,
+                        in_view,
+                    )
+
+                    # find curve with max dispersion
+                    disp = abs(ymx - ymn) / y_start
+
+                    # track the "major" curve as the curve with most
+                    # dispersion.
+                    if disp > mx_disp:
+                        major_viz = viz
+                        mx_disp = disp
+                        major_mn = ymn
+                        major_mx = ymx
+                        major_in_view = in_view
+                        profiler(f'{viz.name}@{chart_name} set new major')
+
+                    # compute directional (up/down) y-range % swing/dispersion
+                    # y_ref = y_med
+                    # up_rng = (ymx - y_ref) / y_ref
+                    # down_rng = (ymn - y_ref) / y_ref
+
+                    # mx_up_rng = max(mx_up_rng, up_rng)
+                    # mn_down_rng = min(mn_down_rng, down_rng)
+
+                    # print(
+                    #     f'{viz.name}@{chart_name} group mxmn calc\n'
+                    #     '--------------------\n'
+                    #     f'y_start: {y_start}\n'
+                    #     f'ymn: {ymn}\n'
+                    #     f'ymx: {ymx}\n'
+                    #     f'mx_disp: {mx_disp}\n'
+                    #     f'up %: {up_rng * 100}\n'
+                    #     f'down %: {down_rng * 100}\n'
+                    #     f'mx up %: {mx_up_rng * 100}\n'
+                    #     f'mn down %: {mn_down_rng * 100}\n'
+                    # )
+                    profiler(f'{viz.name}@{chart_name} MINOR curve scale')
+
+                # non-overlay group case
+                else:
+                    pi.vb._set_yrange(yrange=yrange)
+                    profiler(
+                        f'{viz.name}@{chart_name} simple std `._set_yrange()`'
+                    )
+
+            profiler(f'<{chart_name}>.interact_graphics_cycle({name})')
+            if not start_datums:
+                return
+
+            # if no overlays, set lone chart's yrange and short circuit
+            if (
+                len(start_datums) < 2
+                or not do_overlay_scaling
+            ):
+                if not major_viz:
+                    major_viz = viz
+
+                # print(f'ONLY ranging major: {viz.name}')
+                major_viz.plot.vb._set_yrange(
+                    yrange=yrange,
                 )
+                profiler(f'{viz.name}@{chart_name} single curve yrange')
+                return
 
-                # for each overlay on this chart auto-scale the
-                # y-range to max-min values.
-                if autoscale_overlays:
-                    overlay = chart.pi_overlay
-                    if overlay:
-                        for pi in overlay.overlays:
-                            pi.vb._set_yrange(
-                                # TODO: get the range once up front...
-                                # bars_range=br,
+            # conduct "log-linearized multi-plot" scalings for all groups
+            for (
+                view,
+                (
+                    viz,
+                    y_start,
+                    y_min,
+                    y_max,
+                    # y_med,
+                    read_slc,
+                    minor_in_view,
+                )
+            ) in start_datums.items():
+
+                # we use the ymn/mx verbatim from the major curve
+                # (i.e. the curve measured to have the highest
+                # dispersion in view).
+                if viz is major_viz:
+                    ymn = y_min
+                    ymx = y_max
+                    continue
+
+                else:
+                    key = 'open' if viz.is_ohlc else viz.name
+
+                    # handle case where major and minor curve(s) have
+                    # a disjoint x-domain (one curve is smaller in
+                    # length then the other):
+                    # - find the highest (time) index common to both
+                    #   curves.
+                    # - slice out the first "intersecting" y-value from
+                    #   both curves for use in log-linear scaling such
+                    #   that the intersecting y-value is used as the
+                    #   reference point for scaling minor curve's
+                    #   y-range based on the major curves y-range.
+
+                    # get intersection point y-values for both curves
+                    minor_in_view_start = minor_in_view[0]
+                    minor_i_start = minor_in_view_start['index']
+                    minor_i_start_t = minor_in_view_start['time']
+
+                    major_in_view_start = major_in_view[0]
+                    major_i_start = major_in_view_start['index']
+                    major_i_start_t = major_in_view_start['time']
+
+                    y_major_intersect = major_in_view_start[key]
+                    y_minor_intersect = minor_in_view_start[key]
+
+                    profiler(f'{viz.name}@{chart_name} intersect detection')
+
+                    tdiff = (major_i_start_t - minor_i_start_t)
+                    if debug_print:
+                        print(
+                            f'{major_viz.name} time diff with minor:\n'
+                            f'maj:{major_i_start_t}\n'
+                            '-\n'
+                            f'min:{minor_i_start_t}\n'
+                            f'=> {tdiff}\n'
+                        )
+
+                    # major has later timestamp adjust minor
+                    if tdiff > 0:
+                        slc = slice_from_time(
+                            arr=minor_in_view,
+                            start_t=major_i_start_t,
+                            stop_t=major_i_start_t,
+                        )
+                        y_minor_intersect = minor_in_view[slc.start][key]
+                        profiler(f'{viz.name}@{chart_name} intersect by t')
+
+                    # minor has later timestamp adjust major
+                    elif tdiff < 0:
+                        slc = slice_from_time(
+                            arr=major_in_view,
+                            start_t=minor_i_start_t,
+                            stop_t=minor_i_start_t,
+                        )
+                        y_major_intersect = major_in_view[slc.start][key]
+
+                        profiler(f'{viz.name}@{chart_name} intersect by t')
+
+                    if debug_print:
+                        print(
+                            f'major_i_start: {major_i_start}\n'
+                            f'major_i_start_t: {major_i_start_t}\n'
+                            f'minor_i_start: {minor_i_start}\n'
+                            f'minor_i_start_t: {minor_i_start_t}\n'
+                        )
+
+                    # TODO: probably write this as a compile cpython or
+                    # numba func.
+
+                    # compute directional (up/down) y-range
+                    # % swing/dispersion starting at the reference index
+                    # determined by the above indexing arithmetic.
+                    y_ref = y_major_intersect
+                    if not y_ref:
+                        log.warning(
+                            f'BAD y_major_intersect?!: {y_major_intersect}'
+                        )
+                        # breakpoint()
+
+                    r_up = (major_mx - y_ref) / y_ref
+                    r_down = (major_mn - y_ref) / y_ref
+
+                    minor_y_start = y_minor_intersect
+                    ymn = minor_y_start * (1 + r_down)
+                    ymx = minor_y_start * (1 + r_up)
+
+                    profiler(f'{viz.name}@{chart_name} SCALE minor')
+
+                    # XXX: handle out of view cases where minor curve
+                    # now is outside the range of the major curve. in
+                    # this case we then re-scale the major curve to
+                    # include the range missing now enforced by the
+                    # minor (now new major for this *side*). Note this
+                    # is side (up/down) specific.
+                    new_maj_mxmn: None | tuple[float, float] = None
+                    if y_max > ymx:
+
+                        y_ref = y_minor_intersect
+                        r_up_minor = (y_max - y_ref) / y_ref
+
+                        y_maj_ref = y_major_intersect
+                        new_maj_ymx = y_maj_ref * (1 + r_up_minor)
+                        new_maj_mxmn = (major_mn, new_maj_ymx)
+                        if debug_print:
+                            print(
+                                f'{view.name} OUT OF RANGE:\n'
+                                '--------------------\n'
+                                f'y_max:{y_max} > ymx:{ymx}\n'
                             )
-                    profiler('autoscaled linked plots')
+                        ymx = y_max
+                        profiler(f'{viz.name}@{chart_name} re-SCALE major UP')
 
-                profiler(f'<{chart_name}>.update_graphics_from_flow({name})')
+                    if y_min < ymn:
+
+                        y_ref = y_minor_intersect
+                        r_down_minor = (y_min - y_ref) / y_ref
+
+                        y_maj_ref = y_major_intersect
+                        new_maj_ymn = y_maj_ref * (1 + r_down_minor)
+                        new_maj_mxmn = (
+                            new_maj_ymn,
+                            new_maj_mxmn[1] if new_maj_mxmn else major_mx
+                        )
+                        if debug_print:
+                            print(
+                                f'{view.name} OUT OF RANGE:\n'
+                                '--------------------\n'
+                                f'y_min:{y_min} < ymn:{ymn}\n'
+                            )
+                        ymn = y_min
+
+                        profiler(
+                            f'{viz.name}@{chart_name} re-SCALE major DOWN'
+                        )
+
+                    if new_maj_mxmn:
+                        if debug_print:
+                            print(
+                                f'RESCALE MAJOR {major_viz.name}:\n'
+                                f'previous: {(major_mn, major_mx)}\n'
+                                f'new: {new_maj_mxmn}\n'
+                            )
+                        major_mn, major_mx = new_maj_mxmn
+
+                    if debug_print:
+                        print(
+                            f'{view.name} APPLY group mxmn\n'
+                            '--------------------\n'
+                            f'y_minor_intersect: {y_minor_intersect}\n'
+                            f'y_major_intersect: {y_major_intersect}\n'
+                            # f'mn_down_rng: {mn_down_rng * 100}\n'
+                            # f'mx_up_rng: {mx_up_rng * 100}\n'
+                            f'scaled ymn: {ymn}\n'
+                            f'scaled ymx: {ymx}\n'
+                            f'scaled mx_disp: {mx_disp}\n'
+                        )
+
+                if (
+                    isinf(ymx)
+                    or isinf(ymn)
+                ):
+                    log.warning(
+                        f'BAD ymx/ymn: {(ymn, ymx)}'
+                    )
+                    continue
+
+                view._set_yrange(
+                    yrange=(ymn, ymx),
+                )
+                profiler(f'{viz.name}@{chart_name} log-SCALE minor')
+
+            # NOTE XXX: we have to set the major curve's range once (and
+            # only once) here since we're doing this entire routine
+            # inside of a single render cycle (and apparently calling
+            # `ViewBox.setYRange()` multiple times within one only takes
+            # the first call as serious...) XD
+            if debug_print:
+                print(
+                    f'Scale MAJOR {major_viz.name}:\n'
+                    f'scaled mx_disp: {mx_disp}\n'
+                    f'previous: {(major_mn, major_mx)}\n'
+                    f'new: {new_maj_mxmn}\n'
+                )
+            major_viz.plot.vb._set_yrange(
+                yrange=(major_mn, major_mx),
+            )
+            profiler(f'{viz.name}@{chart_name} log-SCALE major')
+            # major_mx, major_mn = new_maj_mxmn
+            # vrs = major_viz.plot.vb.viewRange()
+            # if vrs[1][0] > major_mn:
+            #     breakpoint()
+
+        profiler.finish()

piker/ui/_l1.py

@@ -26,22 +26,24 @@ from PyQt5.QtCore import QPointF
 
 from ._axes import YAxisLabel
 from ._style import hcolor
+from ._pg_overrides import PlotItem
 
 
 class LevelLabel(YAxisLabel):
-    """Y-axis (vertically) oriented, horizontal label that sticks to
+    '''
+    Y-axis (vertically) oriented, horizontal label that sticks to
     where it's placed despite chart resizing and supports displaying
     multiple fields.
 
 
     TODO: replace the rectangle-text part with our new ``Label`` type.
 
-    """
-    _x_margin = 0
-    _y_margin = 0
+    '''
+    _x_br_offset: float = -16
+    _y_txt_h_scaling: float = 2
 
     # adjustment "further away from" anchor point
-    _x_offset = 9
+    _x_offset = 0
     _y_offset = 0
 
     # fields to be displayed in the label string
@@ -57,12 +59,12 @@ class LevelLabel(YAxisLabel):
         chart,
         parent,
 
-        color: str = 'bracket',
+        color: str = 'default_light',
 
         orient_v: str = 'bottom',
-        orient_h: str = 'left',
+        orient_h: str = 'right',
 
-        opacity: float = 0,
+        opacity: float = 1,
 
         # makes order line labels offset from their parent axis
         # such that they don't collide with the L1/L2 lines/prices
@@ -98,13 +100,15 @@ class LevelLabel(YAxisLabel):
 
         self._h_shift = {
             'left': -1.,
-            'right': 0.
+            'right': 0.,
         }[orient_h]
 
         self.fields = self._fields.copy()
         # ensure default format fields are in correct
         self.set_fmt_str(self._fmt_str, self.fields)
 
+        self.setZValue(10)
+
     @property
     def color(self):
         return self._hcolor
@@ -112,7 +116,10 @@ class LevelLabel(YAxisLabel):
     @color.setter
     def color(self, color: str) -> None:
         self._hcolor = color
-        self._pen = self.pen = pg.mkPen(hcolor(color))
+        self._pen = self.pen = pg.mkPen(
+            hcolor(color),
+            width=3,
+        )
 
     def update_on_resize(self, vr, r):
         """Tiis is a ``.sigRangeChanged()`` handler.
@@ -124,15 +131,16 @@ class LevelLabel(YAxisLabel):
         self,
         fields: dict = None,
     ) -> None:
-        """Update the label's text contents **and** position from
+        '''
+        Update the label's text contents **and** position from
         a view box coordinate datum.
 
-        """
+        '''
         self.fields.update(fields)
         level = self.fields['level']
 
         # map "level" to local coords
-        abs_xy = self._chart.mapFromView(QPointF(0, level))
+        abs_xy = self._pi.mapFromView(QPointF(0, level))
 
         self.update_label(
             abs_xy,
@@ -149,7 +157,7 @@ class LevelLabel(YAxisLabel):
         h, w = self.set_label_str(fields)
 
         if self._adjust_to_l1:
-            self._x_offset = self._chart._max_l1_line_len
+            self._x_offset = self._pi.chart_widget._max_l1_line_len
 
         self.setPos(QPointF(
             self._h_shift * (w + self._x_offset),
@@ -174,7 +182,8 @@ class LevelLabel(YAxisLabel):
         fields: dict,
     ):
         # use space as e3 delim
-        self.label_str = self._fmt_str.format(**fields).replace(',', ' ')
+        self.label_str = self._fmt_str.format(
+            **fields).replace(',', ' ')
 
         br = self.boundingRect()
         h, w = br.height(), br.width()
@@ -187,14 +196,14 @@ class LevelLabel(YAxisLabel):
         self,
         p: QtGui.QPainter,
         rect: QtCore.QRectF
-    ) -> None:
-        p.setPen(self._pen)
 
+    ) -> None:
+
+        p.setPen(self._pen)
         rect = self.rect
 
         if self._orient_v == 'bottom':
             lp, rp = rect.topLeft(), rect.topRight()
-            # p.drawLine(rect.topLeft(), rect.topRight())
 
         elif self._orient_v == 'top':
             lp, rp = rect.bottomLeft(), rect.bottomRight()
@@ -208,6 +217,11 @@ class LevelLabel(YAxisLabel):
             ])
         )
 
+        p.fillRect(
+            self.rect,
+            self.bg_color,
+        )
+
     def highlight(self, pen) -> None:
         self._pen = pen
         self.update()
@@ -236,43 +250,46 @@ class L1Label(LevelLabel):
         # Set a global "max L1 label length" so we can
         # look it up on order lines and adjust their
         # labels not to overlap with it.
-        chart = self._chart
+        chart = self._pi.chart_widget
         chart._max_l1_line_len: float = max(
             chart._max_l1_line_len,
-            w
+            w,
         )
 
         return h, w
 
 
 class L1Labels:
-    """Level 1 bid ask labels for dynamic update on price-axis.
+    '''
+    Level 1 bid ask labels for dynamic update on price-axis.
 
-    """
+    '''
     def __init__(
         self,
-        chart: 'ChartPlotWidget',  # noqa
+        plotitem: PlotItem,
         digits: int = 2,
         size_digits: int = 3,
         font_size: str = 'small',
     ) -> None:
 
-        self.chart = chart
+        chart = self.chart = plotitem.chart_widget
 
-        raxis = chart.getAxis('right')
+        raxis = plotitem.getAxis('right')
         kwargs = {
-            'chart': chart,
+            'chart': plotitem,
             'parent': raxis,
 
-            'opacity': 1,
+            'opacity': .9,
             'font_size': font_size,
-            'fg_color': chart.pen_color,
-            'bg_color': chart.view_color,
+            'fg_color': 'default_light',
+            'bg_color': chart.view_color,  # normally 'papas_special'
         }
 
+        # TODO: add humanized source-asset
+        # info format.
         fmt_str = (
-            ' {size:.{size_digits}f} x '
-            '{level:,.{level_digits}f} '
+            ' {size:.{size_digits}f} u'
+            # '{level:,.{level_digits}f} '
         )
         fields = {
             'level': 0,
@@ -285,12 +302,17 @@ class L1Labels:
             orient_v='bottom',
             **kwargs,
         )
-        bid.set_fmt_str(fmt_str=fmt_str, fields=fields)
+        bid.set_fmt_str(
+            fmt_str='\n' + fmt_str,
+            fields=fields,
+        )
         bid.show()
 
         ask = self.ask_label = L1Label(
             orient_v='top',
             **kwargs,
         )
-        ask.set_fmt_str(fmt_str=fmt_str, fields=fields)
+        ask.set_fmt_str(
+            fmt_str=fmt_str,
+            fields=fields)
         ask.show()

piker/ui/_label.py

@@ -233,6 +233,36 @@ class Label:
     def delete(self) -> None:
         self.vb.scene().removeItem(self.txt)
 
+    # NOTE: pulled out from ``ChartPlotWidget`` from way way old code.
+    # def _label_h(self, yhigh: float, ylow: float) -> float:
+    #     # compute contents label "height" in view terms
+    #     # to avoid having data "contents" overlap with them
+    #     if self._labels:
+    #         label = self._labels[self.name][0]
+
+    #         rect = label.itemRect()
+    #         tl, br = rect.topLeft(), rect.bottomRight()
+    #         vb = self.plotItem.vb
+
+    #         try:
+    #             # on startup labels might not yet be rendered
+    #             top, bottom = (vb.mapToView(tl).y(), vb.mapToView(br).y())
+
+    #             # XXX: magic hack, how do we compute exactly?
+    #             label_h = (top - bottom) * 0.42
+
+    #         except np.linalg.LinAlgError:
+    #             label_h = 0
+    #     else:
+    #         label_h = 0
+
+    #     # print(f'label height {self.name}: {label_h}')
+
+    #     if label_h > yhigh - ylow:
+    #         label_h = 0
+
+    #     print(f"bounds (ylow, yhigh): {(ylow, yhigh)}")
+
 
 class FormatLabel(QLabel):
     '''

piker/ui/_ohlc.py

@@ -18,25 +18,24 @@ Super fast OHLC sampling graphics types.
 
 """
 from __future__ import annotations
-from typing import (
-    Optional,
-    TYPE_CHECKING,
-)
 
 import numpy as np
-import pyqtgraph as pg
-from PyQt5 import QtCore, QtGui, QtWidgets
-from PyQt5.QtCore import QLineF, QPointF
+from PyQt5 import (
+    QtGui,
+    QtWidgets,
+)
+from PyQt5.QtCore import (
+    QLineF,
+    QRectF,
+)
+from PyQt5.QtWidgets import QGraphicsItem
 from PyQt5.QtGui import QPainterPath
 
+from ._curve import FlowGraphic
 from .._profile import pg_profile_enabled, ms_slower_then
-from ._style import hcolor
 from ..log import get_logger
 from .._profile import Profiler
 
-if TYPE_CHECKING:
-    from ._chart import LinkedSplits
-
 
 log = get_logger(__name__)
 
@@ -44,7 +43,8 @@ log = get_logger(__name__)
 def bar_from_ohlc_row(
     row: np.ndarray,
     # 0.5 is no overlap between arms, 1.0 is full overlap
-    w: float = 0.43
+    bar_w: float,
+    bar_gap: float = 0.16
 
 ) -> tuple[QLineF]:
     '''
@@ -52,8 +52,7 @@ def bar_from_ohlc_row(
     OHLC "bar" for use in the "last datum" of a series.
 
     '''
-    open, high, low, close, index = row[
-        ['open', 'high', 'low', 'close', 'index']]
+    open, high, low, close, index = row
 
     # TODO: maybe consider using `QGraphicsLineItem` ??
     # gives us a ``.boundingRect()`` on the objects which may make
@@ -61,9 +60,11 @@ def bar_from_ohlc_row(
     # history path faster since it's done in C++:
     # https://doc.qt.io/qt-5/qgraphicslineitem.html
 
+    mid: float = (bar_w / 2) + index
+
     # high -> low vertical (body) line
     if low != high:
-        hl = QLineF(index, low, index, high)
+        hl = QLineF(mid, low, mid, high)
     else:
         # XXX: if we don't do it renders a weird rectangle?
         # see below for filtering this later...
@@ -74,48 +75,55 @@ def bar_from_ohlc_row(
     # the index's range according to the view mapping coordinates.
 
     # open line
-    o = QLineF(index - w, open, index, open)
+    o = QLineF(index + bar_gap, open, mid, open)
 
     # close line
-    c = QLineF(index, close, index + w, close)
+    c = QLineF(
+        mid, close,
+        index + bar_w - bar_gap, close,
+    )
 
     return [hl, o, c]
 
 
-class BarItems(pg.GraphicsObject):
+class BarItems(FlowGraphic):
     '''
     "Price range" bars graphics rendered from a OHLC sampled sequence.
 
     '''
+    # XXX: causes this weird jitter bug when click-drag panning
+    # where the path curve will awkwardly flicker back and forth?
+    cache_mode: int = QGraphicsItem.NoCache
+
     def __init__(
         self,
-        linked: LinkedSplits,
-        plotitem: 'pg.PlotItem',  # noqa
-        pen_color: str = 'bracket',
-        last_bar_color: str = 'bracket',
-
-        name: Optional[str] = None,
+        *args,
+        **kwargs,
 
     ) -> None:
-        super().__init__()
-        self.linked = linked
-        # XXX: for the mega-lulz increasing width here increases draw
-        # latency...  so probably don't do it until we figure that out.
-        self._color = pen_color
-        self.bars_pen = pg.mkPen(hcolor(pen_color), width=1)
-        self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2)
-        self._name = name
 
-        self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
-        self.path = QPainterPath()
-        self._last_bar_lines: Optional[tuple[QLineF, ...]] = None
+        super().__init__(*args, **kwargs)
+        self._last_bar_lines: tuple[QLineF, ...] | None = None
 
-    def x_uppx(self) -> int:
-        # we expect the downsample curve report this.
-        return 0
+    def x_last(self) -> None | float:
+        '''
+        Return the last most x value of the close line segment
+        or if not drawn yet, ``None``.
 
+        '''
+        if self._last_bar_lines:
+            close_arm_line = self._last_bar_lines[-1]
+            return close_arm_line.x2() if close_arm_line else None
+        else:
+            return None
+
+    # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect
     def boundingRect(self):
-        # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect
+        # profiler = Profiler(
+        #     msg=f'BarItems.boundingRect(): `{self._name}`',
+        #     disabled=not pg_profile_enabled(),
+        #     ms_threshold=ms_slower_then,
+        # )
 
         # TODO: Can we do rect caching to make this faster
         # like `pg.PlotCurveItem` does? In theory it's just
@@ -135,32 +143,37 @@ class BarItems(pg.GraphicsObject):
             hb.topLeft(),
             hb.bottomRight(),
         )
+        mn_y = hb_tl.y()
+        mx_y = hb_br.y()
+        most_left = hb_tl.x()
+        most_right = hb_br.x()
+        # profiler('calc path vertices')
 
         # need to include last bar height or BR will be off
-        mx_y = hb_br.y()
-        mn_y = hb_tl.y()
-
-        last_lines = self._last_bar_lines
+        # OHLC line segments: [hl, o, c]
+        last_lines: tuple[QLineF] | None = self._last_bar_lines
         if last_lines:
-            body_line = self._last_bar_lines[0]
-            if body_line:
-                mx_y = max(mx_y, max(body_line.y1(), body_line.y2()))
-                mn_y = min(mn_y, min(body_line.y1(), body_line.y2()))
+            (
+                hl,
+                o,
+                c,
+            ) = last_lines
+            most_right = c.x2() + 1
+            ymx = ymn = c.y2()
 
-        return QtCore.QRectF(
-
-            # top left
-            QPointF(
-                hb_tl.x(),
-                mn_y,
-            ),
-
-            # bottom right
-            QPointF(
-                hb_br.x() + 1,
-                mx_y,
-            )
+            if hl:
+                y1, y2 = hl.y1(), hl.y2()
+                ymn = min(y1, y2)
+                ymx = max(y1, y2)
+                mx_y = max(ymx, mx_y)
+                mn_y = min(ymn, mn_y)
+                # profiler('calc last bar vertices')
 
+        return QRectF(
+            most_left,
+            mn_y,
+            most_right - most_left + 1,
+            mx_y - mn_y,
         )
 
     def paint(
@@ -184,12 +197,12 @@ class BarItems(pg.GraphicsObject):
         # as is necesarry for what's in "view". Not sure if this will
         # lead to any perf gains other then when zoomed in to less bars
         # in view.
-        p.setPen(self.last_bar_pen)
+        p.setPen(self.last_step_pen)
         if self._last_bar_lines:
             p.drawLines(*tuple(filter(bool, self._last_bar_lines)))
             profiler('draw last bar')
 
-        p.setPen(self.bars_pen)
+        p.setPen(self._pen)
         p.drawPath(self.path)
         profiler(f'draw history path: {self.path.capacity()}')
 
@@ -197,29 +210,40 @@ class BarItems(pg.GraphicsObject):
         self,
         path: QPainterPath,
         src_data: np.ndarray,
-        render_data: np.ndarray,
         reset: bool,
         array_key: str,
-
-        fields: list[str] = [
-            'index',
-            'open',
-            'high',
-            'low',
-            'close',
-        ],
+        index_field: str,
 
     ) -> None:
 
         # relevant fields
+        fields: list[str] = [
+            'open',
+            'high',
+            'low',
+            'close',
+            index_field,
+        ]
         ohlc = src_data[fields]
-        last_row = ohlc[-1:]
+        # last_row = ohlc[-1:]
 
         # individual values
-        last_row = i, o, h, l, last = ohlc[-1]
+        last_row = o, h, l, last, i = ohlc[-1]
+
+        # times = src_data['time']
+        # if times[-1] - times[-2]:
+        #     breakpoint()
+
+        index = src_data[index_field]
+        step_size = index[-1] - index[-2]
 
         # generate new lines objects for updatable "current bar"
-        self._last_bar_lines = bar_from_ohlc_row(last_row)
+        bg: float = 0.16 * step_size
+        self._last_bar_lines = bar_from_ohlc_row(
+            last_row,
+            bar_w=step_size,
+            bar_gap=bg,
+        )
 
         # assert i == graphics.start_index - 1
         # assert i == last_index
@@ -234,10 +258,16 @@ class BarItems(pg.GraphicsObject):
         if l != h:  # noqa
 
             if body is None:
-                body = self._last_bar_lines[0] = QLineF(i, l, i, h)
+                body = self._last_bar_lines[0] = QLineF(
+                    i + bg, l,
+                    i + step_size - bg, h,
+                )
             else:
                 # update body
-                body.setLine(i, l, i, h)
+                body.setLine(
+                    body.x1(), l,
+                    body.x2(), h,
+                )
 
             # XXX: pretty sure this is causing an issue where the
             # bar has a large upward move right before the next
@@ -248,4 +278,4 @@ class BarItems(pg.GraphicsObject):
             # date / from some previous sample. It's weird though
             # because i've seen it do this to bars i - 3 back?
 
-        return ohlc['index'], ohlc['close']
+        return ohlc[index_field], ohlc['close']

piker/ui/_overlay.py

@@ -22,7 +22,6 @@ from collections import defaultdict
 from functools import partial
 from typing import (
     Callable,
-    Optional,
 )
 
 from pyqtgraph.graphicsItems.AxisItem import AxisItem
@@ -92,11 +91,11 @@ class ComposedGridLayout:
     '''
     def __init__(
         self,
-        item: PlotItem,
+        pi: PlotItem,
 
     ) -> None:
 
-        self.items: list[PlotItem] = []
+        self.pitems: list[PlotItem] = []
         self._pi2axes: dict[  # TODO: use a ``bidict`` here?
             int,
             dict[str, AxisItem],
@@ -116,6 +115,7 @@ class ComposedGridLayout:
 
             layout.setContentsMargins(0, 0, 0, 0)
             layout.setSpacing(0)
+            layout.setMinimumWidth(0)
 
             if name in ('top', 'bottom'):
                 orient = Qt.Vertical
@@ -125,7 +125,11 @@ class ComposedGridLayout:
 
             layout.setOrientation(orient)
 
-        self.insert_plotitem(0, item)
+        self.insert_plotitem(
+            0,
+            pi,
+            remove_axes=False,
+        )
 
         # insert surrounding linear layouts into the parent pi's layout
         # such that additional axes can be appended arbitrarily without
@@ -135,13 +139,16 @@ class ComposedGridLayout:
             # TODO: do we need this?
             # axis should have been removed during insert above
             index = _axes_layout_indices[name]
-            axis = item.layout.itemAt(*index)
+            axis = pi.layout.itemAt(*index)
             if axis and axis.isVisible():
                 assert linlayout.itemAt(0) is axis
 
-            # item.layout.removeItem(axis)
-            item.layout.addItem(linlayout, *index)
-            layout = item.layout.itemAt(*index)
+            # XXX: see comment in ``.insert_plotitem()``...
+            # our `PlotItem.removeAxis()` does this internally.
+            # pi.layout.removeItem(axis)
+
+            pi.layout.addItem(linlayout, *index)
+            layout = pi.layout.itemAt(*index)
             assert layout is linlayout
 
     def _register_item(
@@ -157,14 +164,16 @@ class ComposedGridLayout:
             self._pi2axes.setdefault(name, {})[index] = axis
 
         # enter plot into list for index tracking
-        self.items.insert(index, plotitem)
+        self.pitems.insert(index, plotitem)
 
     def insert_plotitem(
         self,
         index: int,
         plotitem: PlotItem,
 
-    ) -> (int, int):
+        remove_axes: bool = False,
+
+    ) -> tuple[int, list[AxisItem]]:
         '''
         Place item at index by inserting all axes into the grid
         at list-order appropriate position.
@@ -175,11 +184,14 @@ class ComposedGridLayout:
                 '`.insert_plotitem()` only supports an index >= 0'
             )
 
+        inserted_axes: list[AxisItem] = []
+
         # add plot's axes in sequence to the embedded linear layouts
         # for each "side" thus avoiding graphics collisions.
         for name, axis_info in plotitem.axes.copy().items():
             linlayout, axes = self.sides[name]
             axis = axis_info['item']
+            inserted_axes.append(axis)
 
             if axis in axes:
                 # TODO: re-order using ``.pop()`` ?
@@ -192,22 +204,16 @@ class ComposedGridLayout:
             if (
                 not axis.isVisible()
 
-                # XXX: we never skip moving the axes for the *first*
+                # XXX: we never skip moving the axes for the *root*
                 # plotitem inserted (even if not shown) since we need to
                 # move all the hidden axes into linear sub-layouts for
                 # that "central" plot in the overlay. Also if we don't
                 # do it there's weird geomoetry calc offsets that make
                 # view coords slightly off somehow .. smh
-                and not len(self.items) == 0
+                and not len(self.pitems) == 0
             ):
                 continue
 
-            # XXX: Remove old axis? No, turns out we don't need this?
-            # DON'T unlink it since we the original ``ViewBox``
-            # to still drive it B)
-            # popped = plotitem.removeAxis(name, unlink=False)
-            # assert axis is popped
-
             # invert insert index for layouts which are
             # not-left-to-right, top-to-bottom insert oriented
             insert_index = index
@@ -220,7 +226,17 @@ class ComposedGridLayout:
 
         self._register_item(index, plotitem)
 
-        return index
+        if remove_axes:
+            for name, axis_info in plotitem.axes.copy().items():
+                axis = axis_info['item']
+                # XXX: Remove old axis?
+                # No, turns out we don't need this?
+                # DON'T UNLINK IT since we need the original ``ViewBox`` to
+                # still drive it with events/handlers B)
+                popped = plotitem.removeAxis(name, unlink=False)
+                assert axis is popped
+
+        return (index, inserted_axes)
 
     def append_plotitem(
         self,
@@ -234,20 +250,20 @@ class ComposedGridLayout:
         '''
         # for left and bottom axes we have to first remove
         # items and re-insert to maintain a list-order.
-        return self.insert_plotitem(len(self.items), item)
+        return self.insert_plotitem(len(self.pitems), item)
 
     def get_axis(
         self,
         plot: PlotItem,
         name: str,
 
-    ) -> Optional[AxisItem]:
+    ) -> AxisItem | None:
         '''
         Retrieve the named axis for overlayed ``plot`` or ``None``
         if axis for that name is not shown.
 
         '''
-        index = self.items.index(plot)
+        index = self.pitems.index(plot)
         named = self._pi2axes[name]
         return named.get(index)
 
@@ -306,14 +322,17 @@ class PlotItemOverlay:
         # events/signals.
         root_plotitem.vb.setZValue(10)
 
-        self.overlays: list[PlotItem] = []
         self.layout = ComposedGridLayout(root_plotitem)
         self._relays: dict[str, Signal] = {}
 
+    @property
+    def overlays(self) -> list[PlotItem]:
+        return self.layout.pitems
+
     def add_plotitem(
         self,
         plotitem: PlotItem,
-        index: Optional[int] = None,
+        index: int | None = None,
 
         # event/signal names which will be broadcasted to all added
         # (relayee) ``PlotItem``s (eg. ``ViewBox.mouseDragEvent``).
@@ -324,11 +343,9 @@ class PlotItemOverlay:
         # (0, 1),  # link both
         link_axes: tuple[int] = (),
 
-    ) -> None:
+    ) -> tuple[int, list[AxisItem]]:
 
-        index = index or len(self.overlays)
         root = self.root_plotitem
-        self.overlays.insert(index, plotitem)
         vb: ViewBox = plotitem.vb
 
         # TODO: some sane way to allow menu event broadcast XD
@@ -361,8 +378,8 @@ class PlotItemOverlay:
                 if not sub_handlers:
 
                     src_handler = getattr(
-                        root.vb,
-                        ev_name,
+                       root.vb,
+                       ev_name,
                     )
 
                     def broadcast(
@@ -370,7 +387,7 @@ class PlotItemOverlay:
 
                         # TODO: drop this viewbox specific input and
                         # allow a predicate to be passed in by user.
-                        axis: 'Optional[int]' = None,
+                        axis: int | None = None,
 
                         *,
 
@@ -476,7 +493,10 @@ class PlotItemOverlay:
         # ``PlotItem`` dynamically.
 
         # append-compose into the layout all axes from this plot
-        self.layout.insert_plotitem(index, plotitem)
+        if index is None:
+            insert_index, axes = self.layout.append_plotitem(plotitem)
+        else:
+            insert_index, axes = self.layout.insert_plotitem(index, plotitem)
 
         plotitem.setGeometry(root.vb.sceneBoundingRect())
 
@@ -496,6 +516,11 @@ class PlotItemOverlay:
 
         vb.setZValue(100)
 
+        return (
+            index,
+            axes,
+        )
+
     def get_axis(
         self,
         plot: PlotItem,

piker/ui/_pathops.py

@@ -1,241 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-"""
-Super fast ``QPainterPath`` generation related operator routines.
-
-"""
-from __future__ import annotations
-from typing import (
-    # Optional,
-    TYPE_CHECKING,
-)
-
-import numpy as np
-from numpy.lib import recfunctions as rfn
-from numba import njit, float64, int64  # , optional
-# import pyqtgraph as pg
-from PyQt5 import QtGui
-# from PyQt5.QtCore import QLineF, QPointF
-
-from ..data._sharedmem import (
-    ShmArray,
-)
-# from .._profile import pg_profile_enabled, ms_slower_then
-from ._compression import (
-    ds_m4,
-)
-
-if TYPE_CHECKING:
-    from ._flows import Renderer
-
-
-def xy_downsample(
-    x,
-    y,
-    uppx,
-
-    x_spacer: float = 0.5,
-
-) -> tuple[
-    np.ndarray,
-    np.ndarray,
-    float,
-    float,
-]:
-
-    # downsample whenever more then 1 pixels per datum can be shown.
-    # always refresh data bounds until we get diffing
-    # working properly, see above..
-    bins, x, y, ymn, ymx = ds_m4(
-        x,
-        y,
-        uppx,
-    )
-
-    # flatten output to 1d arrays suitable for path-graphics generation.
-    x = np.broadcast_to(x[:, None], y.shape)
-    x = (x + np.array(
-        [-x_spacer, 0, 0, x_spacer]
-    )).flatten()
-    y = y.flatten()
-
-    return x, y, ymn, ymx
-
-
-@njit(
-    # TODO: for now need to construct this manually for readonly arrays, see
-    # https://github.com/numba/numba/issues/4511
-    # ntypes.tuple((float64[:], float64[:], float64[:]))(
-    #     numba_ohlc_dtype[::1],  # contiguous
-    #     int64,
-    #     optional(float64),
-    # ),
-    nogil=True
-)
-def path_arrays_from_ohlc(
-    data: np.ndarray,
-    start: int64,
-    bar_gap: float64 = 0.43,
-
-) -> np.ndarray:
-    '''
-    Generate an array of lines objects from input ohlc data.
-
-    '''
-    size = int(data.shape[0] * 6)
-
-    x = np.zeros(
-        # data,
-        shape=size,
-        dtype=float64,
-    )
-    y, c = x.copy(), x.copy()
-
-    # TODO: report bug for assert @
-    # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
-    for i, q in enumerate(data[start:], start):
-
-        # TODO: ask numba why this doesn't work..
-        # open, high, low, close, index = q[
-        #     ['open', 'high', 'low', 'close', 'index']]
-
-        open = q['open']
-        high = q['high']
-        low = q['low']
-        close = q['close']
-        index = float64(q['index'])
-
-        istart = i * 6
-        istop = istart + 6
-
-        # x,y detail the 6 points which connect all vertexes of a ohlc bar
-        x[istart:istop] = (
-            index - bar_gap,
-            index,
-            index,
-            index,
-            index,
-            index + bar_gap,
-        )
-        y[istart:istop] = (
-            open,
-            open,
-            low,
-            high,
-            close,
-            close,
-        )
-
-        # specifies that the first edge is never connected to the
-        # prior bars last edge thus providing a small "gap"/"space"
-        # between bars determined by ``bar_gap``.
-        c[istart:istop] = (1, 1, 1, 1, 1, 0)
-
-    return x, y, c
-
-
-def gen_ohlc_qpath(
-    r: Renderer,
-    data: np.ndarray,
-    array_key: str,  # we ignore this
-    vr: tuple[int, int],
-
-    start: int = 0,  # XXX: do we need this?
-    # 0.5 is no overlap between arms, 1.0 is full overlap
-    w: float = 0.43,
-
-) -> QtGui.QPainterPath:
-    '''
-    More or less direct proxy to ``path_arrays_from_ohlc()``
-    but with closed in kwargs for line spacing.
-
-    '''
-    x, y, c = path_arrays_from_ohlc(
-        data,
-        start,
-        bar_gap=w,
-    )
-    return x, y, c
-
-
-def ohlc_to_line(
-    ohlc_shm: ShmArray,
-    data_field: str,
-    fields: list[str] = ['open', 'high', 'low', 'close']
-
-) -> tuple[
-    np.ndarray,
-    np.ndarray,
-]:
-    '''
-    Convert an input struct-array holding OHLC samples into a pair of
-    flattened x, y arrays with the same size (datums wise) as the source
-    data.
-
-    '''
-    y_out = ohlc_shm.ustruct(fields)
-    first = ohlc_shm._first.value
-    last = ohlc_shm._last.value
-
-    # write pushed data to flattened copy
-    y_out[first:last] = rfn.structured_to_unstructured(
-        ohlc_shm.array[fields]
-    )
-
-    # generate an flat-interpolated x-domain
-    x_out = (
-        np.broadcast_to(
-            ohlc_shm._array['index'][:, None],
-            (
-                ohlc_shm._array.size,
-                # 4,  # only ohlc
-                y_out.shape[1],
-            ),
-        ) + np.array([-0.5, 0, 0, 0.5])
-    )
-    assert y_out.any()
-
-    return (
-        x_out,
-        y_out,
-    )
-
-
-def to_step_format(
-    shm: ShmArray,
-    data_field: str,
-    index_field: str = 'index',
-
-) -> tuple[int, np.ndarray, np.ndarray]:
-    '''
-    Convert an input 1d shm array to a "step array" format
-    for use by path graphics generation.
-
-    '''
-    i = shm._array['index'].copy()
-    out = shm._array[data_field].copy()
-
-    x_out = np.broadcast_to(
-        i[:, None],
-        (i.size, 2),
-    ) + np.array([-0.5, 0.5])
-
-    y_out = np.empty((len(out), 2), dtype=out.dtype)
-    y_out[:] = out[:, np.newaxis]
-
-    # start y at origin level
-    y_out[0, 0] = 0
-    return x_out, y_out

piker/ui/_pg_overrides.py

@@ -26,6 +26,8 @@ from typing import Optional
 
 import pyqtgraph as pg
 
+from ._axes import Axis
+
 
 def invertQTransform(tr):
     """Return a QTransform that is the inverse of *tr*.
@@ -52,6 +54,10 @@ def _do_overrides() -> None:
     pg.functions.invertQTransform = invertQTransform
     pg.PlotItem = PlotItem
 
+    # enable "QPainterPathPrivate for faster arrayToQPath" from
+    # https://github.com/pyqtgraph/pyqtgraph/pull/2324
+    pg.setConfigOption('enableExperimental', True)
+
 
 # NOTE: the below customized type contains all our changes on a method
 # by method basis as per the diff:
@@ -62,6 +68,20 @@ class PlotItem(pg.PlotItem):
     Overrides for the core plot object mostly pertaining to overlayed
     multi-view management as it relates to multi-axis managment.
 
+    This object is the combination of a ``ViewBox`` and multiple
+    ``AxisItem``s and so far we've added additional functionality and
+    APIs for:
+    - removal of axes
+
+    ---
+
+    From ``pyqtgraph`` super type docs:
+    - Manage placement of ViewBox, AxisItems, and LabelItems
+    - Create and manage a list of PlotDataItems displayed inside the
+      ViewBox
+    - Implement a context menu with commonly used display and analysis
+      options
+
     '''
     def __init__(
         self,
@@ -71,7 +91,7 @@ class PlotItem(pg.PlotItem):
         title=None,
         viewBox=None,
         axisItems=None,
-        default_axes=['left', 'bottom'],
+        default_axes=['right', 'bottom'],
         enableMenu=True,
         **kargs
     ):
@@ -86,6 +106,8 @@ class PlotItem(pg.PlotItem):
             enableMenu=enableMenu,
             kargs=kargs,
         )
+        self.name = name
+        self.chart_widget = None
         # self.setAxisItems(
         #     axisItems,
         #     default_axes=default_axes,
@@ -108,7 +130,7 @@ class PlotItem(pg.PlotItem):
 
         If the ``unlink: bool`` is set to ``False`` then the axis will
         stay linked to its view and will only be removed from the
-        layoutonly be removed from the layout.
+        layout.
 
         If no axis with ``name: str`` is found then this is a noop.
 
@@ -122,7 +144,10 @@ class PlotItem(pg.PlotItem):
 
         axis = entry['item']
         self.layout.removeItem(axis)
-        axis.scene().removeItem(axis)
+        scn = axis.scene()
+        if scn:
+            scn.removeItem(axis)
+
         if unlink:
             axis.unlinkFromView()
 
@@ -209,7 +234,12 @@ class PlotItem(pg.PlotItem):
                 # adding this is without it there's some weird
                 # ``ViewBox`` geometry bug.. where a gap for the
                 # 'bottom' axis is somehow left in?
-                axis = pg.AxisItem(orientation=name, parent=self)
+                # axis = pg.AxisItem(orientation=name, parent=self)
+                axis = Axis(
+                    self,
+                    orientation=name,
+                    parent=self,
+                )
 
             axis.linkToView(self.vb)
 

piker/ui/_position.py

@@ -41,7 +41,11 @@ from ._anchors import (
     pp_tight_and_right,  # wanna keep it straight in the long run
     gpath_pin,
 )
-from ..calc import humanize, pnl, puterize
+from ..calc import (
+    humanize,
+    pnl,
+    puterize,
+)
 from ..clearing._allocate import Allocator
 from ..pp import Position
 from ..data._normalize import iterticks
@@ -80,9 +84,9 @@ async def update_pnl_from_feed(
     '''
     global _pnl_tasks
 
-    pp = order_mode.current_pp
-    live = pp.live_pp
-    key = live.symbol.front_fqsn()
+    pp: PositionTracker = order_mode.current_pp
+    live: Position = pp.live_pp
+    key: str = live.symbol.front_fqsn()
 
     log.info(f'Starting pnl display for {pp.alloc.account}')
 
@@ -101,11 +105,22 @@ async def update_pnl_from_feed(
         async with flume.stream.subscribe() as bstream:
             # last_tick = time.time()
             async for quotes in bstream:
-
                 # now = time.time()
                 # period = now - last_tick
 
                 for sym, quote in quotes.items():
+                    # print(f'{key} PnL: sym:{sym}')
+
+                    # TODO: uggggh we probably want a better state
+                    # management then this sincce we want to enable
+                    # updating whatever the current symbol is in
+                    # real-time right?
+                    if sym != key:
+                        continue
+
+                    # watch out for wrong quote msg-data if you muck
+                    # with backend feed subs code..
+                    # assert sym == quote['fqsn']
 
                     for tick in iterticks(quote, types):
                         # print(f'{1/period} Hz')
@@ -119,13 +134,17 @@ async def update_pnl_from_feed(
 
                         else:
                             # compute and display pnl status
-                            order_mode.pane.pnl_label.format(
-                                pnl=copysign(1, size) * pnl(
-                                    # live.ppu,
-                                    order_mode.current_pp.live_pp.ppu,
-                                    tick['price'],
-                                ),
-                            )
+                            pnl_val = (
+                                 copysign(1, size)
+                                 *
+                                 pnl(
+                                     # live.ppu,
+                                     order_mode.current_pp.live_pp.ppu,
+                                     tick['price'],
+                                 )
+                             )
+                            # print(f'formatting PNL {sym} => {pnl_val}')
+                            order_mode.pane.pnl_label.format(pnl=pnl_val)
 
                         # last_tick = time.time()
     finally:

piker/ui/_render.py

@@ -0,0 +1,320 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+High level streaming graphics primitives.
+
+This is an intermediate layer which associates real-time low latency
+graphics primitives with underlying stream/flow related data structures
+for fast incremental update.
+
+'''
+from __future__ import annotations
+from typing import (
+    TYPE_CHECKING,
+)
+
+import msgspec
+import numpy as np
+import pyqtgraph as pg
+from PyQt5.QtGui import QPainterPath
+
+from ..data._formatters import (
+    IncrementalFormatter,
+)
+from ..data._pathops import (
+    xy_downsample,
+)
+from ..log import get_logger
+from .._profile import (
+    Profiler,
+)
+
+if TYPE_CHECKING:
+    from ._dataviz import Viz
+
+
+log = get_logger(__name__)
+
+
+class Renderer(msgspec.Struct):
+
+    viz: Viz
+    fmtr: IncrementalFormatter
+
+    # output graphics rendering, the main object
+    # processed in ``QGraphicsObject.paint()``
+    path: QPainterPath | None = None
+    fast_path: QPainterPath | None = None
+
+    # downsampling state
+    _last_uppx: float = 0
+    _in_ds: bool = False
+
+    def draw_path(
+        self,
+        x: np.ndarray,
+        y: np.ndarray,
+        connect: str | np.ndarray = 'all',
+        path: QPainterPath | None = None,
+        redraw: bool = False,
+
+    ) -> QPainterPath:
+
+        path_was_none = path is None
+
+        if redraw and path:
+            path.clear()
+
+            # TODO: avoid this?
+            if self.fast_path:
+                self.fast_path.clear()
+
+        path = pg.functions.arrayToQPath(
+            x,
+            y,
+            connect=connect,
+            finiteCheck=False,
+
+            # reserve mem allocs see:
+            # - https://doc.qt.io/qt-5/qpainterpath.html#reserve
+            # - https://doc.qt.io/qt-5/qpainterpath.html#capacity
+            # - https://doc.qt.io/qt-5/qpainterpath.html#clear
+            # XXX: right now this is based on ad-hoc checks on a
+            # hidpi 3840x2160 4k monitor but we should optimize for
+            # the target display(s) on the sys.
+            # if no_path_yet:
+            #     graphics.path.reserve(int(500e3))
+            # path=path,  # path re-use / reserving
+        )
+
+        # avoid mem allocs if possible
+        if path_was_none:
+            path.reserve(path.capacity())
+
+        return path
+
+    def render(
+        self,
+
+        new_read,
+        array_key: str,
+        profiler: Profiler,
+        uppx: float = 1,
+
+        # redraw and ds flags
+        should_redraw: bool = False,
+        new_sample_rate: bool = False,
+        should_ds: bool = False,
+        showing_src_data: bool = True,
+
+        do_append: bool = True,
+        use_fpath: bool = True,
+
+        # only render datums "in view" of the ``ChartView``
+        use_vr: bool = True,
+
+    ) -> tuple[QPainterPath, bool]:
+        '''
+        Render the current graphics path(s)
+
+        There are (at least) 3 stages from source data to graphics data:
+        - a data transform (which can be stored in additional shm)
+        - a graphics transform which converts discrete basis data to
+          a `float`-basis view-coords graphics basis. (eg. ``ohlc_flatten()``,
+          ``step_path_arrays_from_1d()``, etc.)
+
+        - blah blah blah (from notes)
+
+        '''
+        # TODO: can the renderer just call ``Viz.read()`` directly?
+        # unpack latest source data read
+        fmtr = self.fmtr
+
+        (
+            _,
+            _,
+            array,
+            ivl,
+            ivr,
+            in_view,
+        ) = new_read
+
+        # xy-path data transform: convert source data to a format
+        # able to be passed to a `QPainterPath` rendering routine.
+        fmt_out = fmtr.format_to_1d(
+            new_read,
+            array_key,
+            profiler,
+
+            slice_to_inview=use_vr,
+        )
+
+        # no history in view case
+        if not fmt_out:
+            # XXX: this might be why the profiler only has exits?
+            return
+
+        (
+            x_1d,
+            y_1d,
+            connect,
+            prepend_length,
+            append_length,
+            view_changed,
+            # append_tres,
+
+        ) = fmt_out
+
+        # redraw conditions
+        if (
+            prepend_length > 0
+            or new_sample_rate
+            or view_changed
+
+            # NOTE: comment this to try and make "append paths"
+            # work below..
+            or append_length > 0
+        ):
+            should_redraw = True
+
+        path: QPainterPath = self.path
+        fast_path: QPainterPath = self.fast_path
+        reset: bool = False
+
+        self.viz.yrange = None
+
+        # redraw the entire source data if we have either of:
+        # - no prior path graphic rendered or,
+        # - we always intend to re-render the data only in view
+        if (
+            path is None
+            or should_redraw
+        ):
+            # print(f"{self.viz.name} -> REDRAWING BRUH")
+            if new_sample_rate and showing_src_data:
+                log.info(f'DE-downsampling -> {array_key}')
+                self._in_ds = False
+
+            elif should_ds and uppx > 1:
+
+                ds_out = xy_downsample(
+                    x_1d,
+                    y_1d,
+                    uppx,
+                )
+                if ds_out is not None:
+                    x_1d, y_1d, ymn, ymx = ds_out
+                    self.viz.yrange = ymn, ymx
+                    # print(f'{self.viz.name} post ds: ymn, ymx: {ymn},{ymx}')
+
+                    reset = True
+                    profiler(f'FULL PATH downsample redraw={should_ds}')
+                    self._in_ds = True
+
+            path = self.draw_path(
+                x=x_1d,
+                y=y_1d,
+                connect=connect,
+                path=path,
+                redraw=True,
+            )
+
+            profiler(
+                'generated fresh path. '
+                f'(should_redraw: {should_redraw} '
+                f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})'
+            )
+
+        # TODO: get this piecewise prepend working - right now it's
+        # giving heck on vwap...
+        # elif prepend_length:
+
+        #     prepend_path = pg.functions.arrayToQPath(
+        #         x[0:prepend_length],
+        #         y[0:prepend_length],
+        #         connect='all'
+        #     )
+
+        #     # swap prepend path in "front"
+        #     old_path = graphics.path
+        #     graphics.path = prepend_path
+        #     # graphics.path.moveTo(new_x[0], new_y[0])
+        #     graphics.path.connectPath(old_path)
+
+        elif (
+            append_length > 0
+            and do_append
+        ):
+            profiler(f'sliced append path {append_length}')
+            # (
+            #     x_1d,
+            #     y_1d,
+            #     connect,
+            # ) = append_tres
+
+            profiler(
+                f'diffed array input, append_length={append_length}'
+            )
+
+            # if should_ds and uppx > 1:
+            #     new_x, new_y = xy_downsample(
+            #         new_x,
+            #         new_y,
+            #         uppx,
+            #     )
+            #     profiler(f'fast path downsample redraw={should_ds}')
+
+            append_path = self.draw_path(
+                x=x_1d,
+                y=y_1d,
+                connect=connect,
+                path=fast_path,
+            )
+            profiler('generated append qpath')
+
+            if use_fpath:
+                # an attempt at trying to make append-updates faster..
+                if fast_path is None:
+                    fast_path = append_path
+                    # fast_path.reserve(int(6e3))
+                else:
+                    # print(
+                    #     f'{self.viz.name}: FAST PATH\n'
+                    #     f"append_path br: {append_path.boundingRect()}\n"
+                    #     f"path size: {size}\n"
+                    #     f"append_path len: {append_path.length()}\n"
+                    #     f"fast_path len: {fast_path.length()}\n"
+                    # )
+
+                    fast_path.connectPath(append_path)
+                    size = fast_path.capacity()
+                    profiler(f'connected fast path w size: {size}')
+
+                    # graphics.path.moveTo(new_x[0], new_y[0])
+                    # path.connectPath(append_path)
+
+                    # XXX: lol this causes a hang..
+                    # graphics.path = graphics.path.simplified()
+            else:
+                size = path.capacity()
+                profiler(f'connected history path w size: {size}')
+                path.connectPath(append_path)
+
+        self.path = path
+        self.fast_path = fast_path
+
+        return self.path, reset

piker/ui/_search.py

@@ -144,15 +144,29 @@ class CompleterView(QTreeView):
         self._font_size: int = 0  # pixels
         self._init: bool = False
 
-    async def on_pressed(self, idx: QModelIndex) -> None:
+    async def on_pressed(
+        self,
+        idx: QModelIndex,
+    ) -> None:
         '''
         Mouse pressed on view handler.
 
         '''
         search = self.parent()
-        await search.chart_current_item()
+
+        await search.chart_current_item(
+            clear_to_cache=True,
+        )
+
+        # XXX: this causes Qt to hang and segfault..lovely
+        # self.show_cache_entries(
+        #     only=True,
+        #     keep_current_item_selected=True,
+        # )
+
         search.focus()
 
+
     def set_font_size(self, size: int = 18):
         # print(size)
         if size < 0:
@@ -288,7 +302,7 @@ class CompleterView(QTreeView):
     def select_first(self) -> QStandardItem:
         '''
         Select the first depth >= 2 entry from the completer tree and
-        return it's item.
+        return its item.
 
         '''
         # ensure we're **not** selecting the first level parent node and
@@ -416,12 +430,26 @@ class CompleterView(QTreeView):
         section: str,
         values: Sequence[str],
         clear_all: bool = False,
+        reverse: bool = False,
 
     ) -> None:
         '''
         Set result-rows for depth = 1 tree section ``section``.
 
         '''
+        if (
+            values
+            and not isinstance(values[0], str)
+        ):
+            flattened: list[str] = []
+            for val in values:
+                flattened.extend(val)
+
+            values = flattened
+
+        if reverse:
+            values = reversed(values)
+
         model = self.model()
         if clear_all:
             # XXX: rewrite the model from scratch if caller requests it
@@ -598,22 +626,50 @@ class SearchWidget(QtWidgets.QWidget):
         self.show()
         self.bar.focus()
 
-    def show_only_cache_entries(self) -> None:
+    def show_cache_entries(
+        self,
+        only: bool = False,
+        keep_current_item_selected: bool = False,
+
+    ) -> None:
         '''
         Clear the search results view and show only cached (aka recently
         loaded with active data) feeds in the results section.
 
         '''
         godw = self.godwidget
+
+        # first entry in the cache is the current symbol(s)
+        fqsns = set()
+        for multi_fqsns in list(godw._chart_cache):
+            for fqsn in set(multi_fqsns):
+                fqsns.add(fqsn)
+
+        if keep_current_item_selected:
+            sel = self.view.selectionModel()
+            cidx = sel.currentIndex()
+
         self.view.set_section_entries(
             'cache',
-            list(reversed(godw._chart_cache)),
+            list(fqsns),
             # remove all other completion results except for cache
-            clear_all=True,
+            clear_all=only,
+            reverse=True,
         )
 
-    def get_current_item(self) -> Optional[tuple[str, str]]:
-        '''Return the current completer tree selection as
+        if (
+            keep_current_item_selected
+            and cidx.isValid()
+        ):
+            # set current selection back to what it was before filling out
+            # the view results.
+            self.view.select_from_idx(cidx)
+        else:
+            self.view.select_first()
+
+    def get_current_item(self) -> tuple[QModelIndex, str, str] | None:
+        '''
+        Return the current completer tree selection as
         a tuple ``(parent: str, child: str)`` if valid, else ``None``.
 
         '''
@@ -639,7 +695,11 @@ class SearchWidget(QtWidgets.QWidget):
             if provider == 'cache':
                 symbol, _, provider = symbol.rpartition('.')
 
-            return provider, symbol
+            return (
+                cidx,
+                provider,
+                symbol,
+            )
 
         else:
             return None
@@ -660,15 +720,16 @@ class SearchWidget(QtWidgets.QWidget):
         if value is None:
             return None
 
-        provider, symbol = value
+        cidx, provider, symbol = value
         godw = self.godwidget
 
-        log.info(f'Requesting symbol: {symbol}.{provider}')
+        fqsn = f'{symbol}.{provider}'
+        log.info(f'Requesting symbol: {fqsn}')
 
+        # assert provider in symbol
         await godw.load_symbols(
-            provider,
-            [symbol],
-            'info',
+            fqsns=[fqsn],
+            loglevel='info',
         )
 
         # fully qualified symbol name (SNS i guess is what we're
@@ -682,13 +743,15 @@ class SearchWidget(QtWidgets.QWidget):
             # Re-order the symbol cache on the chart to display in
             # LIFO order. this is normally only done internally by
             # the chart on new symbols being loaded into memory
-            godw.set_chart_symbol(
-                fqsn, (
+            godw.set_chart_symbols(
+                (fqsn,), (
                     godw.hist_linked,
                     godw.rt_linked,
                 )
             )
-            self.show_only_cache_entries()
+            self.show_cache_entries(
+                only=True,
+            )
 
         self.bar.focus()
         return fqsn
@@ -757,9 +820,10 @@ async def pack_matches(
     with trio.CancelScope() as cs:
         task_status.started(cs)
         # ensure ^ status is updated
-        results = await search(pattern)
+        results = list(await search(pattern))
 
-    if provider != 'cache':  # XXX: don't cache the cache results xD
+    # XXX: don't cache the cache results xD
+    if provider != 'cache':
         matches[(provider, pattern)] = results
 
         # print(f'results from {provider}: {results}')
@@ -806,7 +870,7 @@ async def fill_results(
     has_results: defaultdict[str, set[str]] = defaultdict(set)
 
     # show cached feed list at startup
-    search.show_only_cache_entries()
+    search.show_cache_entries()
     search.on_resize()
 
     while True:
@@ -860,8 +924,9 @@ async def fill_results(
                     # it hasn't already been searched with the current
                     # input pattern (in which case just look up the old
                     # results).
-                    if (period >= pause) and (
-                        provider not in already_has_results
+                    if (
+                        period >= pause
+                        and provider not in already_has_results
                     ):
 
                         # TODO: it may make more sense TO NOT search the
@@ -869,7 +934,9 @@ async def fill_results(
                         # cpu-bound.
                         if provider != 'cache':
                             view.clear_section(
-                                provider, status_field='-> searchin..')
+                                provider,
+                                status_field='-> searchin..',
+                            )
 
                         await n.start(
                             pack_matches,
@@ -890,11 +957,20 @@ async def fill_results(
                         # re-searching it's ``dict`` since it's easier
                         # but it also causes it to be slower then cached
                         # results from other providers on occasion.
-                        if results and provider != 'cache':
-                            view.set_section_entries(
-                                section=provider,
-                                values=results,
-                            )
+                        if (
+                            results
+                        ):
+                            if provider != 'cache':
+                                view.set_section_entries(
+                                    section=provider,
+                                    values=results,
+                                )
+                            else:
+                                # if provider == 'cache':
+                                # for the cache just show what we got
+                                # that matches
+                                search.show_cache_entries()
+
                         else:
                             view.clear_section(provider)
 
@@ -916,11 +992,10 @@ async def handle_keyboard_input(
     global _search_active, _search_enabled
 
     # startup
-    bar = searchbar
-    search = searchbar.parent()
-    godwidget = search.godwidget
-    view = bar.view
-    view.set_font_size(bar.dpi_font.px_size)
+    searchw = searchbar.parent()
+    godwidget = searchw.godwidget
+    view = searchbar.view
+    view.set_font_size(searchbar.dpi_font.px_size)
     send, recv = trio.open_memory_channel(616)
 
     async with trio.open_nursery() as n:
@@ -931,13 +1006,13 @@ async def handle_keyboard_input(
         n.start_soon(
             partial(
                 fill_results,
-                search,
+                searchw,
                 recv,
             )
         )
 
-        bar.focus()
-        search.show_only_cache_entries()
+        searchbar.focus()
+        searchw.show_cache_entries()
         await trio.sleep(0)
 
         async for kbmsg in recv_chan:
@@ -949,20 +1024,29 @@ async def handle_keyboard_input(
             if mods == Qt.ControlModifier:
                 ctl = True
 
-            if key in (Qt.Key_Enter, Qt.Key_Return):
+            if key in (
+                Qt.Key_Enter,
+                Qt.Key_Return
+            ):
                 _search_enabled = False
-                await search.chart_current_item(clear_to_cache=True)
-                search.show_only_cache_entries()
-                view.show_matches()
-                search.focus()
+                await searchw.chart_current_item(clear_to_cache=True)
 
-            elif not ctl and not bar.text():
-                # if nothing in search text show the cache
-                view.set_section_entries(
-                    'cache',
-                    list(reversed(godwidget._chart_cache)),
-                    clear_all=True,
-                )
+                # XXX: causes hang and segfault..
+                # searchw.show_cache_entries(
+                #     only=True,
+                #     keep_current_item_selected=True,
+                # )
+
+                view.show_matches()
+                searchw.focus()
+
+            elif (
+                not ctl
+                and not searchbar.text()
+            ):
+                # TODO: really should factor this somewhere..bc
+                # we're doin it in another spot as well..
+                searchw.show_cache_entries(only=True)
                 continue
 
             # cancel and close
@@ -971,7 +1055,7 @@ async def handle_keyboard_input(
                 Qt.Key_Space,   # i feel like this is the "native" one
                 Qt.Key_Alt,
             }:
-                bar.unfocus()
+                searchbar.unfocus()
 
                 # kill the search and focus back on main chart
                 if godwidget:
@@ -979,41 +1063,54 @@ async def handle_keyboard_input(
 
                 continue
 
-            if ctl and key in {
-                Qt.Key_L,
-            }:
+            if (
+                ctl
+                and key in {Qt.Key_L}
+            ):
                 # like url (link) highlight in a web browser
-                bar.focus()
+                searchbar.focus()
 
             # selection navigation controls
-            elif ctl and key in {
-                Qt.Key_D,
-            }:
+            elif (
+                ctl
+                and key in {Qt.Key_D}
+            ):
                 view.next_section(direction='down')
                 _search_enabled = False
 
-            elif ctl and key in {
-                Qt.Key_U,
-            }:
+            elif (
+                ctl
+                and key in {Qt.Key_U}
+            ):
                 view.next_section(direction='up')
                 _search_enabled = False
 
             # selection navigation controls
-            elif (ctl and key in {
+            elif (
+                ctl and (
+                    key in {
+                        Qt.Key_K,
+                        Qt.Key_J,
+                    }
 
-                Qt.Key_K,
-                Qt.Key_J,
-
-            }) or key in {
-
-                Qt.Key_Up,
-                Qt.Key_Down,
-            }:
+                    or key in {
+                        Qt.Key_Up,
+                        Qt.Key_Down,
+                    }
+                )
+            ):
                 _search_enabled = False
-                if key in {Qt.Key_K, Qt.Key_Up}:
+
+                if key in {
+                    Qt.Key_K,
+                    Qt.Key_Up
+                }:
                     item = view.select_previous()
 
-                elif key in {Qt.Key_J, Qt.Key_Down}:
+                elif key in {
+                    Qt.Key_J,
+                    Qt.Key_Down,
+                }:
                     item = view.select_next()
 
                 if item:
@@ -1022,26 +1119,39 @@ async def handle_keyboard_input(
                     # if we're in the cache section and thus the next
                     # selection is a cache item, switch and show it
                     # immediately since it should be very fast.
-                    if parent_item and parent_item.text() == 'cache':
-                        await search.chart_current_item(clear_to_cache=False)
+                    if (
+                        parent_item
+                        and parent_item.text() == 'cache'
+                    ):
+                        await searchw.chart_current_item(clear_to_cache=False)
 
+            # ACTUAL SEARCH BLOCK #
+            # where we fuzzy complete and fill out sections.
             elif not ctl:
                 # relay to completer task
                 _search_enabled = True
-                send.send_nowait(search.bar.text())
+                send.send_nowait(searchw.bar.text())
                 _search_active.set()
 
 
 async def search_simple_dict(
     text: str,
     source: dict,
+
 ) -> dict[str, Any]:
 
+    tokens = []
+    for key in source:
+        if not isinstance(key, str):
+            tokens.extend(key)
+        else:
+            tokens.append(key)
+
     # search routine can be specified as a function such
     # as in the case of the current app's local symbol cache
     matches = fuzzy.extractBests(
         text,
-        source.keys(),
+        tokens,
         score_cutoff=90,
     )
 

piker/ui/_style.py

@@ -240,12 +240,12 @@ def hcolor(name: str) -> str:
         'gunmetal': '#91A3B0',
         'battleship': '#848482',
 
+        # default ohlc-bars/curve gray
+        'bracket': '#666666',  # like the logo
+
         # bluish
         'charcoal': '#36454F',
 
-        # default bars
-        'bracket': '#666666',  # like the logo
-
         # work well for filled polygons which want a 'bracket' feel
         # going light to dark
         'davies': '#555555',

piker/ui/order_mode.py

@@ -88,7 +88,7 @@ class Dialog(Struct):
     # TODO: use ``pydantic.UUID4`` field
     uuid: str
     order: Order
-    symbol: Symbol
+    symbol: str
     lines: list[LevelLine]
     last_status_close: Callable = lambda: None
     msgs: dict[str, dict] = {}
@@ -349,7 +349,7 @@ class OrderMode:
 
         '''
         if not order:
-            staged = self._staged_order
+            staged: Order = self._staged_order
             # apply order fields for ems
             oid = str(uuid.uuid4())
             order = staged.copy()
@@ -379,7 +379,7 @@ class OrderMode:
         dialog = Dialog(
             uuid=order.oid,
             order=order,
-            symbol=order.symbol,
+            symbol=order.symbol,  # XXX: always a str?
             lines=lines,
             last_status_close=self.multistatus.open_status(
                 f'submitting {order.exec_mode}-{order.action}',
@@ -494,7 +494,7 @@ class OrderMode:
 
         uuid: str,
         price: float,
-        arrow_index: float,
+        time_s: float,
 
         pointing: Optional[str] = None,
 
@@ -513,22 +513,32 @@ class OrderMode:
         '''
         dialog = self.dialogs[uuid]
         lines = dialog.lines
+        chart = self.chart
+
         # XXX: seems to fail on certain types of races?
         # assert len(lines) == 2
         if lines:
-            flume: Flume = self.feed.flumes[self.chart.linked.symbol.fqsn]
+            flume: Flume = self.feed.flumes[chart.linked.symbol.fqsn]
             _, _, ratio = flume.get_ds_info()
-            for i, chart in [
-                (arrow_index, self.chart),
-                (flume.izero_hist
-                 +
-                 round((arrow_index - flume.izero_rt)/ratio),
-                 self.hist_chart)
+
+            for chart, shm in [
+                (self.chart, flume.rt_shm),
+                (self.hist_chart, flume.hist_shm),
             ]:
+                viz = chart.get_viz(chart.name)
+                index_field = viz.index_field
+                arr = shm.array
+
+                # TODO: borked for int index based..
+                index = flume.get_index(time_s, arr)
+
+                # get absolute index for arrow placement
+                arrow_index = arr[index_field][index]
+
                 self.arrows.add(
                     chart.plotItem,
                     uuid,
-                    i,
+                    arrow_index,
                     price,
                     pointing=pointing,
                     color=lines[0].color
@@ -693,7 +703,6 @@ async def open_order_mode(
 
         # symbol id
         symbol = chart.linked.symbol
-        symkey = symbol.front_fqsn()
 
         # map of per-provider account keys to position tracker instances
         trackers: dict[str, PositionTracker] = {}
@@ -854,7 +863,7 @@ async def open_order_mode(
         # the expected symbol key in its positions msg.
         for (broker, acctid), msgs in position_msgs.items():
             for msg in msgs:
-                log.info(f'Loading pp for {symkey}:\n{pformat(msg)}')
+                log.info(f'Loading pp for {acctid}@{broker}:\n{pformat(msg)}')
                 await process_trade_msg(
                     mode,
                     book,
@@ -930,7 +939,6 @@ async def process_trade_msg(
 
 ) -> tuple[Dialog, Status]:
 
-    get_index = mode.chart.get_index
     fmsg = pformat(msg)
     log.debug(f'Received order msg:\n{fmsg}')
     name = msg['name']
@@ -965,6 +973,9 @@ async def process_trade_msg(
     oid = msg.oid
     dialog: Dialog = mode.dialogs.get(oid)
 
+    if dialog:
+        fqsn = dialog.symbol
+
     match msg:
         case Status(
             resp='dark_open' | 'open',
@@ -1034,10 +1045,11 @@ async def process_trade_msg(
             # should only be one "fill" for an alert
             # add a triangle and remove the level line
             req = Order(**req)
+            tm = time.time()
             mode.on_fill(
                 oid,
                 price=req.price,
-                arrow_index=get_index(time.time()),
+                time_s=tm,
             )
             mode.lines.remove_line(uuid=oid)
             msg.req = req
@@ -1065,26 +1077,25 @@ async def process_trade_msg(
             action = order.action
             details = msg.brokerd_msg
 
-            # TODO: some kinda progress system
+            # TODO: put the actual exchange timestamp?
+            # TODO: some kinda progress system?
+
+            # NOTE: currently the ``kraken`` openOrders sub
+            # doesn't deliver their engine timestamp as part of
+            # it's schema, so this value is **not** from them
+            # (see our backend code). We should probably either
+            # include all provider-engine timestamps in the
+            # summary 'closed' status msg and/or figure out
+            # a way to indicate what is a `brokerd` stamp versus
+            # a true backend one? This will require finagling
+            # with how each backend tracks/summarizes time
+            # stamps for the downstream API.
+            tm = details['broker_time']
             mode.on_fill(
                 oid,
                 price=details['price'],
+                time_s=tm,
                 pointing='up' if action == 'buy' else 'down',
-
-                # TODO: put the actual exchange timestamp
-                arrow_index=get_index(
-                    # TODO: note currently the ``kraken`` openOrders sub
-                    # doesn't deliver their engine timestamp as part of
-                    # it's schema, so this value is **not** from them
-                    # (see our backend code). We should probably either
-                    # include all provider-engine timestamps in the
-                    # summary 'closed' status msg and/or figure out
-                    # a way to indicate what is a `brokerd` stamp versus
-                    # a true backend one? This will require finagling
-                    # with how each backend tracks/summarizes time
-                    # stamps for the downstream API.
-                    details['broker_time']
-                ),
             )
 
             # TODO: append these fill events to the position's clear

piker/ui/qt/__init__.py

@@ -1,3 +0,0 @@
-"""
-Super hawt Qt UI components
-"""

piker/ui/qt/stackof_candle.py

@@ -1,67 +0,0 @@
-import sys
-
-from PySide2.QtCharts import QtCharts
-from PySide2.QtWidgets import QApplication, QMainWindow
-from PySide2.QtCore import Qt, QPointF
-from PySide2 import QtGui
-import qdarkstyle
-
-data = ((1, 7380, 7520, 7380, 7510, 7324),
-    (2, 7520, 7580, 7410, 7440, 7372),
-    (3, 7440, 7650, 7310, 7520, 7434),
-    (4, 7450, 7640, 7450, 7550, 7480),
-    (5, 7510, 7590, 7460, 7490, 7502),
-    (6, 7500, 7590, 7480, 7560, 7512),
-    (7, 7560, 7830, 7540, 7800, 7584))
-
-
-app = QApplication([])
-# set dark stylesheet
-# import pdb; pdb.set_trace()
-app.setStyleSheet(qdarkstyle.load_stylesheet_pyside())
-
-series = QtCharts.QCandlestickSeries()
-series.setDecreasingColor(Qt.darkRed)
-series.setIncreasingColor(Qt.darkGreen)
-
-ma5 = QtCharts.QLineSeries()  # 5-days average data line
-tm = []  # stores str type data
-
-# in a loop,  series and ma5 append corresponding data
-for num, o, h, l, c, m in data:
-    candle = QtCharts.QCandlestickSet(o, h, l, c)
-    series.append(candle)
-    ma5.append(QPointF(num, m))
-    tm.append(str(num))
-
-pen = candle.pen()
-# import pdb; pdb.set_trace()
-
-chart = QtCharts.QChart()
-
-# import pdb; pdb.set_trace()
-series.setBodyOutlineVisible(False)
-series.setCapsVisible(False)
-# brush = QtGui.QBrush()
-# brush.setColor(Qt.green)
-# series.setBrush(brush)
-chart.addSeries(series)  # candle
-chart.addSeries(ma5)  # ma5 line
-
-chart.setAnimationOptions(QtCharts.QChart.SeriesAnimations)
-chart.createDefaultAxes()
-chart.legend().hide()
-
-chart.axisX(series).setCategories(tm)
-chart.axisX(ma5).setVisible(False)
-
-view = QtCharts.QChartView(chart)
-view.chart().setTheme(QtCharts.QChart.ChartTheme.ChartThemeDark)
-view.setRubberBand(QtCharts.QChartView.HorizontalRubberBand)
-# chartview.chart().setTheme(QtCharts.QChart.ChartTheme.ChartThemeBlueCerulean)
-
-ui = QMainWindow()
-# ui.setGeometry(50, 50, 500, 300)
-ui.setCentralWidget(view)
-ui.show()
-sys.exit(app.exec_())

setup.py

@@ -61,7 +61,7 @@ setup(
         # 'cryptofeed',
 
         # brokers
-        'asks==2.4.8',
+        'asks',
         'ib_insync',
 
         # numerics

snippets/qt_screen_info.py

@@ -1,22 +1,26 @@
 """
-Resource list for mucking with DPIs on multiple screens:
-
-- https://stackoverflow.com/questions/42141354/convert-pixel-size-to-point-size-for-fonts-on-multiple-platforms
-- https://stackoverflow.com/questions/25761556/qt5-font-rendering-different-on-various-platforms/25929628#25929628
-- https://doc.qt.io/qt-5/highdpi.html
-- https://stackoverflow.com/questions/20464814/changing-dpi-scaling-size-of-display-make-qt-applications-font-size-get-rendere
-- https://stackoverflow.com/a/20465247
-- https://doc.qt.io/archives/qt-4.8/qfontmetrics.html#width
-- https://forum.qt.io/topic/54136/how-do-i-get-the-qscreen-my-widget-is-on-qapplication-desktop-screen-returns-a-qwidget-and-qobject_cast-qscreen-returns-null/3
-- https://forum.qt.io/topic/43625/point-sizes-are-they-reliable/4
-- https://stackoverflow.com/questions/16561879/what-is-the-difference-between-logicaldpix-and-physicaldpix-in-qt
-- https://doc.qt.io/qt-5/qguiapplication.html#screenAt
-
+DPI and info helper script for display metrics.
 """
 
-from pyqtgraph import QtGui
+# Resource list for mucking with DPIs on multiple screens:
+# https://stackoverflow.com/questions/42141354/convert-pixel-size-to-point-size-for-fonts-on-multiple-platforms
+# https://stackoverflow.com/questions/25761556/qt5-font-rendering-different-on-various-platforms/25929628#25929628
+# https://doc.qt.io/qt-5/highdpi.html
+# https://stackoverflow.com/questions/20464814/changing-dpi-scaling-size-of-display-make-qt-applications-font-size-get-rendere
+# https://stackoverflow.com/a/20465247
+# https://doc.qt.io/archives/qt-4.8/qfontmetrics.html#width
+# https://forum.qt.io/topic/54136/how-do-i-get-the-qscreen-my-widget-is-on-qapplication-desktop-screen-returns-a-qwidget-and-qobject_cast-qscreen-returns-null/3
+# https://forum.qt.io/topic/43625/point-sizes-are-they-reliable/4
+# https://stackoverflow.com/questions/16561879/what-is-the-difference-between-logicaldpix-and-physicaldpix-in-qt
+# https://doc.qt.io/qt-5/qguiapplication.html#screenAt
+
+from pyqtgraph import (
+    QtGui,
+    QtWidgets,
+)
 from PyQt5.QtCore import (
-     Qt, QCoreApplication
+     Qt,
+     QCoreApplication,
 )
 
 # Proper high DPI scaling is available in Qt >= 5.6.0. This attibute
@@ -28,55 +32,47 @@ if hasattr(Qt, 'AA_UseHighDpiPixmaps'):
     QCoreApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True)
 
 
-app = QtGui.QApplication([])
-window = QtGui.QMainWindow()
-main_widget = QtGui.QWidget()
+app = QtWidgets.QApplication([])
+window = QtWidgets.QMainWindow()
+main_widget = QtWidgets.QWidget()
 window.setCentralWidget(main_widget)
 window.show()
 
+# TODO: move widget through multiple displays and auto-detect the pixel
+# ratio? (probably is gonna require calls to i3ipc on linux)..
 pxr = main_widget.devicePixelRatioF()
 
-# screen_num = app.desktop().screenNumber()
+# TODO: how to detect list of displays from API?
 # screen = app.screens()[screen_num]
 
+
+def ppscreeninfo(screen: 'QScreen') -> None:
+    # screen_num = app.desktop().screenNumber()
+    name = screen.name()
+    size = screen.size()
+    geo = screen.availableGeometry()
+    phydpi = screen.physicalDotsPerInch()
+    logdpi = screen.logicalDotsPerInch()
+    rr = screen.refreshRate()
+
+    print(
+        # f'screen number: {screen_num}\n',
+        f'screen: {name}\n'
+        f'  size: {size}\n'
+        f'  geometry: {geo}\n'
+        f'  logical dpi: {logdpi}\n'
+        f'  devicePixelRationF(): {pxr}\n'
+        f'  physical dpi: {phydpi}\n'
+        f'  refresh rate: {rr}\n'
+    )
+
+    print('-'*50 + '\n')
+
 screen = app.screenAt(main_widget.geometry().center())
-
-name = screen.name()
-size = screen.size()
-geo = screen.availableGeometry()
-phydpi = screen.physicalDotsPerInch()
-logdpi = screen.logicalDotsPerInch()
-
-print(
-    # f'screen number: {screen_num}\n',
-    f'screen name: {name}\n'
-    f'screen size: {size}\n'
-    f'screen geometry: {geo}\n\n'
-    f'devicePixelRationF(): {pxr}\n'
-    f'physical dpi: {phydpi}\n'
-    f'logical dpi: {logdpi}\n'
-)
-
-print('-'*50)
+ppscreeninfo(screen)
 
 screen = app.primaryScreen()
-
-name = screen.name()
-size = screen.size()
-geo = screen.availableGeometry()
-phydpi = screen.physicalDotsPerInch()
-logdpi = screen.logicalDotsPerInch()
-
-print(
-    # f'screen number: {screen_num}\n',
-    f'screen name: {name}\n'
-    f'screen size: {size}\n'
-    f'screen geometry: {geo}\n\n'
-    f'devicePixelRationF(): {pxr}\n'
-    f'physical dpi: {phydpi}\n'
-    f'logical dpi: {logdpi}\n'
-)
-
+ppscreeninfo(screen)
 
 # app-wide font
 font = QtGui.QFont("Hack")