Mk jsronrpc's underlying ws timeout `float('inf')`

Since currently we're only using this IPC subsys for `deribit`, and
generally speaking we're primarly supporting options markets (which are
fairly "slow moving"), flip to a default of NOT resetting the `NoBsWs`
on timeout since doing so normally breaks the jsron-rpc IPC session.
Without a proper `fixture` passed to `open_autorecon_ws()` (which we
should eventually implement!!) relying on a timeout-to-reset more or
less will just cause breakage issues - a proper reconnect sequence must
be implemented before using that feature.

Deats,
- expose and proxy through the `msg_recv_timeout` from
  `open_jsonrpc_session()` into the underlying `open_autorecon_ws()`
  call.

piker/accounting/_pos.py

@@ -30,7 +30,8 @@ from types import ModuleType
 from typing import (
     Any,
     Iterator,
-    Generator
+    Generator,
+    TYPE_CHECKING,
 )
 
 import pendulum
@@ -59,8 +60,10 @@ from ..clearing._messages import (
     BrokerdPosition,
 )
 from piker.types import Struct
-from piker.data._symcache import SymbologyCache
-from ..log import get_logger
+from piker.log import get_logger
+
+if TYPE_CHECKING:
+    from piker.data._symcache import SymbologyCache
 
 log = get_logger(__name__)
 
@@ -493,6 +496,17 @@ class Account(Struct):
 
         _mktmap_table: dict[str, MktPair] | None = None,
 
+        only_require: list[str]|True = True,
+        # ^list of fqmes that are "required" to be processed from
+        # this ledger pass; we often don't care about others and
+        # definitely shouldn't always error in such cases.
+        # (eg. broker backend loaded that doesn't yet supsport the
+        # symcache but also, inside the paper engine we don't ad-hoc
+        # request `get_mkt_info()` for every symbol in the ledger,
+        # only the one for which we're simulating against).
+        # TODO, not sure if there's a better soln for this, ideally
+        # all backends get symcache support afap i guess..
+
     ) -> dict[str, Position]:
         '''
         Update the internal `.pps[str, Position]` table from input
@@ -535,11 +549,32 @@ class Account(Struct):
                 if _mktmap_table is None:
                     raise
 
+                required: bool = (
+                    only_require is True
+                    or (
+                        only_require is not True
+                        and
+                        fqme in only_require
+                    )
+                )
                 # XXX: caller is allowed to provide a fallback
                 # mktmap table for the case where a new position is
                 # being added and the preloaded symcache didn't
                 # have this entry prior (eg. with frickin IB..)
-                mkt = _mktmap_table[fqme]
+                if (
+                    not (mkt := _mktmap_table.get(fqme))
+                    and
+                    required
+                ):
+                    raise
+
+                elif not required:
+                    continue
+
+                else:
+                    # should be an entry retreived somewhere
+                    assert mkt
+
 
             if not (pos := pps.get(bs_mktid)):
 
@@ -656,7 +691,7 @@ class Account(Struct):
     def write_config(self) -> None:
         '''
         Write the current account state to the user's account TOML file, normally
-        something like ``pps.toml``.
+        something like `pps.toml`.
 
         '''
         # TODO: show diff output?

piker/brokers/__init__.py

@@ -98,13 +98,14 @@ async def open_cached_client(
     If one has not been setup do it and cache it.
 
     '''
-    brokermod = get_brokermod(brokername)
+    brokermod: ModuleType = get_brokermod(brokername)
+
+    # TODO: make abstract or `typing.Protocol`
+    # client: Client
     async with maybe_open_context(
         acm_func=brokermod.get_client,
         kwargs=kwargs,
-
     ) as (cache_hit, client):
-
         if cache_hit:
             log.runtime(f'Reusing existing {client}')
 

piker/brokers/cli.py

@@ -471,11 +471,15 @@ def search(
 
     '''
     # global opts
-    brokermods = list(config['brokermods'].values())
+    brokermods: list[ModuleType] = list(config['brokermods'].values())
+
+    # TODO: this is coming from the `search --pdb` NOT from
+    # the `piker --pdb` XD ..
+    # -[ ] pull from the parent click ctx's values..dumdum
+    # assert pdb
 
     # define tractor entrypoint
     async def main(func):
-
         async with maybe_open_pikerd(
             loglevel=config['loglevel'],
             debug_mode=pdb,

piker/brokers/core.py

@@ -22,7 +22,9 @@ routines should be primitive data types where possible.
 """
 import inspect
 from types import ModuleType
-from typing import List, Dict, Any, Optional
+from typing import (
+    Any,
+)
 
 import trio
 
@@ -34,8 +36,10 @@ from ..accounting import MktPair
 
 
 async def api(brokername: str, methname: str, **kwargs) -> dict:
-    """Make (proxy through) a broker API call by name and return its result.
-    """
+    '''
+    Make (proxy through) a broker API call by name and return its result.
+
+    '''
     brokermod = get_brokermod(brokername)
     async with brokermod.get_client() as client:
         meth = getattr(client, methname, None)
@@ -62,10 +66,14 @@ async def api(brokername: str, methname: str, **kwargs) -> dict:
 
 async def stocks_quote(
     brokermod: ModuleType,
-    tickers: List[str]
-) -> Dict[str, Dict[str, Any]]:
-    """Return quotes dict for ``tickers``.
-    """
+    tickers: list[str]
+
+) -> dict[str, dict[str, Any]]:
+    '''
+    Return a `dict` of snapshot quotes for the provided input
+    `tickers`: a `list` of fqmes.
+
+    '''
     async with brokermod.get_client() as client:
         return await client.quote(tickers)
 
@@ -74,13 +82,15 @@ async def stocks_quote(
 async def option_chain(
     brokermod: ModuleType,
     symbol: str,
-    date: Optional[str] = None,
-) -> Dict[str, Dict[str, Dict[str, Any]]]:
-    """Return option chain for ``symbol`` for ``date``.
+    date: str|None = None,
+) -> dict[str, dict[str, dict[str, Any]]]:
+    '''
+    Return option chain for ``symbol`` for ``date``.
 
     By default all expiries are returned. If ``date`` is provided
     then contract quotes for that single expiry are returned.
-    """
+
+    '''
     async with brokermod.get_client() as client:
         if date:
             id = int((await client.tickers2ids([symbol]))[symbol])
@@ -98,7 +108,7 @@ async def option_chain(
 # async def contracts(
 #     brokermod: ModuleType,
 #     symbol: str,
-# ) -> Dict[str, Dict[str, Dict[str, Any]]]:
+# ) -> dict[str, dict[str, dict[str, Any]]]:
 #     """Return option contracts (all expiries) for ``symbol``.
 #     """
 #     async with brokermod.get_client() as client:
@@ -110,15 +120,24 @@ async def bars(
     brokermod: ModuleType,
     symbol: str,
     **kwargs,
-) -> Dict[str, Dict[str, Dict[str, Any]]]:
-    """Return option contracts (all expiries) for ``symbol``.
-    """
+) -> dict[str, dict[str, dict[str, Any]]]:
+    '''
+    Return option contracts (all expiries) for ``symbol``.
+
+    '''
     async with brokermod.get_client() as client:
         return await client.bars(symbol, **kwargs)
 
 
-async def search_w_brokerd(name: str, pattern: str) -> dict:
+async def search_w_brokerd(
+    name: str,
+    pattern: str,
+) -> dict:
 
+    # TODO: WHY NOT WORK!?!
+    # when we `step` through the next block?
+    # import tractor
+    # await tractor.pause()
     async with open_cached_client(name) as client:
 
         # TODO: support multiple asset type concurrent searches.
@@ -130,12 +149,12 @@ async def symbol_search(
     pattern: str,
     **kwargs,
 
-) -> Dict[str, Dict[str, Dict[str, Any]]]:
+) -> dict[str, dict[str, dict[str, Any]]]:
     '''
     Return symbol info from broker.
 
     '''
-    results = []
+    results: list[str] = []
 
     async def search_backend(
         brokermod: ModuleType
@@ -143,6 +162,13 @@ async def symbol_search(
 
         brokername: str = mod.name
 
+        # TODO: figure this the FUCK OUT
+        # -> ok so obvi in the root actor any async task that's
+        # spawned outside the main tractor-root-actor task needs to
+        # call this..
+        # await tractor.devx._debug.maybe_init_greenback()
+        # tractor.pause_from_sync()
+
         async with maybe_spawn_brokerd(
             mod.name,
             infect_asyncio=getattr(
@@ -162,7 +188,6 @@ async def symbol_search(
             ))
 
     async with trio.open_nursery() as n:
-
         for mod in brokermods:
             n.start_soon(search_backend, mod.name)
 
@@ -172,11 +197,13 @@ async def symbol_search(
 async def mkt_info(
     brokermod: ModuleType,
     fqme: str,
+
     **kwargs,
 
 ) -> MktPair:
     '''
-    Return MktPair info from broker including src and dst assets.
+    Return the `piker.accounting.MktPair` info struct from a given
+    backend broker tradable src/dst asset pair.
 
     '''
     async with open_cached_client(brokermod.name) as client:

piker/clearing/_client.py

@@ -168,7 +168,6 @@ class OrderClient(Struct):
 
 
 async def relay_orders_from_sync_code(
-
     client: OrderClient,
     symbol_key: str,
     to_ems_stream: tractor.MsgStream,
@@ -242,6 +241,11 @@ async def open_ems(
 
     async with maybe_open_emsd(
         broker,
+        # XXX NOTE, LOL so this determines the daemon `emsd` loglevel
+        # then FYI.. that's kinda wrong no?
+        # -[ ] shouldn't it be set by `pikerd -l` or no?
+        # -[ ] would make a lot more sense to have a subsys ctl for
+        #     levels.. like `-l emsd.info` or something?
         loglevel=loglevel,
     ) as portal:
 

piker/clearing/_ems.py

@@ -653,7 +653,11 @@ class Router(Struct):
             flume = feed.flumes[fqme]
             first_quote: dict = flume.first_quote
             book: DarkBook = self.get_dark_book(broker)
-            book.lasts[fqme]: float = float(first_quote['last'])
+
+            if not (last := first_quote.get('last')):
+                last: float = flume.rt_shm.array[-1]['close']
+
+            book.lasts[fqme]: float = float(last)
 
             async with self.maybe_open_brokerd_dialog(
                 brokermod=brokermod,
@@ -716,7 +720,7 @@ class Router(Struct):
             subs = self.subscribers[sub_key]
 
         sent_some: bool = False
-        for client_stream in subs:
+        for client_stream in subs.copy():
             try:
                 await client_stream.send(msg)
                 sent_some = True
@@ -1010,10 +1014,14 @@ async def translate_and_relay_brokerd_events(
                 status_msg.brokerd_msg = msg
                 status_msg.src = msg.broker_details['name']
 
-                await router.client_broadcast(
-                    status_msg.req.symbol,
-                    status_msg,
-                )
+                if not status_msg.req:
+                    # likely some order change state?
+                    await tractor.pause()
+                else:
+                    await router.client_broadcast(
+                        status_msg.req.symbol,
+                        status_msg,
+                    )
 
                 if status == 'closed':
                     log.info(f'Execution for {oid} is complete!')

piker/clearing/_paper_engine.py

@@ -297,6 +297,8 @@ class PaperBoi(Struct):
 
         # transmit pp msg to ems
         pp: Position = self.acnt.pps[bs_mktid]
+        # TODO, this will break if `require_only=True` was passed to
+        # `.update_from_ledger()`
 
         pp_msg = BrokerdPosition(
             broker=self.broker,
@@ -653,6 +655,7 @@ async def open_trade_dialog(
                 # in) use manually constructed table from calling
                 # the `.get_mkt_info()` provider EP above.
                 _mktmap_table=mkt_by_fqme,
+                only_require=list(mkt_by_fqme),
             )
 
             pp_msgs: list[BrokerdPosition] = []

piker/clearing/_util.py

@@ -30,6 +30,7 @@ subsys: str = 'piker.clearing'
 
 log = get_logger(subsys)
 
+# TODO, oof doesn't this ignore the `loglevel` then???
 get_console_log = partial(
     get_console_log,
     name=subsys,

piker/cli/__init__.py

@@ -140,11 +140,10 @@ def pikerd(
 
         if pdb:
             log.warning((
-                "\n"
-                "!!! YOU HAVE ENABLED DAEMON DEBUG MODE !!!\n"
-                "When a `piker` daemon crashes it will block the "
-                "task-thread until resumed from console!\n"
-                "\n"
+                '\n\n'
+                '!!! YOU HAVE ENABLED DAEMON DEBUG MODE !!!\n'
+                'When a `piker` daemon crashes it will block the '
+                'task-thread until resumed from console!\n'
             ))
 
         # service-actor registry endpoint socket-address set
@@ -177,7 +176,7 @@ def pikerd(
         from .. import service
 
         async def main():
-            service_mngr: service.Services
+            service_mngr: service.ServiceMngr
 
             async with (
                 service.open_pikerd(

piker/data/_sampling.py

@@ -95,6 +95,12 @@ class Sampler:
     # history loading.
     incr_task_cs: trio.CancelScope | None = None
 
+    bcast_errors: tuple[Exception] = (
+        trio.BrokenResourceError,
+        trio.ClosedResourceError,
+        trio.EndOfChannel,
+    )
+
     # holds all the ``tractor.Context`` remote subscriptions for
     # a particular sample period increment event: all subscribers are
     # notified on a step.
@@ -258,14 +264,15 @@ class Sampler:
         subs: set
         last_ts, subs = pair
 
-        task = trio.lowlevel.current_task()
-        log.debug(
-            f'SUBS {self.subscribers}\n'
-            f'PAIR {pair}\n'
-            f'TASK: {task}: {id(task)}\n'
-            f'broadcasting {period_s} -> {last_ts}\n'
-            # f'consumers: {subs}'
-        )
+        # NOTE, for debugging pub-sub issues
+        # task = trio.lowlevel.current_task()
+        # log.debug(
+        #     f'AlL-SUBS@{period_s!r}: {self.subscribers}\n'
+        #     f'PAIR: {pair}\n'
+        #     f'TASK: {task}: {id(task)}\n'
+        #     f'broadcasting {period_s} -> {last_ts}\n'
+        #     f'consumers: {subs}'
+        # )
         borked: set[MsgStream] = set()
         sent: set[MsgStream] = set()
         while True:
@@ -282,12 +289,11 @@ class Sampler:
                         await stream.send(msg)
                         sent.add(stream)
 
-                    except (
-                        trio.BrokenResourceError,
-                        trio.ClosedResourceError
-                    ):
+                    except self.bcast_errors as err:
                         log.error(
-                            f'{stream._ctx.chan.uid} dropped connection'
+                            f'Connection dropped for IPC ctx\n'
+                            f'{stream._ctx}\n\n'
+                            f'Due to {type(err)}'
                         )
                         borked.add(stream)
                 else:
@@ -394,7 +400,8 @@ async def register_with_sampler(
                 finally:
                     if (
                         sub_for_broadcasts
-                        and subs
+                        and
+                        subs
                     ):
                         try:
                             subs.remove(stream)
@@ -561,8 +568,7 @@ async def open_sample_stream(
 
 
 async def sample_and_broadcast(
-
-    bus: _FeedsBus,  # noqa
+    bus: _FeedsBus,
     rt_shm: ShmArray,
     hist_shm: ShmArray,
     quote_stream: trio.abc.ReceiveChannel,
@@ -582,11 +588,33 @@ async def sample_and_broadcast(
 
     overruns = Counter()
 
+    # NOTE, only used for debugging live-data-feed issues, though
+    # this should be resolved more correctly in the future using the
+    # new typed-msgspec feats of `tractor`!
+    #
+    # XXX, a multiline nested `dict` formatter (since rn quote-msgs
+    # are just that).
+    # pfmt: Callable[[str], str] = mk_repr()
+
     # iterate stream delivered by broker
     async for quotes in quote_stream:
         # print(quotes)
 
-        # TODO: ``numba`` this!
+        # XXX WARNING XXX only enable for debugging bc ow can cost
+        # ALOT of perf with HF-feedz!!!
+        #
+        # log.info(
+        #     'Rx live quotes:\n'
+        #     f'{pfmt(quotes)}'
+        # )
+
+        # TODO,
+        # -[ ] `numba` or `cython`-nize this loop possibly?
+        #  |_alternatively could we do it in rust somehow by upacking
+        #    arrow msgs instead of using `msgspec`?
+        # -[ ] use `msgspec.Struct` support in new typed-msging from
+        #     `tractor` to ensure only allowed msgs are transmitted?
+        #
         for broker_symbol, quote in quotes.items():
             # TODO: in theory you can send the IPC msg *before* writing
             # to the sharedmem array to decrease latency, however, that
@@ -659,6 +687,21 @@ async def sample_and_broadcast(
             sub_key: str = broker_symbol.lower()
             subs: set[Sub] = bus.get_subs(sub_key)
 
+            # TODO, figure out how to make this useful whilst
+            # incoporating feed "pausing" ..
+            #
+            # if not subs:
+            #     all_bs_fqmes: list[str] = list(
+            #         bus._subscribers.keys()
+            #     )
+            #     log.warning(
+            #         f'No subscribers for {brokername!r} live-quote ??\n'
+            #         f'broker_symbol: {broker_symbol}\n\n'
+
+            #         f'Maybe the backend-sys symbol does not match one of,\n'
+            #         f'{pfmt(all_bs_fqmes)}\n'
+            #     )
+
             # NOTE: by default the broker backend doesn't append
             # it's own "name" into the fqme schema (but maybe it
             # should?) so we have to manually generate the correct
@@ -728,18 +771,14 @@ async def sample_and_broadcast(
                         if lags > 10:
                             await tractor.pause()
 
-                except (
-                    trio.BrokenResourceError,
-                    trio.ClosedResourceError,
-                    trio.EndOfChannel,
-                ):
+                except Sampler.bcast_errors as ipc_err:
                     ctx: Context = ipc._ctx
                     chan: Channel = ctx.chan
                     if ctx:
                         log.warning(
-                            'Dropped `brokerd`-quotes-feed connection:\n'
-                            f'{broker_symbol}:'
-                            f'{ctx.cid}@{chan.uid}'
+                            f'Dropped `brokerd`-feed for {broker_symbol!r} due to,\n'
+                            f'x>) {ctx.cid}@{chan.uid}'
+                            f'|_{ipc_err!r}\n\n'
                         )
                     if sub.throttle_rate:
                         assert ipc._closed
@@ -756,12 +795,11 @@ async def sample_and_broadcast(
 
 
 async def uniform_rate_send(
-
     rate: float,
     quote_stream: trio.abc.ReceiveChannel,
     stream: MsgStream,
 
-    task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
+    task_status: TaskStatus[None] = trio.TASK_STATUS_IGNORED,
 
 ) -> None:
     '''
@@ -779,13 +817,16 @@ async def uniform_rate_send(
     https://gist.github.com/njsmith/7ea44ec07e901cb78ebe1dd8dd846cb9
 
     '''
-    # TODO: compute the approx overhead latency per cycle
-    left_to_sleep = throttle_period = 1/rate - 0.000616
+    # ?TODO? dynamically compute the **actual** approx overhead latency per cycle
+    # instead of this magic # bidinezz?
+    throttle_period: float = 1/rate - 0.000616
+    left_to_sleep: float = throttle_period
 
     # send cycle state
+    first_quote: dict|None
     first_quote = last_quote = None
-    last_send = time.time()
-    diff = 0
+    last_send: float = time.time()
+    diff: float = 0
 
     task_status.started()
     ticks_by_type: dict[
@@ -796,22 +837,28 @@ async def uniform_rate_send(
     clear_types = _tick_groups['clears']
 
     while True:
-
         # compute the remaining time to sleep for this throttled cycle
-        left_to_sleep = throttle_period - diff
+        left_to_sleep: float = throttle_period - diff
 
         if left_to_sleep > 0:
+            cs: trio.CancelScope
             with trio.move_on_after(left_to_sleep) as cs:
+                sym: str
+                last_quote: dict
                 try:
                     sym, last_quote = await quote_stream.receive()
                 except trio.EndOfChannel:
-                    log.exception(f"feed for {stream} ended?")
+                    log.exception(
+                        f'Live stream for feed for ended?\n'
+                        f'<=c\n'
+                        f'  |_[{stream!r}\n'
+                    )
                     break
 
-                diff = time.time() - last_send
+                diff: float = time.time() - last_send
 
                 if not first_quote:
-                    first_quote = last_quote
+                    first_quote: float = last_quote
                     # first_quote['tbt'] = ticks_by_type
 
                 if (throttle_period - diff) > 0:
@@ -872,7 +919,9 @@ async def uniform_rate_send(
         # TODO: now if only we could sync this to the display
         # rate timing exactly lul
         try:
-            await stream.send({sym: first_quote})
+            await stream.send({
+                sym: first_quote
+            })
         except tractor.RemoteActorError as rme:
             if rme.type is not tractor._exceptions.StreamOverrun:
                 raise
@@ -883,19 +932,28 @@ async def uniform_rate_send(
                 f'{sym}:{ctx.cid}@{chan.uid}'
             )
 
+        # NOTE: any of these can be raised by `tractor`'s IPC
+        # transport-layer and we want to be highly resilient
+        # to consumers which crash or lose network connection.
+        # I.e. we **DO NOT** want to crash and propagate up to
+        # ``pikerd`` these kinds of errors!
         except (
-            # NOTE: any of these can be raised by ``tractor``'s IPC
-            # transport-layer and we want to be highly resilient
-            # to consumers which crash or lose network connection.
-            # I.e. we **DO NOT** want to crash and propagate up to
-            # ``pikerd`` these kinds of errors!
-            trio.ClosedResourceError,
-            trio.BrokenResourceError,
             ConnectionResetError,
-        ):
-            # if the feed consumer goes down then drop
-            # out of this rate limiter
-            log.warning(f'{stream} closed')
+        ) + Sampler.bcast_errors as ipc_err:
+            match ipc_err:
+                case trio.EndOfChannel():
+                    log.info(
+                        f'{stream} terminated by peer,\n'
+                        f'{ipc_err!r}'
+                    )
+                case _:
+                    # if the feed consumer goes down then drop
+                    # out of this rate limiter
+                    log.warning(
+                        f'{stream} closed due to,\n'
+                        f'{ipc_err!r}'
+                    )
+
             await stream.aclose()
             return
 

piker/data/_symcache.py

@@ -31,6 +31,7 @@ from pathlib import Path
 from pprint import pformat
 from typing import (
     Any,
+    Callable,
     Sequence,
     Hashable,
     TYPE_CHECKING,
@@ -56,7 +57,7 @@ from piker.brokers import (
 )
 
 if TYPE_CHECKING:
-    from ..accounting import (
+    from piker.accounting import (
         Asset,
         MktPair,
     )
@@ -149,57 +150,68 @@ class SymbologyCache(Struct):
                     'Implement `Client.get_assets()`!'
                 )
 
-            if get_mkt_pairs := getattr(client, 'get_mkt_pairs', None):
-
-                pairs: dict[str, Struct] = await get_mkt_pairs()
-                for bs_fqme, pair in pairs.items():
-
-                    # NOTE: every backend defined pair should
-                    # declare it's ns path for roundtrip
-                    # serialization lookup.
-                    if not getattr(pair, 'ns_path', None):
-                        raise TypeError(
-                            f'Pair-struct for {self.mod.name} MUST define a '
-                            '`.ns_path: str`!\n'
-                            f'{pair}'
-                        )
-
-                    entry = await self.mod.get_mkt_info(pair.bs_fqme)
-                    if not entry:
-                        continue
-
-                    mkt: MktPair
-                    pair: Struct
-                    mkt, _pair = entry
-                    assert _pair is pair, (
-                        f'`{self.mod.name}` backend probably has a '
-                        'keying-symmetry problem between the pair-`Struct` '
-                        'returned from `Client.get_mkt_pairs()`and the '
-                        'module level endpoint: `.get_mkt_info()`\n\n'
-                        "Here's the struct diff:\n"
-                        f'{_pair - pair}'
-                    )
-                    # NOTE XXX: this means backends MUST implement
-                    # a `Struct.bs_mktid: str` field to provide
-                    # a native-keyed map to their own symbol
-                    # set(s).
-                    self.pairs[pair.bs_mktid] = pair
-
-                    # NOTE: `MktPair`s are keyed here using piker's
-                    # internal FQME schema so that search,
-                    # accounting and feed init can be accomplished
-                    # a sane, uniform, normalized basis.
-                    self.mktmaps[mkt.fqme] = mkt
-
-                self.pair_ns_path: str = tractor.msg.NamespacePath.from_ref(
-                    pair,
-                )
-
-            else:
+            get_mkt_pairs: Callable|None = getattr(
+                client,
+                'get_mkt_pairs',
+                None,
+            )
+            if not get_mkt_pairs:
                 log.warning(
                     'No symbology cache `Pair` support for `{provider}`..\n'
                     'Implement `Client.get_mkt_pairs()`!'
                 )
+                return self
+
+            pairs: dict[str, Struct] = await get_mkt_pairs()
+            if not pairs:
+                log.warning(
+                    'No pairs from intial {provider!r} sym-cache request?\n\n'
+                    '`Client.get_mkt_pairs()` -> {pairs!r} ?'
+                )
+                return self
+
+            for bs_fqme, pair in pairs.items():
+                if not getattr(pair, 'ns_path', None):
+                    # XXX: every backend defined pair must declare
+                    # a `.ns_path: tractor.NamespacePath` to enable
+                    # roundtrip serialization lookup from a local
+                    # cache file.
+                    raise TypeError(
+                        f'Pair-struct for {self.mod.name} MUST define a '
+                        '`.ns_path: str`!\n\n'
+                        f'{pair!r}'
+                    )
+
+                entry = await self.mod.get_mkt_info(pair.bs_fqme)
+                if not entry:
+                    continue
+
+                mkt: MktPair
+                pair: Struct
+                mkt, _pair = entry
+                assert _pair is pair, (
+                    f'`{self.mod.name}` backend probably has a '
+                    'keying-symmetry problem between the pair-`Struct` '
+                    'returned from `Client.get_mkt_pairs()`and the '
+                    'module level endpoint: `.get_mkt_info()`\n\n'
+                    "Here's the struct diff:\n"
+                    f'{_pair - pair}'
+                )
+                # NOTE XXX: this means backends MUST implement
+                # a `Struct.bs_mktid: str` field to provide
+                # a native-keyed map to their own symbol
+                # set(s).
+                self.pairs[pair.bs_mktid] = pair
+
+                # NOTE: `MktPair`s are keyed here using piker's
+                # internal FQME schema so that search,
+                # accounting and feed init can be accomplished
+                # a sane, uniform, normalized basis.
+                self.mktmaps[mkt.fqme] = mkt
+
+            self.pair_ns_path: str = tractor.msg.NamespacePath.from_ref(
+                pair,
+            )
 
         return self
 

piker/data/_web_bs.py

@@ -273,7 +273,7 @@ async def _reconnect_forever(
                 nobsws._connected.set()
                 await trio.sleep_forever()
         except HandshakeError:
-            log.exception(f'Retrying connection')
+            log.exception('Retrying connection')
 
         # ws & nursery block ends
 
@@ -359,8 +359,8 @@ async def open_autorecon_ws(
 
 
 '''
-JSONRPC response-request style machinery for transparent multiplexing of msgs
-over a NoBsWs.
+JSONRPC response-request style machinery for transparent multiplexing
+of msgs over a `NoBsWs`.
 
 '''
 
@@ -377,43 +377,82 @@ async def open_jsonrpc_session(
     url: str,
     start_id: int = 0,
     response_type: type = JSONRPCResult,
-    request_type: Optional[type] = None,
-    request_hook: Optional[Callable] = None,
-    error_hook: Optional[Callable] = None,
+    msg_recv_timeout: float = float('inf'),
+    # ^NOTE, since only `deribit` is using this jsonrpc stuff atm
+    # and options mkts are generally "slow moving"..
+    #
+    # FURTHER if we break the underlying ws connection then since we
+    # don't pass a `fixture` to the task that manages `NoBsWs`, i.e.
+    # `_reconnect_forever()`, the jsonrpc "transport pipe" get's
+    # broken and never restored with wtv init sequence is required to
+    # re-establish a working req-resp session.
+
 ) -> Callable[[str, dict], dict]:
+    '''
+    Init a json-RPC-over-websocket connection to the provided `url`.
+
+    A `json_rpc: Callable[[str, dict], dict` is delivered to the
+    caller for sending requests and a bg-`trio.Task` handles
+    processing of response msgs including error reporting/raising in
+    the parent/caller task.
+
+    '''
+    # NOTE, store all request msgs so we can raise errors on the
+    # caller side!
+    req_msgs: dict[int, dict] = {}
 
     async with (
-        trio.open_nursery() as n,
-        open_autorecon_ws(url) as ws
+        trio.open_nursery() as tn,
+        open_autorecon_ws(
+            url=url,
+            msg_recv_timeout=msg_recv_timeout,
+        ) as ws
     ):
-        rpc_id: Iterable = count(start_id)
+        rpc_id: Iterable[int] = count(start_id)
         rpc_results: dict[int, dict] = {}
 
-        async def json_rpc(method: str, params: dict) -> dict:
+        async def json_rpc(
+            method: str,
+            params: dict,
+        ) -> dict:
             '''
             perform a json rpc call and wait for the result, raise exception in
             case of error field present on response
             '''
+            nonlocal req_msgs
+
+            req_id: int = next(rpc_id)
             msg = {
                 'jsonrpc': '2.0',
-                'id': next(rpc_id),
+                'id': req_id,
                 'method': method,
                 'params': params
             }
             _id = msg['id']
 
-            rpc_results[_id] = {
+            result = rpc_results[_id] = {
                 'result': None,
-                'event': trio.Event()
+                'error': None,
+                'event': trio.Event(),  # signal caller resp arrived
             }
+            req_msgs[_id] = msg
 
             await ws.send_msg(msg)
 
+            # wait for reponse before unblocking requester code
             await rpc_results[_id]['event'].wait()
 
-            ret = rpc_results[_id]['result']
+            if (maybe_result := result['result']):
+                ret = maybe_result
+                del rpc_results[_id]
 
-            del rpc_results[_id]
+            else:
+                err = result['error']
+                raise Exception(
+                    f'JSONRPC request failed\n'
+                    f'req: {msg}\n'
+                    f'resp: {err}\n'
+                )
 
             if ret.error is not None:
                 raise Exception(json.dumps(ret.error, indent=4))
@@ -428,6 +467,7 @@ async def open_jsonrpc_session(
             the server side.
 
             '''
+            nonlocal req_msgs
             async for msg in ws:
                 match msg:
                     case {
@@ -451,19 +491,28 @@ async def open_jsonrpc_session(
                         'params': _,
                     }:
                         log.debug(f'Recieved\n{msg}')
-                        if request_hook:
-                            await request_hook(request_type(**msg))
 
                     case {
                         'error': error
                     }:
-                        log.warning(f'Recieved\n{error}')
-                        if error_hook:
-                            await error_hook(response_type(**msg))
+                        # retreive orig request msg, set error
+                        # response in original "result" msg,
+                        # THEN FINALLY set the event to signal caller
+                        # to raise the error in the parent task.
+                        req_id: int = error['id']
+                        req_msg: dict = req_msgs[req_id]
+                        result: dict = rpc_results[req_id]
+                        result['error'] = error
+                        result['event'].set()
+                        log.error(
+                            f'JSONRPC request failed\n'
+                            f'req: {req_msg}\n'
+                            f'resp: {error}\n'
+                        )
 
                     case _:
                         log.warning(f'Unhandled JSON-RPC msg!?\n{msg}')
 
-        n.start_soon(recv_task)
+        tn.start_soon(recv_task)
         yield json_rpc
-        n.cancel_scope.cancel()
+        tn.cancel_scope.cancel()

piker/data/feed.py

@@ -786,7 +786,6 @@ async def install_brokerd_search(
 
 @acm
 async def maybe_open_feed(
-
     fqmes: list[str],
     loglevel: str | None = None,
 
@@ -840,13 +839,12 @@ async def maybe_open_feed(
 
 @acm
 async def open_feed(
-
     fqmes: list[str],
 
-    loglevel: str | None = None,
+    loglevel: str|None = None,
     allow_overruns: bool = True,
     start_stream: bool = True,
-    tick_throttle: float | None = None,  # Hz
+    tick_throttle: float|None = None,  # Hz
 
     allow_remote_ctl_ui: bool = False,
 

piker/data/flows.py

@@ -36,10 +36,10 @@ from ._sharedmem import (
     ShmArray,
     _Token,
 )
+from piker.accounting import MktPair
 
 if TYPE_CHECKING:
-    from ..accounting import MktPair
-    from .feed import Feed
+    from piker.data.feed import Feed
 
 
 class Flume(Struct):
@@ -82,7 +82,7 @@ class Flume(Struct):
 
     # TODO: do we need this really if we can pull the `Portal` from
     # ``tractor``'s internals?
-    feed: Feed | None = None
+    feed: Feed|None = None
 
     @property
     def rt_shm(self) -> ShmArray:

piker/data/validate.py

@@ -113,9 +113,9 @@ def validate_backend(
             )
             if ep is None:
                 log.warning(
-                    f'Provider backend {mod.name} is missing '
-                    f'{daemon_name} support :(\n'
-                    f'The following endpoint is missing: {name}'
+                    f'Provider backend {mod.name!r} is missing '
+                    f'{daemon_name!r} support?\n'
+                    f'|_module endpoint-func missing: {name!r}\n'
                 )
 
     inits: list[

piker/log.py

@@ -19,6 +19,10 @@ Log like a forester!
 """
 import logging
 import json
+import reprlib
+from typing import (
+    Callable,
+)
 
 import tractor
 from pygments import (
@@ -84,3 +88,29 @@ def colorize_json(
         # likeable styles: algol_nu, tango, monokai
         formatters.TerminalTrueColorFormatter(style=style)
     )
+
+
+# TODO, eventually defer to the version in `modden` once
+# it becomes a dep!
+def mk_repr(
+    **repr_kws,
+) -> Callable[[str], str]:
+    '''
+    Allocate and deliver a `repr.Repr` instance with provided input
+    settings using the std-lib's `reprlib` mod,
+     * https://docs.python.org/3/library/reprlib.html
+
+    ------ Ex. ------
+    An up to 6-layer-nested `dict` as multi-line:
+    - https://stackoverflow.com/a/79102479
+    - https://docs.python.org/3/library/reprlib.html#reprlib.Repr.maxlevel
+
+    '''
+    def_kws: dict[str, int] = dict(
+        indent=2,
+        maxlevel=6,  # recursion levels
+        maxstring=66,  # match editor line-len limit
+    )
+    def_kws |= repr_kws
+    reprr = reprlib.Repr(**def_kws)
+    return reprr.repr

piker/service/_actor_runtime.py

@@ -119,6 +119,10 @@ async def open_piker_runtime(
                 # spawn other specialized daemons I think?
                 enable_modules=enable_modules,
 
+                # TODO: how to configure this?
+                # keep it on by default if debug mode is set?
+                maybe_enable_greenback=False,
+
                 **tractor_kwargs,
             ) as actor,
 

piker/storage/cli.py

@@ -386,6 +386,8 @@ def ldshm(
             open_annot_ctl() as actl,
         ):
             shm_df: pl.DataFrame | None = None
+            tf2aids: dict[float, dict] = {}
+
             for (
                 shmfile,
                 shm,
@@ -526,16 +528,17 @@ def ldshm(
                             new_df,
                             step_gaps,
                         )
-
                         # last chance manual overwrites in REPL
-                        await tractor.pause()
+                        # await tractor.pause()
                         assert aids
+                        tf2aids[period_s] = aids
 
                 else:
                     # allow interaction even when no ts problems.
-                    await tractor.pause()
-                    # assert not diff
+                    assert not diff
 
+            await tractor.pause()
+            log.info('Exiting TSP shm anal-izer!')
 
             if shm_df is None:
                 log.error(

piker/storage/nativedb.py

@@ -161,7 +161,13 @@ class NativeStorageClient:
 
     def index_files(self):
         for path in self._datadir.iterdir():
-            if path.name in {'borked', 'expired',}:
+            if (
+                path.is_dir()
+                or
+                '.parquet' not in str(path)
+                # or
+                # path.name in {'borked', 'expired',}
+            ):
                 continue
 
             key: str = path.name.rstrip('.parquet')

piker/tsp/__init__.py

@@ -44,8 +44,10 @@ import trio
 from trio_typing import TaskStatus
 import tractor
 from pendulum import (
+    Interval,
     DateTime,
     Duration,
+    duration as mk_duration,
     from_timestamp,
 )
 import numpy as np
@@ -214,7 +216,8 @@ async def maybe_fill_null_segments(
         # pair, immediately stop backfilling?
         if (
             start_dt
-            and end_dt < start_dt
+            and
+            end_dt < start_dt
         ):
             await tractor.pause()
             break
@@ -262,6 +265,7 @@ async def maybe_fill_null_segments(
         except tractor.ContextCancelled:
             # log.exception
             await tractor.pause()
+            raise
 
     null_segs_detected.set()
     # RECHECK for more null-gaps
@@ -349,7 +353,7 @@ async def maybe_fill_null_segments(
 
 async def start_backfill(
     get_hist,
-    frame_types: dict[str, Duration] | None,
+    def_frame_duration: Duration,
     mod: ModuleType,
     mkt: MktPair,
     shm: ShmArray,
@@ -379,22 +383,23 @@ async def start_backfill(
         update_start_on_prepend: bool = False
         if backfill_until_dt is None:
 
-            # TODO: drop this right and just expose the backfill
-            # limits inside a [storage] section in conf.toml?
-            # when no tsdb "last datum" is provided, we just load
-            # some near-term history.
-            # periods = {
-            #     1: {'days': 1},
-            #     60: {'days': 14},
-            # }
-
-            # do a decently sized backfill and load it into storage.
+            # TODO: per-provider default history-durations?
+            # -[ ] inside the `open_history_client()` config allow
+            #    declaring the history duration limits instead of
+            #    guessing and/or applying the same limits to all?
+            #
+            # -[ ] allow declaring (default) per-provider backfill
+            #     limits inside a [storage] sub-section in conf.toml?
+            #
+            # NOTE, when no tsdb "last datum" is provided, we just
+            # load some near-term history by presuming a "decently
+            # large" 60s duration limit and a much shorter 1s range.
             periods = {
                 1: {'days': 2},
                 60: {'years': 6},
             }
             period_duration: int = periods[timeframe]
-            update_start_on_prepend = True
+            update_start_on_prepend: bool = True
 
             # NOTE: manually set the "latest" datetime which we intend to
             # backfill history "until" so as to adhere to the history
@@ -416,7 +421,6 @@ async def start_backfill(
                 f'backfill_until_dt: {backfill_until_dt}\n'
                 f'last_start_dt: {last_start_dt}\n'
             )
-
             try:
                 (
                     array,
@@ -426,71 +430,114 @@ async def start_backfill(
                     timeframe,
                     end_dt=last_start_dt,
                 )
-
             except NoData as _daterr:
-                # 3 cases:
-                # - frame in the middle of a legit venue gap
-                # - history actually began at the `last_start_dt`
-                # - some other unknown error (ib blocking the
-                #   history bc they don't want you seeing how they
-                #   cucked all the tinas..)
-                if dur := frame_types.get(timeframe):
-                    # decrement by a frame's worth of duration and
-                    # retry a few times.
-                    last_start_dt.subtract(
-                        seconds=dur.total_seconds()
+                orig_last_start_dt: datetime = last_start_dt
+                gap_report: str = (
+                    f'EMPTY FRAME for `end_dt: {last_start_dt}`?\n'
+                    f'{mod.name} -> tf@fqme: {timeframe}@{mkt.fqme}\n'
+                    f'last_start_dt: {orig_last_start_dt}\n\n'
+                    f'bf_until: {backfill_until_dt}\n'
+                )
+                # EMPTY FRAME signal with 3 (likely) causes:
+                #
+                # 1. range contains legit gap in venue history
+                # 2. history actually (edge case) **began** at the
+                #    value `last_start_dt`
+                # 3. some other unknown error (ib blocking the
+                #    history-query bc they don't want you seeing how
+                #    they cucked all the tinas.. like with options
+                #    hist)
+                #
+                if def_frame_duration:
+                    # decrement by a duration's (frame) worth of time
+                    # as maybe indicated by the backend to see if we
+                    # can get older data before this possible
+                    # "history gap".
+                    last_start_dt: datetime = last_start_dt.subtract(
+                        seconds=def_frame_duration.total_seconds()
                     )
-                    log.warning(
-                        f'{mod.name} -> EMPTY FRAME for end_dt?\n'
-                        f'tf@fqme: {timeframe}@{mkt.fqme}\n'
-                        'bf_until <- last_start_dt:\n'
-                        f'{backfill_until_dt} <- {last_start_dt}\n'
-                        f'Decrementing `end_dt` by {dur} and retry..\n'
+                    gap_report += (
+                        f'Decrementing `end_dt` and retrying with,\n'
+                        f'def_frame_duration: {def_frame_duration}\n'
+                        f'(new) last_start_dt: {last_start_dt}\n'
                     )
+                    log.warning(gap_report)
+                    # skip writing to shm/tsdb and try the next
+                    # duration's worth of prior history.
                     continue
 
-            # broker says there never was or is no more history to pull
-            except DataUnavailable:
-                log.warning(
-                    f'NO-MORE-DATA in range?\n'
-                    f'`{mod.name}` halted history:\n'
-                    f'tf@fqme: {timeframe}@{mkt.fqme}\n'
-                    'bf_until <- last_start_dt:\n'
-                    f'{backfill_until_dt} <- {last_start_dt}\n'
-                )
+                else:
+                    # await tractor.pause()
+                    raise DataUnavailable(gap_report)
 
-                # ugh, what's a better way?
-                # TODO: fwiw, we probably want a way to signal a throttle
-                # condition (eg. with ib) so that we can halt the
-                # request loop until the condition is resolved?
-                if timeframe > 1:
-                    await tractor.pause()
+            # broker says there never was or is no more history to pull
+            except DataUnavailable as due:
+                message: str = due.args[0]
+                log.warning(
+                    f'Provider {mod.name!r} halted backfill due to,\n\n'
+
+                    f'{message}\n'
+
+                    f'fqme: {mkt.fqme}\n'
+                    f'timeframe: {timeframe}\n'
+                    f'last_start_dt: {last_start_dt}\n'
+                    f'bf_until: {backfill_until_dt}\n'
+                )
+                # UGH: what's a better way?
+                # TODO: backends are responsible for being correct on
+                # this right!?
+                # -[ ] in the `ib` case we could maybe offer some way
+                #     to halt the request loop until the condition is
+                #     resolved or should the backend be entirely in
+                #     charge of solving such faults? yes, right?
                 return
 
+            time: np.ndarray = array['time']
             assert (
-                array['time'][0]
+                time[0]
                 ==
                 next_start_dt.timestamp()
             )
 
-            diff = last_start_dt - next_start_dt
-            frame_time_diff_s = diff.seconds
+            assert time[-1] == next_end_dt.timestamp()
+
+            expected_dur: Interval = last_start_dt - next_start_dt
 
             # frame's worth of sample-period-steps, in seconds
             frame_size_s: float = len(array) * timeframe
-            expected_frame_size_s: float = frame_size_s + timeframe
-            if frame_time_diff_s > expected_frame_size_s:
-
+            recv_frame_dur: Duration = (
+                from_timestamp(array[-1]['time'])
+                -
+                from_timestamp(array[0]['time'])
+            )
+            if (
+                (lt_frame := (recv_frame_dur < expected_dur))
+                or
+                (null_frame := (frame_size_s == 0))
+                # ^XXX, should NEVER hit now!
+            ):
                 # XXX: query result includes a start point prior to our
                 # expected "frame size" and thus is likely some kind of
                 # history gap (eg. market closed period, outage, etc.)
                 # so just report it to console for now.
+                if lt_frame:
+                    reason = 'Possible GAP (or first-datum)'
+                else:
+                    assert null_frame
+                    reason = 'NULL-FRAME'
+
+                missing_dur: Interval = expected_dur.end - recv_frame_dur.end
                 log.warning(
-                    'GAP DETECTED:\n'
-                    f'last_start_dt: {last_start_dt}\n'
-                    f'diff: {diff}\n'
-                    f'frame_time_diff_s: {frame_time_diff_s}\n'
+                    f'{timeframe}s-series {reason} detected!\n'
+                    f'fqme: {mkt.fqme}\n'
+                    f'last_start_dt: {last_start_dt}\n\n'
+                    f'recv interval: {recv_frame_dur}\n'
+                    f'expected interval: {expected_dur}\n\n'
+
+                    f'Missing duration of history of {missing_dur.in_words()!r}\n'
+                    f'{missing_dur}\n'
                 )
+                # await tractor.pause()
 
             to_push = diff_history(
                 array,
@@ -565,22 +612,27 @@ async def start_backfill(
             # long-term storage.
             if (
                 storage is not None
-                and write_tsdb
+                and
+                write_tsdb
             ):
                 log.info(
                     f'Writing {ln} frame to storage:\n'
                     f'{next_start_dt} -> {last_start_dt}'
                 )
 
-                # always drop the src asset token for
+                # NOTE, always drop the src asset token for
                 # non-currency-pair like market types (for now)
+                #
+                # THAT IS, for now our table key schema is NOT
+                # including the dst[/src] source asset token. SO,
+                # 'tsla.nasdaq.ib' over 'tsla/usd.nasdaq.ib' for
+                # historical reasons ONLY.
                 if mkt.dst.atype not in {
                     'crypto',
                     'crypto_currency',
                     'fiat',  # a "forex pair"
+                    'perpetual_future',  # stupid "perps" from cex land
                 }:
-                    # for now, our table key schema is not including
-                    # the dst[/src] source asset token.
                     col_sym_key: str = mkt.get_fqme(
                         delim_char='',
                         without_src=True,
@@ -685,7 +737,7 @@ async def back_load_from_tsdb(
         last_tsdb_dt
         and latest_start_dt
     ):
-        backfilled_size_s = (
+        backfilled_size_s: Duration = (
             latest_start_dt - last_tsdb_dt
         ).seconds
         # if the shm buffer len is not large enough to contain
@@ -908,6 +960,8 @@ async def tsdb_backfill(
             f'{pformat(config)}\n'
         )
 
+        # concurrently load the provider's most-recent-frame AND any
+        # pre-existing tsdb history already saved in `piker` storage.
         dt_eps: list[DateTime, DateTime] = []
         async with trio.open_nursery() as tn:
             tn.start_soon(
@@ -918,7 +972,6 @@ async def tsdb_backfill(
                 timeframe,
                 config,
             )
-
             tsdb_entry: tuple = await load_tsdb_hist(
                 storage,
                 mkt,
@@ -947,6 +1000,25 @@ async def tsdb_backfill(
                 mr_end_dt,
             ) = dt_eps
 
+            first_frame_dur_s: Duration = (mr_end_dt - mr_start_dt).seconds
+            calced_frame_size: Duration = mk_duration(
+                seconds=first_frame_dur_s,
+            )
+            # NOTE, attempt to use the backend declared default frame
+            # sizing (as allowed by their time-series query APIs) and
+            # if not provided try to construct a default from the
+            # first frame received above.
+            def_frame_durs: dict[
+                int,
+                Duration,
+            ]|None = config.get('frame_types', None)
+            if def_frame_durs:
+                def_frame_size: Duration = def_frame_durs[timeframe]
+                assert def_frame_size == calced_frame_size
+            else:
+                # use what we calced from first frame above.
+                def_frame_size = calced_frame_size
+
             # NOTE: when there's no offline data, there's 2 cases:
             # - data backend doesn't support timeframe/sample
             #   period (in which case `dt_eps` should be `None` and
@@ -977,7 +1049,7 @@ async def tsdb_backfill(
                     partial(
                         start_backfill,
                         get_hist=get_hist,
-                        frame_types=config.get('frame_types', None),
+                        def_frame_duration=def_frame_size,
                         mod=mod,
                         mkt=mkt,
                         shm=shm,

piker/tsp/_anal.py

@@ -616,6 +616,18 @@ def detect_price_gaps(
     # ])
     ...
 
+# TODO: probably just use the null_segs impl above?
+def detect_vlm_gaps(
+    df: pl.DataFrame,
+    col: str = 'volume',
+
+) -> pl.DataFrame:
+
+    vnull: pl.DataFrame = w_dts.filter(
+        pl.col(col) == 0
+    )
+    return vnull
+
 
 def dedupe(
     src_df: pl.DataFrame,
@@ -626,7 +638,6 @@ def dedupe(
 
 ) -> tuple[
     pl.DataFrame,  # with dts
-    pl.DataFrame,  # gaps
     pl.DataFrame,  # with deduplicated dts (aka gap/repeat removal)
     int,  # len diff between input and deduped
 ]:
@@ -639,19 +650,22 @@ def dedupe(
     '''
     wdts: pl.DataFrame = with_dts(src_df)
 
-    # maybe sort on any time field
-    if sort:
-        wdts = wdts.sort(by='time')
-        # TODO: detect out-of-order segments which were corrected!
-        # -[ ] report in log msg
-        # -[ ] possibly return segment sections which were moved?
+    deduped = wdts
 
     # remove duplicated datetime samples/sections
     deduped: pl.DataFrame = wdts.unique(
-        subset=['dt'],
+        # subset=['dt'],
+        subset=['time'],
         maintain_order=True,
     )
 
+    # maybe sort on any time field
+    if sort:
+        deduped = deduped.sort(by='time')
+        # TODO: detect out-of-order segments which were corrected!
+        # -[ ] report in log msg
+        # -[ ] possibly return segment sections which were moved?
+
     diff: int = (
         wdts.height
         -

piker/types.py

@@ -21,230 +21,4 @@ Extensions to built-in or (heavily used but 3rd party) friend-lib
 types.
 
 '''
-from __future__ import annotations
-from collections import UserList
-from pprint import (
-    saferepr,
-)
-from typing import Any
-
-from msgspec import (
-    msgpack,
-    Struct as _Struct,
-    structs,
-)
-
-
-class DiffDump(UserList):
-    '''
-    Very simple list delegator that repr() dumps (presumed) tuple
-    elements of the form `tuple[str, Any, Any]` in a nice
-    multi-line readable form for analyzing `Struct` diffs.
-
-    '''
-    def __repr__(self) -> str:
-        if not len(self):
-            return super().__repr__()
-
-        # format by displaying item pair's ``repr()`` on multiple,
-        # indented lines such that they are more easily visually
-        # comparable when printed to console when printed to
-        # console.
-        repstr: str = '[\n'
-        for k, left, right in self:
-            repstr += (
-                f'({k},\n'
-                f'\t{repr(left)},\n'
-                f'\t{repr(right)},\n'
-                ')\n'
-            )
-        repstr += ']\n'
-        return repstr
-
-
-class Struct(
-    _Struct,
-
-    # https://jcristharif.com/msgspec/structs.html#tagged-unions
-    # tag='pikerstruct',
-    # tag=True,
-):
-    '''
-    A "human friendlier" (aka repl buddy) struct subtype.
-
-    '''
-    def _sin_props(self) -> Iterator[
-        tuple[
-            structs.FieldIinfo,
-            str,
-            Any,
-        ]
-    ]:
-        '''
-        Iterate over all non-@property fields of this struct.
-
-        '''
-        fi: structs.FieldInfo
-        for fi in structs.fields(self):
-            key: str = fi.name
-            val: Any = getattr(self, key)
-            yield fi, key, val
-
-    def to_dict(
-        self,
-        include_non_members: bool = True,
-
-    ) -> dict:
-        '''
-        Like it sounds.. direct delegation to:
-        https://jcristharif.com/msgspec/api.html#msgspec.structs.asdict
-
-        BUT, by default we pop all non-member (aka not defined as
-        struct fields) fields by default.
-
-        '''
-        asdict: dict = structs.asdict(self)
-        if include_non_members:
-            return asdict
-
-        # only return a dict of the struct members
-        # which were provided as input, NOT anything
-        # added as type-defined `@property` methods!
-        sin_props: dict = {}
-        fi: structs.FieldInfo
-        for fi, k, v in self._sin_props():
-            sin_props[k] = asdict[k]
-
-        return sin_props
-
-    def pformat(
-        self,
-        field_indent: int = 2,
-        indent: int = 0,
-
-    ) -> str:
-        '''
-        Recursion-safe `pprint.pformat()` style formatting of
-        a `msgspec.Struct` for sane reading by a human using a REPL.
-
-        '''
-        # global whitespace indent
-        ws: str = ' '*indent
-
-        # field whitespace indent
-        field_ws: str = ' '*(field_indent + indent)
-
-        # qtn: str = ws + self.__class__.__qualname__
-        qtn: str = self.__class__.__qualname__
-
-        obj_str: str = ''  # accumulator
-        fi: structs.FieldInfo
-        k: str
-        v: Any
-        for fi, k, v in self._sin_props():
-
-            # TODO: how can we prefer `Literal['option1',  'option2,
-            # ..]` over .__name__ == `Literal` but still get only the
-            # latter for simple types like `str | int | None` etc..?
-            ft: type = fi.type
-            typ_name: str = getattr(ft, '__name__', str(ft))
-
-            # recurse to get sub-struct's `.pformat()` output Bo
-            if isinstance(v, Struct):
-                val_str: str =  v.pformat(
-                    indent=field_indent + indent,
-                    field_indent=indent + field_indent,
-                )
-
-            else:  # the `pprint` recursion-safe format:
-                # https://docs.python.org/3.11/library/pprint.html#pprint.saferepr
-                val_str: str = saferepr(v)
-
-            obj_str += (field_ws + f'{k}: {typ_name} = {val_str},\n')
-
-        return (
-            f'{qtn}(\n'
-            f'{obj_str}'
-            f'{ws})'
-        )
-
-    # TODO: use a pprint.PrettyPrinter instance around ONLY rendering
-    # inside a known tty?
-    # def __repr__(self) -> str:
-    #     ...
-
-    # __str__ = __repr__ = pformat
-    __repr__ = pformat
-
-    def copy(
-        self,
-        update: dict | None = None,
-
-    ) -> Struct:
-        '''
-        Validate-typecast all self defined fields, return a copy of
-        us with all such fields.
-
-        NOTE: This is kinda like the default behaviour in
-        `pydantic.BaseModel` except a copy of the object is
-        returned making it compat with `frozen=True`.
-
-        '''
-        if update:
-            for k, v in update.items():
-                setattr(self, k, v)
-
-        # NOTE: roundtrip serialize to validate
-        # - enode to msgpack binary format,
-        # - decode that back to a struct.
-        return msgpack.Decoder(type=type(self)).decode(
-            msgpack.Encoder().encode(self)
-        )
-
-    def typecast(
-        self,
-
-        # TODO: allow only casting a named subset?
-        # fields: set[str] | None = None,
-
-    ) -> None:
-        '''
-        Cast all fields using their declared type annotations
-        (kinda like what `pydantic` does by default).
-
-        NOTE: this of course won't work on frozen types, use
-        ``.copy()`` above in such cases.
-
-        '''
-        # https://jcristharif.com/msgspec/api.html#msgspec.structs.fields
-        fi: structs.FieldInfo
-        for fi in structs.fields(self):
-            setattr(
-                self,
-                fi.name,
-                fi.type(getattr(self, fi.name)),
-            )
-
-    def __sub__(
-        self,
-        other: Struct,
-
-    ) -> DiffDump[tuple[str, Any, Any]]:
-        '''
-        Compare fields/items key-wise and return a ``DiffDump``
-        for easy visual REPL comparison B)
-
-        '''
-        diffs: DiffDump[tuple[str, Any, Any]] = DiffDump()
-        for fi in structs.fields(self):
-            attr_name: str = fi.name
-            ours: Any = getattr(self, attr_name)
-            theirs: Any = getattr(other, attr_name)
-            if ours != theirs:
-                diffs.append((
-                    attr_name,
-                    ours,
-                    theirs,
-                ))
-
-        return diffs
+from tractor.msg import Struct as Struct

pyproject.toml

@@ -18,22 +18,6 @@
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
-# ------ - ------
-
-[tool.ruff.lint]
-# https://docs.astral.sh/ruff/settings/#lint_ignore
-ignore = []
-
-# https://docs.astral.sh/ruff/settings/#lint_per-file-ignores
-"piker/ui/qt.py" = [
-  "E402",
-  'F401',  # unused imports (without __all__ or blah as blah)
-  # "F841", # unused variable rules
-]
-# ignore-init-module-imports = false
-
-# ------ - ------
-
 [project]
 name = "piker"
 version = "0.1.0a0dev0"
@@ -103,15 +87,23 @@ uis = [
     "pyqt6 >=6.7.0, <7.0.0",
     "pyqtgraph",
 
-    # ------ - ------
+    # for consideration,
+    # - 'visidata'
 
     # TODO: add an `--only daemon` group for running non-ui / pikerd
     # service tree in distributed mode B)
     # https://docs.astral.sh/uv/concepts/projects/dependencies/#optional-dependencies
-    # [project.optional-dependencies]
 ]
 
 [dependency-groups]
+# TODO: a toolset that makes debugging a `pikerd` service (tree) easy
+# to hack on directly using more or less the local env:
+# - xonsh + xxh
+# - rsyscall + pdbp
+# - actor runtime control console like BEAM/OTP
+#
+# console ehancements and eventually remote debugging extras/helpers.
+# use `uv --dev` to enable
 dev = [
     "pytest >=6.0.0, <7.0.0",
     "elasticsearch >=8.9.0, <9.0.0",
@@ -119,13 +111,7 @@ dev = [
     "prompt-toolkit ==3.0.40",
     "cython >=3.0.0, <4.0.0",
     "greenback >=1.1.1, <2.0.0",
-    # console ehancements and eventually remote debugging
-    # extras/helpers.
-    # TODO: add a toolset that makes debugging a `pikerd` service
-    # (tree) easy to hack on directly using more or less the local env:
-    # - xonsh + xxh
-    # - rsyscall + pdbp
-    # - actor runtime control console like BEAM/OTP
+    "ruff>=0.9.6",
 ]
 
 [project.scripts]

ruff.toml

@@ -0,0 +1,93 @@
+# from default `ruff.toml` @
+# https://docs.astral.sh/ruff/configuration/
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+]
+
+# Same as Black.
+line-length = 88
+indent-width = 4
+
+# Assume Python 3.9
+target-version = "py312"
+
+# ------ - ------
+# TODO, stop warnings around `anext()` builtin use?
+# tool.ruff.target-version = "py310"
+
+
+[lint]
+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E4", "E7", "E9", "F"]
+ignore = []
+ignore-init-module-imports = false
+
+[lint.per-file-ignores]
+"piker/ui/qt.py" = [
+  "E402",
+  'F401',  # unused imports (without __all__ or blah as blah)
+  # "F841", # unused variable rules
+]
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[format]
+# Use single quotes in `ruff format`.
+quote-style = "single"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"
+
+# Enable auto-formatting of code examples in docstrings. Markdown,
+# reStructuredText code/literal blocks and doctests are all supported.
+#
+# This is currently disabled by default, but it is planned for this
+# to be opt-out in the future.
+docstring-code-format = false
+
+# Set the line length limit used when formatting code snippets in
+# docstrings.
+#
+# This only has an effect when the `docstring-code-format` setting is
+# enabled.
+docstring-code-line-length = "dynamic"

uv.lock

@@ -708,6 +708,7 @@ dev = [
     { name = "greenback" },
     { name = "prompt-toolkit" },
     { name = "pytest" },
+    { name = "ruff" },
     { name = "xonsh" },
 ]
 
@@ -739,7 +740,7 @@ requires-dist = [
     { name = "tomli", specifier = ">=2.0.1,<3.0.0" },
     { name = "tomli-w", specifier = ">=1.0.0,<2.0.0" },
     { name = "tomlkit", git = "https://github.com/pikers/tomlkit.git?branch=piker_pin" },
-    { name = "tractor", directory = "../tractor" },
+    { name = "tractor", editable = "../tractor" },
     { name = "trio", specifier = ">=0.24,<0.25" },
     { name = "trio-util", specifier = ">=0.7.0,<0.8.0" },
     { name = "trio-websocket", specifier = ">=0.10.3,<0.11.0" },
@@ -754,6 +755,7 @@ dev = [
     { name = "greenback", specifier = ">=1.1.1,<2.0.0" },
     { name = "prompt-toolkit", specifier = "==3.0.40" },
     { name = "pytest", specifier = ">=6.0.0,<7.0.0" },
+    { name = "ruff", specifier = ">=0.9.6" },
     { name = "xonsh", specifier = ">=0.14.2,<0.15.0" },
 ]
 
@@ -1073,6 +1075,31 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/19/71/39c7c0d87f8d4e6c020a393182060eaefeeae6c01dab6a84ec346f2567df/rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90", size = 242424 },
 ]
 
+[[package]]
+name = "ruff"
+version = "0.9.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2a/e1/e265aba384343dd8ddd3083f5e33536cd17e1566c41453a5517b5dd443be/ruff-0.9.6.tar.gz", hash = "sha256:81761592f72b620ec8fa1068a6fd00e98a5ebee342a3642efd84454f3031dca9", size = 3639454 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/76/e3/3d2c022e687e18cf5d93d6bfa2722d46afc64eaa438c7fbbdd603b3597be/ruff-0.9.6-py3-none-linux_armv6l.whl", hash = "sha256:2f218f356dd2d995839f1941322ff021c72a492c470f0b26a34f844c29cdf5ba", size = 11714128 },
+    { url = "https://files.pythonhosted.org/packages/e1/22/aff073b70f95c052e5c58153cba735748c9e70107a77d03420d7850710a0/ruff-0.9.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b908ff4df65dad7b251c9968a2e4560836d8f5487c2f0cc238321ed951ea0504", size = 11682539 },
+    { url = "https://files.pythonhosted.org/packages/75/a7/f5b7390afd98a7918582a3d256cd3e78ba0a26165a467c1820084587cbf9/ruff-0.9.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b109c0ad2ececf42e75fa99dc4043ff72a357436bb171900714a9ea581ddef83", size = 11132512 },
+    { url = "https://files.pythonhosted.org/packages/a6/e3/45de13ef65047fea2e33f7e573d848206e15c715e5cd56095589a7733d04/ruff-0.9.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1de4367cca3dac99bcbd15c161404e849bb0bfd543664db39232648dc00112dc", size = 11929275 },
+    { url = "https://files.pythonhosted.org/packages/7d/f2/23d04cd6c43b2e641ab961ade8d0b5edb212ecebd112506188c91f2a6e6c/ruff-0.9.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac3ee4d7c2c92ddfdaedf0bf31b2b176fa7aa8950efc454628d477394d35638b", size = 11466502 },
+    { url = "https://files.pythonhosted.org/packages/b5/6f/3a8cf166f2d7f1627dd2201e6cbc4cb81f8b7d58099348f0c1ff7b733792/ruff-0.9.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5dc1edd1775270e6aa2386119aea692039781429f0be1e0949ea5884e011aa8e", size = 12676364 },
+    { url = "https://files.pythonhosted.org/packages/f5/c4/db52e2189983c70114ff2b7e3997e48c8318af44fe83e1ce9517570a50c6/ruff-0.9.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:4a091729086dffa4bd070aa5dab7e39cc6b9d62eb2bef8f3d91172d30d599666", size = 13335518 },
+    { url = "https://files.pythonhosted.org/packages/66/44/545f8a4d136830f08f4d24324e7db957c5374bf3a3f7a6c0bc7be4623a37/ruff-0.9.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1bbc6808bf7b15796cef0815e1dfb796fbd383e7dbd4334709642649625e7c5", size = 12823287 },
+    { url = "https://files.pythonhosted.org/packages/c5/26/8208ef9ee7431032c143649a9967c3ae1aae4257d95e6f8519f07309aa66/ruff-0.9.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:589d1d9f25b5754ff230dce914a174a7c951a85a4e9270613a2b74231fdac2f5", size = 14592374 },
+    { url = "https://files.pythonhosted.org/packages/31/70/e917781e55ff39c5b5208bda384fd397ffd76605e68544d71a7e40944945/ruff-0.9.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc61dd5131742e21103fbbdcad683a8813be0e3c204472d520d9a5021ca8b217", size = 12500173 },
+    { url = "https://files.pythonhosted.org/packages/84/f5/e4ddee07660f5a9622a9c2b639afd8f3104988dc4f6ba0b73ffacffa9a8c/ruff-0.9.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5e2d9126161d0357e5c8f30b0bd6168d2c3872372f14481136d13de9937f79b6", size = 11906555 },
+    { url = "https://files.pythonhosted.org/packages/f1/2b/6ff2fe383667075eef8656b9892e73dd9b119b5e3add51298628b87f6429/ruff-0.9.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:68660eab1a8e65babb5229a1f97b46e3120923757a68b5413d8561f8a85d4897", size = 11538958 },
+    { url = "https://files.pythonhosted.org/packages/3c/db/98e59e90de45d1eb46649151c10a062d5707b5b7f76f64eb1e29edf6ebb1/ruff-0.9.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c4cae6c4cc7b9b4017c71114115db0445b00a16de3bcde0946273e8392856f08", size = 12117247 },
+    { url = "https://files.pythonhosted.org/packages/ec/bc/54e38f6d219013a9204a5a2015c09e7a8c36cedcd50a4b01ac69a550b9d9/ruff-0.9.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:19f505b643228b417c1111a2a536424ddde0db4ef9023b9e04a46ed8a1cb4656", size = 12554647 },
+    { url = "https://files.pythonhosted.org/packages/a5/7d/7b461ab0e2404293c0627125bb70ac642c2e8d55bf590f6fce85f508f1b2/ruff-0.9.6-py3-none-win32.whl", hash = "sha256:194d8402bceef1b31164909540a597e0d913c0e4952015a5b40e28c146121b5d", size = 9949214 },
+    { url = "https://files.pythonhosted.org/packages/ee/30/c3cee10f915ed75a5c29c1e57311282d1a15855551a64795c1b2bbe5cf37/ruff-0.9.6-py3-none-win_amd64.whl", hash = "sha256:03482d5c09d90d4ee3f40d97578423698ad895c87314c4de39ed2af945633caa", size = 10999914 },
+    { url = "https://files.pythonhosted.org/packages/e8/a8/d71f44b93e3aa86ae232af1f2126ca7b95c0f515ec135462b3e1f351441c/ruff-0.9.6-py3-none-win_arm64.whl", hash = "sha256:0e2bb706a2be7ddfea4a4af918562fdc1bcb16df255e5fa595bbd800ce322a5a", size = 10177499 },
+]
+
 [[package]]
 name = "shellingham"
 version = "1.5.4"
@@ -1188,7 +1215,7 @@ source = { git = "https://github.com/pikers/tomlkit.git?branch=piker_pin#8e0239a
 [[package]]
 name = "tractor"
 version = "0.1.0a6.dev0"
-source = { directory = "../tractor" }
+source = { editable = "../tractor" }
 dependencies = [
     { name = "colorlog" },
     { name = "msgspec" },