`.brokers.cli`: module type and todo for `--pdb` flag to NOT src from sub-cmd

- use more compact optional value style with `|`-union
- fix `.flows` typing-only import since we need `MktPair` to be
  immediately defined for use on a `msgspec.Struct` field.
- more "tree-like" warning msg in `.validate()` reporting.

piker/clearing/_client.py

@@ -168,6 +168,7 @@ class OrderClient(Struct):
 
 
 async def relay_orders_from_sync_code(
+
     client: OrderClient,
     symbol_key: str,
     to_ems_stream: tractor.MsgStream,
@@ -241,11 +242,6 @@ async def open_ems(
 
     async with maybe_open_emsd(
         broker,
-        # XXX NOTE, LOL so this determines the daemon `emsd` loglevel
-        # then FYI.. that's kinda wrong no?
-        # -[ ] shouldn't it be set by `pikerd -l` or no?
-        # -[ ] would make a lot more sense to have a subsys ctl for
-        #     levels.. like `-l emsd.info` or something?
         loglevel=loglevel,
     ) as portal:
 

piker/clearing/_ems.py

@@ -653,11 +653,7 @@ class Router(Struct):
             flume = feed.flumes[fqme]
             first_quote: dict = flume.first_quote
             book: DarkBook = self.get_dark_book(broker)
-
-            if not (last := first_quote.get('last')):
-                last: float = flume.rt_shm.array[-1]['close']
-
-            book.lasts[fqme]: float = float(last)
+            book.lasts[fqme]: float = float(first_quote['last'])
 
             async with self.maybe_open_brokerd_dialog(
                 brokermod=brokermod,
@@ -720,7 +716,7 @@ class Router(Struct):
             subs = self.subscribers[sub_key]
 
         sent_some: bool = False
-        for client_stream in subs.copy():
+        for client_stream in subs:
             try:
                 await client_stream.send(msg)
                 sent_some = True
@@ -1014,14 +1010,10 @@ async def translate_and_relay_brokerd_events(
                 status_msg.brokerd_msg = msg
                 status_msg.src = msg.broker_details['name']
 
-                if not status_msg.req:
-                    # likely some order change state?
-                    await tractor.pause()
-                else:
-                    await router.client_broadcast(
-                        status_msg.req.symbol,
-                        status_msg,
-                    )
+                await router.client_broadcast(
+                    status_msg.req.symbol,
+                    status_msg,
+                )
 
                 if status == 'closed':
                     log.info(f'Execution for {oid} is complete!')

piker/clearing/_paper_engine.py

@@ -297,8 +297,6 @@ class PaperBoi(Struct):
 
         # transmit pp msg to ems
         pp: Position = self.acnt.pps[bs_mktid]
-        # TODO, this will break if `require_only=True` was passed to
-        # `.update_from_ledger()`
 
         pp_msg = BrokerdPosition(
             broker=self.broker,

piker/clearing/_util.py

@@ -30,7 +30,6 @@ subsys: str = 'piker.clearing'
 
 log = get_logger(subsys)
 
-# TODO, oof doesn't this ignore the `loglevel` then???
 get_console_log = partial(
     get_console_log,
     name=subsys,

piker/data/_sampling.py

@@ -95,12 +95,6 @@ class Sampler:
     # history loading.
     incr_task_cs: trio.CancelScope | None = None
 
-    bcast_errors: tuple[Exception] = (
-        trio.BrokenResourceError,
-        trio.ClosedResourceError,
-        trio.EndOfChannel,
-    )
-
     # holds all the ``tractor.Context`` remote subscriptions for
     # a particular sample period increment event: all subscribers are
     # notified on a step.
@@ -264,15 +258,14 @@ class Sampler:
         subs: set
         last_ts, subs = pair
 
-        # NOTE, for debugging pub-sub issues
-        # task = trio.lowlevel.current_task()
-        # log.debug(
-        #     f'AlL-SUBS@{period_s!r}: {self.subscribers}\n'
-        #     f'PAIR: {pair}\n'
-        #     f'TASK: {task}: {id(task)}\n'
-        #     f'broadcasting {period_s} -> {last_ts}\n'
-        #     f'consumers: {subs}'
-        # )
+        task = trio.lowlevel.current_task()
+        log.debug(
+            f'SUBS {self.subscribers}\n'
+            f'PAIR {pair}\n'
+            f'TASK: {task}: {id(task)}\n'
+            f'broadcasting {period_s} -> {last_ts}\n'
+            # f'consumers: {subs}'
+        )
         borked: set[MsgStream] = set()
         sent: set[MsgStream] = set()
         while True:
@@ -289,11 +282,12 @@ class Sampler:
                         await stream.send(msg)
                         sent.add(stream)
 
-                    except self.bcast_errors as err:
+                    except (
+                        trio.BrokenResourceError,
+                        trio.ClosedResourceError
+                    ):
                         log.error(
-                            f'Connection dropped for IPC ctx\n'
-                            f'{stream._ctx}\n\n'
-                            f'Due to {type(err)}'
+                            f'{stream._ctx.chan.uid} dropped connection'
                         )
                         borked.add(stream)
                 else:
@@ -400,8 +394,7 @@ async def register_with_sampler(
                 finally:
                     if (
                         sub_for_broadcasts
-                        and
-                        subs
+                        and subs
                     ):
                         try:
                             subs.remove(stream)
@@ -568,7 +561,8 @@ async def open_sample_stream(
 
 
 async def sample_and_broadcast(
-    bus: _FeedsBus,
+
+    bus: _FeedsBus,  # noqa
     rt_shm: ShmArray,
     hist_shm: ShmArray,
     quote_stream: trio.abc.ReceiveChannel,
@@ -588,33 +582,11 @@ async def sample_and_broadcast(
 
     overruns = Counter()
 
-    # NOTE, only used for debugging live-data-feed issues, though
-    # this should be resolved more correctly in the future using the
-    # new typed-msgspec feats of `tractor`!
-    #
-    # XXX, a multiline nested `dict` formatter (since rn quote-msgs
-    # are just that).
-    # pfmt: Callable[[str], str] = mk_repr()
-
     # iterate stream delivered by broker
     async for quotes in quote_stream:
         # print(quotes)
 
-        # XXX WARNING XXX only enable for debugging bc ow can cost
-        # ALOT of perf with HF-feedz!!!
-        #
-        # log.info(
-        #     'Rx live quotes:\n'
-        #     f'{pfmt(quotes)}'
-        # )
-
-        # TODO,
-        # -[ ] `numba` or `cython`-nize this loop possibly?
-        #  |_alternatively could we do it in rust somehow by upacking
-        #    arrow msgs instead of using `msgspec`?
-        # -[ ] use `msgspec.Struct` support in new typed-msging from
-        #     `tractor` to ensure only allowed msgs are transmitted?
-        #
+        # TODO: ``numba`` this!
         for broker_symbol, quote in quotes.items():
             # TODO: in theory you can send the IPC msg *before* writing
             # to the sharedmem array to decrease latency, however, that
@@ -687,21 +659,6 @@ async def sample_and_broadcast(
             sub_key: str = broker_symbol.lower()
             subs: set[Sub] = bus.get_subs(sub_key)
 
-            # TODO, figure out how to make this useful whilst
-            # incoporating feed "pausing" ..
-            #
-            # if not subs:
-            #     all_bs_fqmes: list[str] = list(
-            #         bus._subscribers.keys()
-            #     )
-            #     log.warning(
-            #         f'No subscribers for {brokername!r} live-quote ??\n'
-            #         f'broker_symbol: {broker_symbol}\n\n'
-
-            #         f'Maybe the backend-sys symbol does not match one of,\n'
-            #         f'{pfmt(all_bs_fqmes)}\n'
-            #     )
-
             # NOTE: by default the broker backend doesn't append
             # it's own "name" into the fqme schema (but maybe it
             # should?) so we have to manually generate the correct
@@ -771,14 +728,18 @@ async def sample_and_broadcast(
                         if lags > 10:
                             await tractor.pause()
 
-                except Sampler.bcast_errors as ipc_err:
+                except (
+                    trio.BrokenResourceError,
+                    trio.ClosedResourceError,
+                    trio.EndOfChannel,
+                ):
                     ctx: Context = ipc._ctx
                     chan: Channel = ctx.chan
                     if ctx:
                         log.warning(
-                            f'Dropped `brokerd`-feed for {broker_symbol!r} due to,\n'
-                            f'x>) {ctx.cid}@{chan.uid}'
-                            f'|_{ipc_err!r}\n\n'
+                            'Dropped `brokerd`-quotes-feed connection:\n'
+                            f'{broker_symbol}:'
+                            f'{ctx.cid}@{chan.uid}'
                         )
                     if sub.throttle_rate:
                         assert ipc._closed
@@ -795,11 +756,12 @@ async def sample_and_broadcast(
 
 
 async def uniform_rate_send(
+
     rate: float,
     quote_stream: trio.abc.ReceiveChannel,
     stream: MsgStream,
 
-    task_status: TaskStatus[None] = trio.TASK_STATUS_IGNORED,
+    task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
 
 ) -> None:
     '''
@@ -817,16 +779,13 @@ async def uniform_rate_send(
     https://gist.github.com/njsmith/7ea44ec07e901cb78ebe1dd8dd846cb9
 
     '''
-    # ?TODO? dynamically compute the **actual** approx overhead latency per cycle
-    # instead of this magic # bidinezz?
-    throttle_period: float = 1/rate - 0.000616
-    left_to_sleep: float = throttle_period
+    # TODO: compute the approx overhead latency per cycle
+    left_to_sleep = throttle_period = 1/rate - 0.000616
 
     # send cycle state
-    first_quote: dict|None
     first_quote = last_quote = None
-    last_send: float = time.time()
-    diff: float = 0
+    last_send = time.time()
+    diff = 0
 
     task_status.started()
     ticks_by_type: dict[
@@ -837,28 +796,22 @@ async def uniform_rate_send(
     clear_types = _tick_groups['clears']
 
     while True:
+
         # compute the remaining time to sleep for this throttled cycle
-        left_to_sleep: float = throttle_period - diff
+        left_to_sleep = throttle_period - diff
 
         if left_to_sleep > 0:
-            cs: trio.CancelScope
             with trio.move_on_after(left_to_sleep) as cs:
-                sym: str
-                last_quote: dict
                 try:
                     sym, last_quote = await quote_stream.receive()
                 except trio.EndOfChannel:
-                    log.exception(
-                        f'Live stream for feed for ended?\n'
-                        f'<=c\n'
-                        f'  |_[{stream!r}\n'
-                    )
+                    log.exception(f"feed for {stream} ended?")
                     break
 
-                diff: float = time.time() - last_send
+                diff = time.time() - last_send
 
                 if not first_quote:
-                    first_quote: float = last_quote
+                    first_quote = last_quote
                     # first_quote['tbt'] = ticks_by_type
 
                 if (throttle_period - diff) > 0:
@@ -919,9 +872,7 @@ async def uniform_rate_send(
         # TODO: now if only we could sync this to the display
         # rate timing exactly lul
         try:
-            await stream.send({
-                sym: first_quote
-            })
+            await stream.send({sym: first_quote})
         except tractor.RemoteActorError as rme:
             if rme.type is not tractor._exceptions.StreamOverrun:
                 raise
@@ -932,28 +883,19 @@ async def uniform_rate_send(
                 f'{sym}:{ctx.cid}@{chan.uid}'
             )
 
-        # NOTE: any of these can be raised by `tractor`'s IPC
-        # transport-layer and we want to be highly resilient
-        # to consumers which crash or lose network connection.
-        # I.e. we **DO NOT** want to crash and propagate up to
-        # ``pikerd`` these kinds of errors!
         except (
+            # NOTE: any of these can be raised by ``tractor``'s IPC
+            # transport-layer and we want to be highly resilient
+            # to consumers which crash or lose network connection.
+            # I.e. we **DO NOT** want to crash and propagate up to
+            # ``pikerd`` these kinds of errors!
+            trio.ClosedResourceError,
+            trio.BrokenResourceError,
             ConnectionResetError,
-        ) + Sampler.bcast_errors as ipc_err:
-            match ipc_err:
-                case trio.EndOfChannel():
-                    log.info(
-                        f'{stream} terminated by peer,\n'
-                        f'{ipc_err!r}'
-                    )
-                case _:
-                    # if the feed consumer goes down then drop
-                    # out of this rate limiter
-                    log.warning(
-                        f'{stream} closed due to,\n'
-                        f'{ipc_err!r}'
-                    )
-
+        ):
+            # if the feed consumer goes down then drop
+            # out of this rate limiter
+            log.warning(f'{stream} closed')
             await stream.aclose()
             return
 

piker/log.py

@@ -19,10 +19,6 @@ Log like a forester!
 """
 import logging
 import json
-import reprlib
-from typing import (
-    Callable,
-)
 
 import tractor
 from pygments import (
@@ -88,29 +84,3 @@ def colorize_json(
         # likeable styles: algol_nu, tango, monokai
         formatters.TerminalTrueColorFormatter(style=style)
     )
-
-
-# TODO, eventually defer to the version in `modden` once
-# it becomes a dep!
-def mk_repr(
-    **repr_kws,
-) -> Callable[[str], str]:
-    '''
-    Allocate and deliver a `repr.Repr` instance with provided input
-    settings using the std-lib's `reprlib` mod,
-     * https://docs.python.org/3/library/reprlib.html
-
-    ------ Ex. ------
-    An up to 6-layer-nested `dict` as multi-line:
-    - https://stackoverflow.com/a/79102479
-    - https://docs.python.org/3/library/reprlib.html#reprlib.Repr.maxlevel
-
-    '''
-    def_kws: dict[str, int] = dict(
-        indent=2,
-        maxlevel=6,  # recursion levels
-        maxstring=66,  # match editor line-len limit
-    )
-    def_kws |= repr_kws
-    reprr = reprlib.Repr(**def_kws)
-    return reprr.repr

piker/service/_actor_runtime.py

@@ -119,10 +119,6 @@ async def open_piker_runtime(
                 # spawn other specialized daemons I think?
                 enable_modules=enable_modules,
 
-                # TODO: how to configure this?
-                # keep it on by default if debug mode is set?
-                maybe_enable_greenback=False,
-
                 **tractor_kwargs,
             ) as actor,
 

piker/types.py

@@ -21,4 +21,230 @@ Extensions to built-in or (heavily used but 3rd party) friend-lib
 types.
 
 '''
-from tractor.msg import Struct as Struct
+from __future__ import annotations
+from collections import UserList
+from pprint import (
+    saferepr,
+)
+from typing import Any
+
+from msgspec import (
+    msgpack,
+    Struct as _Struct,
+    structs,
+)
+
+
+class DiffDump(UserList):
+    '''
+    Very simple list delegator that repr() dumps (presumed) tuple
+    elements of the form `tuple[str, Any, Any]` in a nice
+    multi-line readable form for analyzing `Struct` diffs.
+
+    '''
+    def __repr__(self) -> str:
+        if not len(self):
+            return super().__repr__()
+
+        # format by displaying item pair's ``repr()`` on multiple,
+        # indented lines such that they are more easily visually
+        # comparable when printed to console when printed to
+        # console.
+        repstr: str = '[\n'
+        for k, left, right in self:
+            repstr += (
+                f'({k},\n'
+                f'\t{repr(left)},\n'
+                f'\t{repr(right)},\n'
+                ')\n'
+            )
+        repstr += ']\n'
+        return repstr
+
+
+class Struct(
+    _Struct,
+
+    # https://jcristharif.com/msgspec/structs.html#tagged-unions
+    # tag='pikerstruct',
+    # tag=True,
+):
+    '''
+    A "human friendlier" (aka repl buddy) struct subtype.
+
+    '''
+    def _sin_props(self) -> Iterator[
+        tuple[
+            structs.FieldIinfo,
+            str,
+            Any,
+        ]
+    ]:
+        '''
+        Iterate over all non-@property fields of this struct.
+
+        '''
+        fi: structs.FieldInfo
+        for fi in structs.fields(self):
+            key: str = fi.name
+            val: Any = getattr(self, key)
+            yield fi, key, val
+
+    def to_dict(
+        self,
+        include_non_members: bool = True,
+
+    ) -> dict:
+        '''
+        Like it sounds.. direct delegation to:
+        https://jcristharif.com/msgspec/api.html#msgspec.structs.asdict
+
+        BUT, by default we pop all non-member (aka not defined as
+        struct fields) fields by default.
+
+        '''
+        asdict: dict = structs.asdict(self)
+        if include_non_members:
+            return asdict
+
+        # only return a dict of the struct members
+        # which were provided as input, NOT anything
+        # added as type-defined `@property` methods!
+        sin_props: dict = {}
+        fi: structs.FieldInfo
+        for fi, k, v in self._sin_props():
+            sin_props[k] = asdict[k]
+
+        return sin_props
+
+    def pformat(
+        self,
+        field_indent: int = 2,
+        indent: int = 0,
+
+    ) -> str:
+        '''
+        Recursion-safe `pprint.pformat()` style formatting of
+        a `msgspec.Struct` for sane reading by a human using a REPL.
+
+        '''
+        # global whitespace indent
+        ws: str = ' '*indent
+
+        # field whitespace indent
+        field_ws: str = ' '*(field_indent + indent)
+
+        # qtn: str = ws + self.__class__.__qualname__
+        qtn: str = self.__class__.__qualname__
+
+        obj_str: str = ''  # accumulator
+        fi: structs.FieldInfo
+        k: str
+        v: Any
+        for fi, k, v in self._sin_props():
+
+            # TODO: how can we prefer `Literal['option1',  'option2,
+            # ..]` over .__name__ == `Literal` but still get only the
+            # latter for simple types like `str | int | None` etc..?
+            ft: type = fi.type
+            typ_name: str = getattr(ft, '__name__', str(ft))
+
+            # recurse to get sub-struct's `.pformat()` output Bo
+            if isinstance(v, Struct):
+                val_str: str =  v.pformat(
+                    indent=field_indent + indent,
+                    field_indent=indent + field_indent,
+                )
+
+            else:  # the `pprint` recursion-safe format:
+                # https://docs.python.org/3.11/library/pprint.html#pprint.saferepr
+                val_str: str = saferepr(v)
+
+            obj_str += (field_ws + f'{k}: {typ_name} = {val_str},\n')
+
+        return (
+            f'{qtn}(\n'
+            f'{obj_str}'
+            f'{ws})'
+        )
+
+    # TODO: use a pprint.PrettyPrinter instance around ONLY rendering
+    # inside a known tty?
+    # def __repr__(self) -> str:
+    #     ...
+
+    # __str__ = __repr__ = pformat
+    __repr__ = pformat
+
+    def copy(
+        self,
+        update: dict | None = None,
+
+    ) -> Struct:
+        '''
+        Validate-typecast all self defined fields, return a copy of
+        us with all such fields.
+
+        NOTE: This is kinda like the default behaviour in
+        `pydantic.BaseModel` except a copy of the object is
+        returned making it compat with `frozen=True`.
+
+        '''
+        if update:
+            for k, v in update.items():
+                setattr(self, k, v)
+
+        # NOTE: roundtrip serialize to validate
+        # - enode to msgpack binary format,
+        # - decode that back to a struct.
+        return msgpack.Decoder(type=type(self)).decode(
+            msgpack.Encoder().encode(self)
+        )
+
+    def typecast(
+        self,
+
+        # TODO: allow only casting a named subset?
+        # fields: set[str] | None = None,
+
+    ) -> None:
+        '''
+        Cast all fields using their declared type annotations
+        (kinda like what `pydantic` does by default).
+
+        NOTE: this of course won't work on frozen types, use
+        ``.copy()`` above in such cases.
+
+        '''
+        # https://jcristharif.com/msgspec/api.html#msgspec.structs.fields
+        fi: structs.FieldInfo
+        for fi in structs.fields(self):
+            setattr(
+                self,
+                fi.name,
+                fi.type(getattr(self, fi.name)),
+            )
+
+    def __sub__(
+        self,
+        other: Struct,
+
+    ) -> DiffDump[tuple[str, Any, Any]]:
+        '''
+        Compare fields/items key-wise and return a ``DiffDump``
+        for easy visual REPL comparison B)
+
+        '''
+        diffs: DiffDump[tuple[str, Any, Any]] = DiffDump()
+        for fi in structs.fields(self):
+            attr_name: str = fi.name
+            ours: Any = getattr(self, attr_name)
+            theirs: Any = getattr(other, attr_name)
+            if ours != theirs:
+                diffs.append((
+                    attr_name,
+                    ours,
+                    theirs,
+                ))
+
+        return diffs