Port EMS to typed messaging + bidir streaming

This moves the entire clearing system to use typed messages using
`pydantic.BaseModel` such that the streamed request-response order
submission protocols can be explicitly viewed in terms of message
schema, flow, and sequencing. Using the explicit message formats we can
now dig into simplifying and normalizing across broker provider apis to
get the best uniformity and simplicity.

The order submission sequence is now fully async: an order request is
expected to be explicitly acked with a new message and if cancellation
is requested by the client before the ack arrives, the cancel message is
stashed and then later sent immediately on receipt of the order
submission's ack from the backend broker. Backend brokers are now
controlled using a 2-way request-response streaming dialogue which is
fully api agnostic of the clearing system's core processing; This
leverages the new bi-directional streaming apis from `tractor`.  The
clearing core (emsd) was also simplified by moving the paper engine to
it's own sub-actor and making it api-symmetric with expected `brokerd`
endpoints.

A couple of the ems status messages were changed/added:
'dark_executed' -> 'dark_triggered'
added 'alert_triggered'

More cleaning of old code to come!

piker/clearing/_client.py

@@ -19,34 +19,23 @@ Orders and execution client API.
 
 """
 from contextlib import asynccontextmanager
-from typing import Dict, Tuple, List
+from typing import Dict
 from pprint import pformat
 from dataclasses import dataclass, field
 
 import trio
 import tractor
-# import msgspec
 
 from ..data._source import Symbol
 from ..log import get_logger
 from ._ems import _emsd_main
 from .._daemon import maybe_open_emsd
+from ._messages import Order, Cancel
 
 
 log = get_logger(__name__)
 
 
-# TODO: some kinda validation like this
-# class Order(msgspec.Struct):
-#     action: str
-#     price: float
-#     size: float
-#     symbol: str
-#     brokers: List[str]
-#     oid: str
-#     exec_mode: str
-
-
 @dataclass
 class OrderBook:
     """Buy-side (client-side ?) order book ctl and tracking.
@@ -64,31 +53,34 @@ class OrderBook:
     _to_ems: trio.abc.SendChannel
     _from_order_book: trio.abc.ReceiveChannel
 
-    _sent_orders: Dict[str, dict] = field(default_factory=dict)
+    _sent_orders: Dict[str, Order] = field(default_factory=dict)
     _ready_to_receive: trio.Event = trio.Event()
 
     def send(
+
         self,
         uuid: str,
         symbol: str,
-        brokers: List[str],
+        brokers: list[str],
         price: float,
         size: float,
         action: str,
         exec_mode: str,
+
     ) -> dict:
-        cmd = {
+        msg = Order(
-            'action': action,
+            action=action,
-            'price': price,
+            price=price,
-            'size': size,
+            size=size,
-            'symbol': symbol,
+            symbol=symbol,
-            'brokers': brokers,
+            brokers=brokers,
-            'oid': uuid,
+            oid=uuid,
-            'exec_mode': exec_mode,  # dark or live
+            exec_mode=exec_mode,  # dark or live
-        }
+        )
-        self._sent_orders[uuid] = cmd
+
-        self._to_ems.send_nowait(cmd)
+        self._sent_orders[uuid] = msg
-        return cmd
+        self._to_ems.send_nowait(msg.dict())
+        return msg
 
     def update(
         self,
@@ -98,28 +90,27 @@ class OrderBook:
         cmd = self._sent_orders[uuid]
         msg = cmd.dict()
         msg.update(data)
-        self._sent_orders[uuid] = OrderMsg(**msg)
+        self._sent_orders[uuid] = Order(**msg)
         self._to_ems.send_nowait(msg)
         return cmd
 
     def cancel(self, uuid: str) -> bool:
-        """Cancel an order (or alert) from the EMS.
+        """Cancel an order (or alert) in the EMS.
 
         """
         cmd = self._sent_orders[uuid]
-        msg = {
+        msg = Cancel(
-            'action': 'cancel',
+            oid=uuid,
-            'oid': uuid,
+            symbol=cmd.symbol,
-            'symbol': cmd['symbol'],
+        )
-        }
+        self._to_ems.send_nowait(msg.dict())
-        self._to_ems.send_nowait(msg)
 
 
 _orders: OrderBook = None
 
 
 def get_orders(
-    emsd_uid: Tuple[str, str] = None
+    emsd_uid: tuple[str, str] = None
 ) -> OrderBook:
     """"
     OrderBook singleton factory per actor.
@@ -139,7 +130,10 @@ def get_orders(
     return _orders
 
 
+# TODO: we can get rid of this relay loop once we move
+# order_mode inputs to async code!
 async def relay_order_cmds_from_sync_code(
+
     symbol_key: str,
     to_ems_stream: tractor.MsgStream,
 
@@ -184,7 +178,8 @@ async def relay_order_cmds_from_sync_code(
 async def open_ems(
     broker: str,
     symbol: Symbol,
-) -> None:
+
+) -> (OrderBook, tractor.MsgStream, dict):
     """Spawn an EMS daemon and begin sending orders and receiving
     alerts.
 
@@ -232,9 +227,9 @@ async def open_ems(
                 broker=broker,
                 symbol=symbol.key,
 
-            # TODO: ``first`` here should be the active orders/execs
+                # TODO: ``first`` here should be the active orders/execs
-            # persistent on the ems so that loca UI's can be populated.
+                # persistent on the ems so that loca UI's can be populated.
-            ) as (ctx, first),
+            ) as (ctx, positions),
 
             # open 2-way trade command stream
             ctx.open_stream() as trades_stream,
@@ -246,4 +241,4 @@ async def open_ems(
                     trades_stream
                 )
 
-                yield book, trades_stream
+                yield book, trades_stream, positions

piker/ui/order_mode.py

@@ -127,9 +127,9 @@ class OrderMode:
 
         """
         line = self.lines.commit_line(uuid)
-        req_msg = self.book._sent_orders.get(uuid)
+        # req_msg = self.book._sent_orders.get(uuid)
-        if req_msg:
+        # if req_msg:
-            req_msg.ack_time_ns = time.time_ns()
+        #     req_msg.ack_time_ns = time.time_ns()
 
         return line
 
@@ -317,10 +317,14 @@ async def start_order_mode(
 
     # spawn EMS actor-service
     async with (
-        open_ems(brokername, symbol) as (book, trades_stream),
+        open_ems(brokername, symbol) as (book, trades_stream, positions),
         open_order_mode(symbol, chart, book) as order_mode
     ):
 
+        # update any exising positions
+        for sym, msg in positions.items():
+            order_mode.on_position_update(msg)
+
         def get_index(time: float):
 
             # XXX: not sure why the time is so off here
@@ -343,16 +347,15 @@ async def start_order_mode(
             fmsg = pformat(msg)
             log.info(f'Received order msg:\n{fmsg}')
 
-            resp = msg['resp']
+            name = msg['name']
-
+            if name in (
-            if resp in (
                 'position',
             ):
                 # show line label once order is live
                 order_mode.on_position_update(msg)
                 continue
 
-            # delete the line from view
+            resp = msg['resp']
             oid = msg['oid']
 
             # response to 'action' request (buy/sell)
@@ -375,21 +378,21 @@ async def start_order_mode(
                 order_mode.on_cancel(oid)
 
             elif resp in (
-                'dark_executed'
+                'dark_triggered'
             ):
                 log.info(f'Dark order triggered for {fmsg}')
 
-                # for alerts add a triangle and remove the
+            elif resp in (
-                # level line
+                'alert_triggered'
-                if msg['cmd']['action'] == 'alert':
+            ):
-
+                # should only be one "fill" for an alert
-                    # should only be one "fill" for an alert
+                # add a triangle and remove the level line
-                    order_mode.on_fill(
+                order_mode.on_fill(
-                        oid,
+                    oid,
-                        price=msg['trigger_price'],
+                    price=msg['trigger_price'],
-                        arrow_index=get_index(time.time())
+                    arrow_index=get_index(time.time())
-                    )
+                )
-                    await order_mode.on_exec(oid, msg)
+                await order_mode.on_exec(oid, msg)
 
             # response to completed 'action' request for buy/sell
             elif resp in (
@@ -400,12 +403,15 @@ async def start_order_mode(
             # each clearing tick is responded individually
             elif resp in ('broker_filled',):
 
-                action = msg['action']
+                action = book._sent_orders[oid].action
+                details = msg['brokerd_msg']
 
                 # TODO: some kinda progress system
                 order_mode.on_fill(
                     oid,
-                    price=msg['price'],
+                    price=details['price'],
-                    arrow_index=get_index(msg['broker_time']),
                     pointing='up' if action == 'buy' else 'down',
+
+                    # TODO: put the actual exchange timestamp
+                    arrow_index=get_index(details['broker_time']),
                 )