From 9e36dbe47f8a326c8eefa00ebf3614d7737f5f81 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 5 Aug 2022 14:50:10 -0400 Subject: [PATCH 01/33] Relay existing open orders from ib on startup --- piker/brokers/ib/broker.py | 41 +++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index c2f03a4f..a91a8318 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -41,6 +41,7 @@ from ib_insync.contract import ( from ib_insync.order import ( Trade, OrderStatus, + Order, ) from ib_insync.objects import ( Fill, @@ -451,7 +452,6 @@ async def trades_dialogue( # we might also want to delegate a specific actor for # ledger writing / reading for speed? async with ( - # trio.open_nursery() as nurse, open_client_proxies() as (proxies, aioclients), ): # Open a trade ledgers stack for appending trade records over @@ -481,6 +481,42 @@ async def trades_dialogue( for account, proxy in proxies.items(): client = aioclients[account] + trades: list[Trade] = client.ib.openTrades() + order_msgs = [] + for trade in trades: + + order = trade.order + quant = trade.order.totalQuantity + size = { + 'SELL': -1, + 'BUY': 1, + }[order.action] * quant + fqsn, _ = con2fqsn(trade.contract) + + # TODO: maybe embed a ``BrokerdOrder`` instead + # since then we can directly load it on the client + # side in the order mode loop? + msg = BrokerdStatus( + reqid=order.orderId, + time_ns=time.time_ns(), + account=accounts_def.inverse[order.account], + status='submitted', + size=size, + price=order.lmtPrice, + filled=0, + reason='Existing live order', + + # this seems to not be necessarily up to date in + # the execDetails event.. so we have to send it + # here I guess? + remaining=quant, + broker_details={ + 'name': 'ib', + 'fqsn': fqsn, + }, + ) + order_msgs.append(msg) + # process pp value reported from ib's system. we only use these # to cross-check sizing since average pricing on their end uses # the so called (bs) "FIFO" style which more or less results in @@ -615,6 +651,9 @@ async def trades_dialogue( ctx.open_stream() as ems_stream, trio.open_nursery() as n, ): + # relay existing open orders to ems + for msg in order_msgs: + await ems_stream.send(msg) for client in set(aioclients.values()): trade_event_stream = await n.start( From 682a0191ef728ab1736f3fd20cd51a441daf9d51 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 5 Aug 2022 14:51:15 -0400 Subject: [PATCH 02/33] First draft: relay open orders through ems and display on chart --- piker/clearing/_client.py | 8 +- piker/clearing/_ems.py | 161 ++++++++++++++++++++---------------- piker/clearing/_messages.py | 4 + piker/ui/order_mode.py | 102 ++++++++++++++++------- 4 files changed, 170 insertions(+), 105 deletions(-) diff --git a/piker/clearing/_client.py b/piker/clearing/_client.py index 95d80986..ef5647db 100644 --- a/piker/clearing/_client.py +++ b/piker/clearing/_client.py @@ -149,10 +149,14 @@ async def relay_order_cmds_from_sync_code( book = get_orders() async with book._from_order_book.subscribe() as orders_stream: async for cmd in orders_stream: - if cmd.symbol == symbol_key: - log.info(f'Send order cmd:\n{pformat(cmd)}') + sym = cmd.symbol + msg = pformat(cmd) + if sym == symbol_key: + log.info(f'Send order cmd:\n{msg}') # send msg over IPC / wire await to_ems_stream.send(cmd) + else: + log.warning(f'Ignoring unmatched order cmd for {sym}: {msg}') @acm diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 81288899..b847333a 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -188,9 +188,9 @@ async def clear_dark_triggers( tuple(execs.items()) ): if ( - not pred or - ttype not in tf or - not pred(price) + not pred + or ttype not in tf + or not pred(price) ): # log.runtime( # f'skipping quote for {sym} ' @@ -345,7 +345,7 @@ class Router(Struct): already exists. ''' - relay = self.relays.get(feed.mod.name) + relay: TradesRelay = self.relays.get(feed.mod.name) if ( relay is None @@ -452,7 +452,6 @@ async def open_brokerd_trades_dialogue( async with ( open_trades_endpoint as (brokerd_ctx, (positions, accounts,)), brokerd_ctx.open_stream() as brokerd_trades_stream, - ): # XXX: really we only want one stream per `emsd` actor # to relay global `brokerd` order events unless we're @@ -718,6 +717,43 @@ async def translate_and_relay_brokerd_events( # one of {submitted, cancelled} resp = 'broker_' + msg.status + # unknown valid BrokerdStatus + case { + 'name': 'status', + 'status': status, + 'reqid': reqid, # brokerd generated order-request id + 'broker_details': details, + }: + # TODO: we probably want some kind of "tagging" system + # for external order submissions like this eventually + # to be able to more formally handle multi-player + # trading... + + if status == 'submitted': + msg = BrokerdStatus(**brokerd_msg) + log.info('Relaying existing open order:\n {brokerd_msg}') + + # use backend request id as our ems id though this + # may end up with collisions? + broker = details['name'] + # oid = f'{broker}-{reqid}' + oid = reqid + book._ems_entries[oid] = msg + # attempt to avoid collisions + msg.reqid = oid + resp = 'broker_submitted' + + # register this existing broker-side dialog + book._ems2brokerd_ids[oid] = reqid + + else: + log.error( + f'Unknown status msg:\n' + f'{pformat(brokerd_msg)}\n' + 'Unable to relay message to client side!?' + ) + continue + # BrokerdFill case { 'name': 'fill', @@ -731,79 +767,62 @@ async def translate_and_relay_brokerd_events( resp = 'broker_filled' log.info(f'\nFill for {oid} cleared with:\n{pformat(resp)}') - # unknown valid message case? - # case { - # 'name': name, - # 'symbol': sym, - # 'reqid': reqid, # brokerd generated order-request id - # # 'oid': oid, # ems order-dialog id - # 'broker_details': details, - - # } if ( - # book._ems2brokerd_ids.inverse.get(reqid) is None - # ): - # # TODO: pretty sure we can drop this now? - - # # XXX: paper clearing special cases - # # paper engine race case: ``Client.submit_limit()`` hasn't - # # returned yet and provided an output reqid to register - # # locally, so we need to retreive the oid that was already - # # packed at submission since we already know it ahead of - # # time - # paper = details.get('paper_info') - # ext = details.get('external') - - # if paper: - # # paperboi keeps the ems id up front - # oid = paper['oid'] - - # elif ext: - # # may be an order msg specified as "external" to the - # # piker ems flow (i.e. generated by some other - # # external broker backend client (like tws for ib) - # log.error(f"External trade event {name}@{ext}") - - # else: - # # something is out of order, we don't have an oid for - # # this broker-side message. - # log.error( - # f'Unknown oid: {oid} for msg {name}:\n' - # f'{pformat(brokerd_msg)}\n' - # 'Unable to relay message to client side!?' - # ) - - # continue - case _: raise ValueError(f'Brokerd message {brokerd_msg} is invalid') # retrieve existing live flow entry = book._ems_entries[oid] - assert entry.oid == oid - old_reqid = entry.reqid - if old_reqid and old_reqid != reqid: - log.warning( - f'Brokerd order id change for {oid}:\n' - f'{old_reqid} -> {reqid}' - ) - - # Create and relay response status message - # to requesting EMS client - try: - ems_client_order_stream = router.dialogues[oid] - await ems_client_order_stream.send( - Status( - oid=oid, - resp=resp, - time_ns=time.time_ns(), - broker_reqid=reqid, - brokerd_msg=msg, + if getattr(entry, 'oid', None): + assert entry.oid == oid + old_reqid = entry.reqid + if old_reqid and old_reqid != reqid: + log.warning( + f'Brokerd order id change for {oid}:\n' + f'{old_reqid} -> {reqid}' ) - ) - except KeyError: - log.error( - f'Received `brokerd` msg for unknown client with oid: {oid}') + + # Create and relay response status message + # to requesting EMS client + try: + ems_client_order_stream = router.dialogues[oid] + await ems_client_order_stream.send( + Status( + oid=oid, + resp=resp, + time_ns=time.time_ns(), + broker_reqid=reqid, + brokerd_msg=msg, + ) + ) + except KeyError: + log.error( + f'Received `brokerd` msg for unknown client oid: {oid}') + + else: + # existing open order relay + assert oid == entry.reqid + + # fan-out-relay position msgs immediately by + # broadcasting updates on all client streams + for client_stream in router.clients.copy(): + try: + await client_stream.send( + Status( + oid=oid, + resp=resp, + time_ns=time.time_ns(), + broker_reqid=reqid, + brokerd_msg=msg, + ) + ) + except( + trio.ClosedResourceError, + trio.BrokenResourceError, + ): + router.clients.remove(client_stream) + log.warning( + f'client for {client_stream} was already closed?') # TODO: do we want this to keep things cleaned up? # it might require a special status from brokerd to affirm the diff --git a/piker/clearing/_messages.py b/piker/clearing/_messages.py index c30ada54..eb94b147 100644 --- a/piker/clearing/_messages.py +++ b/piker/clearing/_messages.py @@ -194,6 +194,10 @@ class BrokerdStatus(Struct): # } status: str + # +ve is buy, -ve is sell + size: float = 0.0 + price: float = 0.0 + filled: float = 0.0 reason: str = '' remaining: float = 0.0 diff --git a/piker/ui/order_mode.py b/piker/ui/order_mode.py index 41078e05..5b56e06f 100644 --- a/piker/ui/order_mode.py +++ b/piker/ui/order_mode.py @@ -152,10 +152,7 @@ class OrderMode: def line_from_order( self, - order: Order, - symbol: Symbol, - **line_kwargs, ) -> LevelLine: @@ -173,8 +170,7 @@ class OrderMode: color=self._colors[order.action], dotted=True if ( - order.exec_mode == 'dark' and - order.action != 'alert' + order.exec_mode == 'dark' and order.action != 'alert' ) else False, **line_kwargs, @@ -236,7 +232,6 @@ class OrderMode: line = self.line_from_order( order, - symbol, show_markers=True, # just for the stage line to avoid @@ -262,6 +257,8 @@ class OrderMode: def submit_order( self, + send_msg: bool = True, + order: Optional[Order] = None, ) -> OrderDialog: ''' @@ -269,18 +266,19 @@ class OrderMode: represent the order on a chart. ''' - staged = self._staged_order - symbol: Symbol = staged.symbol - oid = str(uuid.uuid4()) + if not order: + staged = self._staged_order + oid = str(uuid.uuid4()) + # symbol: Symbol = staged.symbol - # format order data for ems - order = staged.copy() - order.oid = oid - order.symbol = symbol.front_fqsn() + # format order data for ems + order = staged.copy() + order.oid = oid + + order.symbol = order.symbol.front_fqsn() line = self.line_from_order( order, - symbol, show_markers=True, only_show_markers_on_hover=True, @@ -298,17 +296,17 @@ class OrderMode: # color once the submission ack arrives. self.lines.submit_line( line=line, - uuid=oid, + uuid=order.oid, ) dialog = OrderDialog( - uuid=oid, + uuid=order.oid, order=order, - symbol=symbol, + symbol=order.symbol, line=line, last_status_close=self.multistatus.open_status( - f'submitting {self._trigger_type}-{order.action}', - final_msg=f'submitted {self._trigger_type}-{order.action}', + f'submitting {order.exec_mode}-{order.action}', + final_msg=f'submitted {order.exec_mode}-{order.action}', clear_on_next=True, ) ) @@ -318,14 +316,21 @@ class OrderMode: # enter submission which will be popped once a response # from the EMS is received to move the order to a different# status - self.dialogs[oid] = dialog + self.dialogs[order.oid] = dialog # hook up mouse drag handlers line._on_drag_start = self.order_line_modify_start line._on_drag_end = self.order_line_modify_complete # send order cmd to ems - self.book.send(order) + if send_msg: + self.book.send(order) + else: + # just register for control over this order + # TODO: some kind of mini-perms system here based on + # an out-of-band tagging/auth sub-sys for multiplayer + # order control? + self.book._sent_orders[order.oid] = order return dialog @@ -502,7 +507,7 @@ class OrderMode: oid = dialog.uuid cancel_status_close = self.multistatus.open_status( - f'cancelling order {oid[:6]}', + f'cancelling order {oid}', group_key=key, ) dialog.last_status_close = cancel_status_close @@ -596,10 +601,10 @@ async def open_order_mode( sym = msg['symbol'] if ( - sym == symkey or - # mega-UGH, i think we need to fix the FQSN stuff sooner - # then later.. - sym == symkey.removesuffix(f'.{broker}') + (sym == symkey) or ( + # mega-UGH, i think we need to fix the FQSN + # stuff sooner then later.. + sym == symkey.removesuffix(f'.{broker}')) ): pps_by_account[acctid] = msg @@ -653,7 +658,7 @@ async def open_order_mode( # setup order mode sidepane widgets form: FieldsForm = chart.sidepane form.vbox.setSpacing( - int((1 + 5/8)*_font.px_size) + int((1 + 5 / 8) * _font.px_size) ) from ._feedstatus import mk_feed_label @@ -814,15 +819,48 @@ async def process_trades_and_update_ui( continue resp = msg['resp'] - oid = msg['oid'] - + oid = str(msg['oid']) dialog = mode.dialogs.get(oid) + if dialog is None: log.warning(f'received msg for untracked dialog:\n{fmsg}') - # TODO: enable pure tracking / mirroring of dialogs - # is desired. - continue + size = msg['brokerd_msg']['size'] + if size >= 0: + action = 'buy' + else: + action = 'sell' + + acct = msg['brokerd_msg']['account'] + price = msg['brokerd_msg']['price'] + deats = msg['brokerd_msg']['broker_details'] + fqsn = ( + deats['fqsn'] + '.' + deats['name'] + ) + symbol = Symbol.from_fqsn( + fqsn=fqsn, + info={}, + ) + # map to order composite-type + order = Order( + action=action, + price=price, + account=acct, + size=size, + symbol=symbol, + brokers=symbol.brokers, + oid=oid, + exec_mode='live', # dark or live + ) + + dialog = mode.submit_order( + send_msg=False, + order=order, + ) + + # # TODO: enable pure tracking / mirroring of dialogs + # # is desired. + # continue # record message to dialog tracking dialog.msgs[oid] = msg From 016b669d63686b1b102f31c1e0f92111636ec6b3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 5 Aug 2022 16:13:19 -0400 Subject: [PATCH 03/33] Drop staged line runtime guard --- piker/ui/_editors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/piker/ui/_editors.py b/piker/ui/_editors.py index 03fd208e..38d30da4 100644 --- a/piker/ui/_editors.py +++ b/piker/ui/_editors.py @@ -140,9 +140,9 @@ class LineEditor: ) -> LevelLine: - staged_line = self._active_staged_line - if not staged_line: - raise RuntimeError("No line is currently staged!?") + # staged_line = self._active_staged_line + # if not staged_line: + # raise RuntimeError("No line is currently staged!?") # for now, until submission reponse arrives line.hide_labels() From 151038373833322ae7707374c0e232dbcb5c2918 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 5 Aug 2022 17:07:50 -0400 Subject: [PATCH 04/33] Always cast ems `requid` values to `int` --- piker/brokers/ib/broker.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index a91a8318..1bc4d632 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -41,7 +41,6 @@ from ib_insync.contract import ( from ib_insync.order import ( Trade, OrderStatus, - Order, ) from ib_insync.objects import ( Fill, @@ -123,10 +122,11 @@ async def handle_order_requests( f'An IB account number for name {account} is not found?\n' 'Make sure you have all TWS and GW instances running.' ) - await ems_order_stream.send(BrokerdError( - oid=request_msg['oid'], - symbol=request_msg['symbol'], - reason=f'No account found: `{account}` ?', + await ems_order_stream.send( + BrokerdError( + oid=request_msg['oid'], + symbol=request_msg['symbol'], + reason=f'No account found: `{account}` ?', )) continue @@ -147,6 +147,14 @@ async def handle_order_requests( # validate order = BrokerdOrder(**request_msg) + # XXX: by default 0 tells ``ib_insync`` methods that + # there is no existing order so ask the client to create + # a new one (which it seems to do by allocating an int + # counter - collision prone..) + reqid = order.reqid + if reqid is not None: + reqid = int(reqid) + # call our client api to submit the order reqid = client.submit_limit( oid=order.oid, @@ -155,12 +163,7 @@ async def handle_order_requests( action=order.action, size=order.size, account=acct_number, - - # XXX: by default 0 tells ``ib_insync`` methods that - # there is no existing order so ask the client to create - # a new one (which it seems to do by allocating an int - # counter - collision prone..) - reqid=order.reqid, + reqid=reqid, ) if reqid is None: await ems_order_stream.send(BrokerdError( @@ -182,7 +185,7 @@ async def handle_order_requests( elif action == 'cancel': msg = BrokerdCancel(**request_msg) - client.submit_cancel(reqid=msg.reqid) + client.submit_cancel(reqid=int(msg.reqid)) else: log.error(f'Unknown order command: {request_msg}') From e34ea94f9f7430e99e66e4b6c0b4702b618481ae Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 5 Aug 2022 18:29:40 -0400 Subject: [PATCH 05/33] Start brokerd relay loop after opening client stream In order to avoid missed existing order message emissions on startup we need to be sure the client side stream is registered with the router first. So break out the starting of the `translate_and_relay_brokerd_events()` task until inside the client stream block and start the task using the dark clearing loop nursery. Also, ensure `oid` (and thus for `ib` the equivalent re-used `reqid`) are cast to `str` before registering the dark book. Deliver the dark book entries as part of the `_emsd_main()` context `.started()` values. --- piker/clearing/_ems.py | 81 ++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index b847333a..01cc25a2 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -501,14 +501,9 @@ async def open_brokerd_trades_dialogue( task_status.started(relay) - await translate_and_relay_brokerd_events( - broker, - brokerd_trades_stream, - _router, - ) - # this context should block here indefinitely until # the ``brokerd`` task either dies or is cancelled + await trio.sleep_forever() finally: # parent context must have been closed @@ -567,8 +562,8 @@ async def translate_and_relay_brokerd_events( {'presubmitted', 'submitted', 'cancelled', 'inactive'} ''' - book = router.get_dark_book(broker) - relay = router.relays[broker] + book: _DarkBook = router.get_dark_book(broker) + relay: TradesRelay = router.relays[broker] assert relay.brokerd_dialogue == brokerd_trades_stream @@ -731,14 +726,16 @@ async def translate_and_relay_brokerd_events( if status == 'submitted': msg = BrokerdStatus(**brokerd_msg) - log.info('Relaying existing open order:\n {brokerd_msg}') + log.info( + f'Relaying existing open order:\n {brokerd_msg}' + ) # use backend request id as our ems id though this # may end up with collisions? broker = details['name'] - # oid = f'{broker}-{reqid}' - oid = reqid + oid = str(reqid) book._ems_entries[oid] = msg + # attempt to avoid collisions msg.reqid = oid resp = 'broker_submitted' @@ -848,7 +845,9 @@ async def process_client_order_cmds( async for cmd in client_order_stream: log.info(f'Received order cmd:\n{pformat(cmd)}') - oid = cmd['oid'] + # CAWT DAMN we need struct support! + oid = str(cmd['oid']) + # register this stream as an active dialogue for this order id # such that translated message from the brokerd backend can be # routed (relayed) to **just** that client stream (and in theory @@ -892,24 +891,24 @@ async def process_client_order_cmds( 'action': 'cancel', 'oid': oid, } if not live_entry: - try: - # remove from dark book clearing - dark_book.orders[symbol].pop(oid, None) + # try: + # remove from dark book clearing + dark_book.orders[symbol].pop(oid, None) - # tell client side that we've cancelled the - # dark-trigger order - await client_order_stream.send( - Status( - resp='dark_cancelled', - oid=oid, - time_ns=time.time_ns(), - ) + # tell client side that we've cancelled the + # dark-trigger order + await client_order_stream.send( + Status( + resp='dark_cancelled', + oid=oid, + time_ns=time.time_ns(), ) - # de-register this client dialogue - router.dialogues.pop(oid) + ) + # de-register this client dialogue + router.dialogues.pop(oid) - except KeyError: - log.exception(f'No dark order for {symbol}?') + # except KeyError: + # log.exception(f'No dark order for {symbol}?') # live order submission case { @@ -932,8 +931,11 @@ async def process_client_order_cmds( sym = fqsn.replace(f'.{broker}', '') if live_entry is not None: - # sanity check on emsd id - assert live_entry.oid == oid + # sanity check on emsd id, but it won't work + # for pre-existing orders that we load since + # the only msg will be a ``BrokerdStatus`` + # assert live_entry.oid == oid + reqid = live_entry.reqid # if we already had a broker order id then # this is likely an order update commmand. @@ -1118,10 +1120,9 @@ async def _emsd_main( ): # XXX: this should be initial price quote from target provider - first_quote = feed.first_quotes[fqsn] - - book = _router.get_dark_book(broker) - book.lasts[fqsn] = first_quote['last'] + first_quote: dict = feed.first_quotes[fqsn] + book: _DarkBook = _router.get_dark_book(broker) + book.lasts[fqsn]: float = first_quote['last'] # open a stream with the brokerd backend for order # flow dialogue @@ -1148,12 +1149,25 @@ async def _emsd_main( await ems_ctx.started(( relay.positions, list(relay.accounts), + book._ems_entries, )) # establish 2-way stream with requesting order-client and # begin handling inbound order requests and updates async with ems_ctx.open_stream() as ems_client_order_stream: + # register the client side before startingn the + # brokerd-side relay task to ensure the client is + # delivered all exisiting open orders on startup. + _router.clients.add(ems_client_order_stream) + + n.start_soon( + translate_and_relay_brokerd_events, + broker, + brokerd_stream, + _router, + ) + # trigger scan and exec loop n.start_soon( clear_dark_triggers, @@ -1168,7 +1182,6 @@ async def _emsd_main( # start inbound (from attached client) order request processing try: - _router.clients.add(ems_client_order_stream) # main entrypoint, run here until cancelled. await process_client_order_cmds( From 1cfa04927dcfe2db74e814a3aee74e269497467c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 5 Aug 2022 19:05:05 -0400 Subject: [PATCH 06/33] Lol, handle failed-to-cancel statuses.. --- piker/brokers/ib/broker.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index 1bc4d632..b2d6bd0d 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -823,6 +823,14 @@ async def deliver_trade_events( # unwrap needed data from ib_insync internal types trade: Trade = item status: OrderStatus = trade.orderStatus + status_key = status.status.lower() + + # double check there is no error when + # cancelling.. gawwwd + if status_key == 'cancelled': + last_log = trade.log[-1] + if last_log.message: + status_key = trade.log[-2].status # skip duplicate filled updates - we get the deats # from the execution details event @@ -833,7 +841,7 @@ async def deliver_trade_events( account=accounts_def.inverse[trade.order.account], # everyone doin camel case.. - status=status.status.lower(), # force lower case + status=status_key, # force lower case filled=status.filled, reason=status.whyHeld, From 2548aae73d6c3a5dd2411406501c2a38baf5f35c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 5 Aug 2022 20:39:00 -0400 Subject: [PATCH 07/33] Deliver existing dialog (msgs) to every EMS client Ideally every client that connects to the ems can know its state (immediately) meaning relay all the order dialogs that are currently active. This adds full (hacky WIP) support to receive those dialog (msgs) from the `open_ems()` startup values via the `.started()` msg from `_emsd_main()`. Further this adds support to the order mode chart-UI to display existing (live) orders on the chart during startup. Details include, - add a `OrderMode.load_unknown_dialog_from_msg()` for processing and displaying a ``BrokerdStatus`` (for now) msg from the EMS that was not previously created by the current ems client and registering and displaying it on the chart. - break out the ems msg processing into a new `order_mode.process_trade_msg()` func so that it can be called on the startup dialog-msg set as well as eventually used a more general low level auto-strat API (eg. when we get to displaying auto-strat and group trading automatically on an observing chart UI. - hackyness around msg-processing for the dialogs delivery since we're technically delivering `BrokerdStatus` msgs when the client-side processing technically expects `Status` msgs.. we'll rectify this soon! --- piker/clearing/_client.py | 18 +- piker/ui/order_mode.py | 347 +++++++++++++++++++++----------------- 2 files changed, 205 insertions(+), 160 deletions(-) diff --git a/piker/clearing/_client.py b/piker/clearing/_client.py index ef5647db..3e87ab96 100644 --- a/piker/clearing/_client.py +++ b/piker/clearing/_client.py @@ -224,11 +224,19 @@ async def open_ems( fqsn=fqsn, exec_mode=mode, - ) as (ctx, (positions, accounts)), + ) as ( + ctx, + ( + positions, + accounts, + dialogs, + ) + ), # open 2-way trade command stream ctx.open_stream() as trades_stream, ): + # start sync code order msg delivery task async with trio.open_nursery() as n: n.start_soon( relay_order_cmds_from_sync_code, @@ -236,4 +244,10 @@ async def open_ems( trades_stream ) - yield book, trades_stream, positions, accounts + yield ( + book, + trades_stream, + positions, + accounts, + dialogs, + ) diff --git a/piker/ui/order_mode.py b/piker/ui/order_mode.py index 5b56e06f..b8dd37f9 100644 --- a/piker/ui/order_mode.py +++ b/piker/ui/order_mode.py @@ -517,6 +517,48 @@ class OrderMode: return ids + def load_unknown_dialog_from_msg( + self, + msg: dict, + + ) -> OrderDialog: + + oid = str(msg['oid']) + size = msg['brokerd_msg']['size'] + if size >= 0: + action = 'buy' + else: + action = 'sell' + + acct = msg['brokerd_msg']['account'] + price = msg['brokerd_msg']['price'] + deats = msg['brokerd_msg']['broker_details'] + fqsn = ( + deats['fqsn'] + '.' + deats['name'] + ) + symbol = Symbol.from_fqsn( + fqsn=fqsn, + info={}, + ) + # map to order composite-type + order = Order( + action=action, + price=price, + account=acct, + size=size, + symbol=symbol, + brokers=symbol.brokers, + oid=oid, + exec_mode='live', # dark or live + ) + + dialog = self.submit_order( + send_msg=False, + order=order, + ) + assert self.dialogs[oid] == dialog + return dialog + @asynccontextmanager async def open_order_mode( @@ -554,6 +596,7 @@ async def open_order_mode( trades_stream, position_msgs, brokerd_accounts, + ems_dialog_msgs, ), trio.open_nursery() as tn, @@ -760,195 +803,183 @@ async def open_order_mode( # to handle input since the ems connection is ready started.set() + for oid, msg in ems_dialog_msgs.items(): + + # HACK ALERT: ensure a resp field is filled out since + # techincally the call below expects a ``Status``. TODO: + # parse into proper ``Status`` equivalents ems-side? + msg.setdefault('resp', msg['broker_details']['resp']) + msg.setdefault('oid', msg['broker_details']['oid']) + msg['brokerd_msg'] = msg + + await process_trade_msg( + mode, + book, + msg, + ) + tn.start_soon( process_trades_and_update_ui, - tn, - feed, - mode, trades_stream, + mode, book, ) + yield mode async def process_trades_and_update_ui( - n: trio.Nursery, - feed: Feed, - mode: OrderMode, trades_stream: tractor.MsgStream, + mode: OrderMode, book: OrderBook, ) -> None: - get_index = mode.chart.get_index - global _pnl_tasks - # this is where we receive **back** messages # about executions **from** the EMS actor async for msg in trades_stream: + await process_trade_msg( + mode, + book, + msg, + ) - fmsg = pformat(msg) - log.info(f'Received order msg:\n{fmsg}') - name = msg['name'] - if name in ( - 'position', +async def process_trade_msg( + mode: OrderMode, + book: OrderBook, + msg: dict, + +) -> None: + + get_index = mode.chart.get_index + fmsg = pformat(msg) + log.info(f'Received order msg:\n{fmsg}') + name = msg['name'] + if name in ( + 'position', + ): + sym = mode.chart.linked.symbol + pp_msg_symbol = msg['symbol'].lower() + fqsn = sym.front_fqsn() + broker, key = sym.front_feed() + if ( + pp_msg_symbol == fqsn + or pp_msg_symbol == fqsn.removesuffix(f'.{broker}') ): - sym = mode.chart.linked.symbol - pp_msg_symbol = msg['symbol'].lower() - fqsn = sym.front_fqsn() - broker, key = sym.front_feed() - if ( - pp_msg_symbol == fqsn - or pp_msg_symbol == fqsn.removesuffix(f'.{broker}') - ): - log.info(f'{fqsn} matched pp msg: {fmsg}') - tracker = mode.trackers[msg['account']] - tracker.live_pp.update_from_msg(msg) - # update order pane widgets - tracker.update_from_pp() - mode.pane.update_status_ui(tracker) + log.info(f'{fqsn} matched pp msg: {fmsg}') + tracker = mode.trackers[msg['account']] + tracker.live_pp.update_from_msg(msg) + # update order pane widgets + tracker.update_from_pp() + mode.pane.update_status_ui(tracker) - if tracker.live_pp.size: - # display pnl - mode.pane.display_pnl(tracker) + if tracker.live_pp.size: + # display pnl + mode.pane.display_pnl(tracker) - # short circuit to next msg to avoid - # unnecessary msg content lookups - continue + # short circuit to next msg to avoid + # unnecessary msg content lookups + return + # continue - resp = msg['resp'] - oid = str(msg['oid']) - dialog = mode.dialogs.get(oid) + resp = msg['resp'] + oid = str(msg['oid']) + dialog = mode.dialogs.get(oid) - if dialog is None: - log.warning(f'received msg for untracked dialog:\n{fmsg}') + if dialog is None: + log.warning( + f'received msg for untracked dialog:\n{fmsg}' + ) + dialog = mode.load_unknown_dialog_from_msg(msg) - size = msg['brokerd_msg']['size'] - if size >= 0: - action = 'buy' - else: - action = 'sell' + # record message to dialog tracking + dialog.msgs[oid] = msg - acct = msg['brokerd_msg']['account'] - price = msg['brokerd_msg']['price'] - deats = msg['brokerd_msg']['broker_details'] - fqsn = ( - deats['fqsn'] + '.' + deats['name'] - ) - symbol = Symbol.from_fqsn( - fqsn=fqsn, - info={}, - ) - # map to order composite-type - order = Order( - action=action, - price=price, - account=acct, - size=size, - symbol=symbol, - brokers=symbol.brokers, - oid=oid, - exec_mode='live', # dark or live - ) + # response to 'action' request (buy/sell) + if resp in ( + 'dark_submitted', + 'broker_submitted' + ): + # show line label once order is live + mode.on_submit(oid) - dialog = mode.submit_order( - send_msg=False, - order=order, - ) + # resp to 'cancel' request or error condition + # for action request + elif resp in ( + 'broker_inactive', + 'broker_errored', + ): + # delete level line from view + mode.on_cancel(oid) + broker_msg = msg['brokerd_msg'] + log.error( + f'Order {oid}->{resp} with:\n{pformat(broker_msg)}' + ) - # # TODO: enable pure tracking / mirroring of dialogs - # # is desired. + elif resp in ( + 'broker_cancelled', + 'dark_cancelled' + ): + # delete level line from view + mode.on_cancel(oid) + broker_msg = msg['brokerd_msg'] + log.cancel( + f'Order {oid}->{resp} with:\n{pformat(broker_msg)}' + ) + + elif resp in ( + 'dark_triggered' + ): + log.info(f'Dark order triggered for {fmsg}') + + elif resp in ( + 'alert_triggered' + ): + # should only be one "fill" for an alert + # add a triangle and remove the level line + mode.on_fill( + oid, + price=msg['trigger_price'], + arrow_index=get_index(time.time()), + ) + mode.lines.remove_line(uuid=oid) + await mode.on_exec(oid, msg) + + # response to completed 'action' request for buy/sell + elif resp in ( + 'broker_executed', + ): + # right now this is just triggering a system alert + await mode.on_exec(oid, msg) + + if msg['brokerd_msg']['remaining'] == 0: + mode.lines.remove_line(uuid=oid) + + # each clearing tick is responded individually + elif resp in ( + 'broker_filled', + ): + known_order = book._sent_orders.get(oid) + if not known_order: + log.warning(f'order {oid} is unknown') + return # continue - # record message to dialog tracking - dialog.msgs[oid] = msg + action = known_order.action + details = msg['brokerd_msg'] - # response to 'action' request (buy/sell) - if resp in ( - 'dark_submitted', - 'broker_submitted' - ): + # TODO: some kinda progress system + mode.on_fill( + oid, + price=details['price'], + pointing='up' if action == 'buy' else 'down', - # show line label once order is live - mode.on_submit(oid) + # TODO: put the actual exchange timestamp + arrow_index=get_index(details['broker_time']), + ) - # resp to 'cancel' request or error condition - # for action request - elif resp in ( - 'broker_inactive', - 'broker_errored', - ): - # delete level line from view - mode.on_cancel(oid) - broker_msg = msg['brokerd_msg'] - log.error( - f'Order {oid}->{resp} with:\n{pformat(broker_msg)}' - ) - - elif resp in ( - 'broker_cancelled', - 'dark_cancelled' - ): - # delete level line from view - mode.on_cancel(oid) - broker_msg = msg['brokerd_msg'] - log.cancel( - f'Order {oid}->{resp} with:\n{pformat(broker_msg)}' - ) - - elif resp in ( - 'dark_triggered' - ): - log.info(f'Dark order triggered for {fmsg}') - - elif resp in ( - 'alert_triggered' - ): - # should only be one "fill" for an alert - # add a triangle and remove the level line - mode.on_fill( - oid, - price=msg['trigger_price'], - arrow_index=get_index(time.time()), - ) - mode.lines.remove_line(uuid=oid) - await mode.on_exec(oid, msg) - - # response to completed 'action' request for buy/sell - elif resp in ( - 'broker_executed', - ): - # right now this is just triggering a system alert - await mode.on_exec(oid, msg) - - if msg['brokerd_msg']['remaining'] == 0: - mode.lines.remove_line(uuid=oid) - - # each clearing tick is responded individually - elif resp in ( - 'broker_filled', - ): - - known_order = book._sent_orders.get(oid) - if not known_order: - log.warning(f'order {oid} is unknown') - continue - - action = known_order.action - details = msg['brokerd_msg'] - - # TODO: some kinda progress system - mode.on_fill( - oid, - price=details['price'], - pointing='up' if action == 'buy' else 'down', - - # TODO: put the actual exchange timestamp - arrow_index=get_index(details['broker_time']), - ) - - # TODO: how should we look this up? - # tracker = mode.trackers[msg['account']] - # tracker.live_pp.fills.append(msg) + # TODO: how should we look this up? + # tracker = mode.trackers[msg['account']] + # tracker.live_pp.fills.append(msg) From 87ed9abefa8d516f102c6722315be3819a309c36 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 8 Aug 2022 13:35:01 -0400 Subject: [PATCH 08/33] WIP playing with a `ChainMap` of messages --- piker/brokers/ib/broker.py | 23 +++- piker/clearing/_ems.py | 263 ++++++++++++++++++++++++------------ piker/clearing/_messages.py | 27 ++-- piker/ui/order_mode.py | 44 ++++-- 4 files changed, 236 insertions(+), 121 deletions(-) diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index b2d6bd0d..96f8572d 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -127,7 +127,8 @@ async def handle_order_requests( oid=request_msg['oid'], symbol=request_msg['symbol'], reason=f'No account found: `{account}` ?', - )) + ) + ) continue client = _accounts2clients.get(account) @@ -495,17 +496,16 @@ async def trades_dialogue( 'BUY': 1, }[order.action] * quant fqsn, _ = con2fqsn(trade.contract) + reqid = order.orderId # TODO: maybe embed a ``BrokerdOrder`` instead # since then we can directly load it on the client # side in the order mode loop? msg = BrokerdStatus( - reqid=order.orderId, - time_ns=time.time_ns(), - account=accounts_def.inverse[order.account], + reqid=reqid, + time_ns=(ts := time.time_ns()), status='submitted', - size=size, - price=order.lmtPrice, + account=accounts_def.inverse[order.account], filled=0, reason='Existing live order', @@ -516,6 +516,17 @@ async def trades_dialogue( broker_details={ 'name': 'ib', 'fqsn': fqsn, + # this is a embedded/boxed order + # msg that can be loaded by the ems + # and for relay to clients. + 'order': BrokerdOrder( + symbol=fqsn, + account=accounts_def.inverse[order.account], + oid=reqid, + time_ns=ts, + size=size, + price=order.lmtPrice, + ), }, ) order_msgs.append(msg) diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 01cc25a2..857460d6 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -18,8 +18,8 @@ In da suit parlances: "Execution management systems" """ +from collections import defaultdict, ChainMap from contextlib import asynccontextmanager -from dataclasses import dataclass, field from math import isnan from pprint import pformat import time @@ -41,9 +41,15 @@ from ..data.types import Struct from .._daemon import maybe_spawn_brokerd from . import _paper_engine as paper from ._messages import ( - Status, Order, - BrokerdCancel, BrokerdOrder, BrokerdOrderAck, BrokerdStatus, - BrokerdFill, BrokerdError, BrokerdPosition, + Order, + Status, + BrokerdCancel, + BrokerdOrder, + BrokerdOrderAck, + BrokerdStatus, + BrokerdFill, + BrokerdError, + BrokerdPosition, ) @@ -90,8 +96,7 @@ def mk_check( ) -@dataclass -class _DarkBook: +class _DarkBook(Struct): ''' EMS-trigger execution book. @@ -116,17 +121,23 @@ class _DarkBook: dict, # cmd / msg type ] ] - ] = field(default_factory=dict) + ] = {} # tracks most recent values per symbol each from data feed lasts: dict[ str, float, - ] = field(default_factory=dict) + ] = {} - # mapping of piker ems order ids to current brokerd order flow message - _ems_entries: dict[str, str] = field(default_factory=dict) - _ems2brokerd_ids: dict[str, str] = field(default_factory=bidict) + # _ems_entries: dict[str, str] = {} + + # mapping of ems dialog ids to msg flow history + _msgflows: defaultdict[ + int, + ChainMap[dict[str, dict]], + ] = defaultdict(ChainMap) + + _ems2brokerd_ids: dict[str, str] = bidict() # XXX: this is in place to prevent accidental positions that are too @@ -240,7 +251,8 @@ async def clear_dark_triggers( # a ``BrokerdOrderAck`` msg including the # allocated unique ``BrokerdOrderAck.reqid`` key # generated by the broker's own systems. - book._ems_entries[oid] = live_req + # book._ems_entries[oid] = live_req + book._msgflows[oid].append(live_req) case _: raise ValueError(f'Invalid dark book entry: {cmd}') @@ -281,8 +293,7 @@ async def clear_dark_triggers( # print(f'execs scan took: {time.time() - start}') -@dataclass -class TradesRelay: +class TradesRelay(Struct): # for now we keep only a single connection open with # each ``brokerd`` for simplicity. @@ -318,7 +329,10 @@ class Router(Struct): # order id to client stream map clients: set[tractor.MsgStream] = set() - dialogues: dict[str, list[tractor.MsgStream]] = {} + dialogues: dict[ + str, + list[tractor.MsgStream] + ] = {} # brokername to trades-dialogues streams with ``brokerd`` actors relays: dict[str, TradesRelay] = {} @@ -341,8 +355,9 @@ class Router(Struct): loglevel: str, ) -> tuple[dict, tractor.MsgStream]: - '''Open and yield ``brokerd`` trades dialogue context-stream if none - already exists. + ''' + Open and yield ``brokerd`` trades dialogue context-stream if + none already exists. ''' relay: TradesRelay = self.relays.get(feed.mod.name) @@ -614,7 +629,8 @@ async def translate_and_relay_brokerd_events( 'reqid': reqid, # brokerd generated order-request id 'oid': oid, # ems order-dialog id } if ( - entry := book._ems_entries.get(oid) + # entry := book._ems_entries.get(oid) + flow := book._msgflows.get(oid) ): # initial response to brokerd order request # if name == 'ack': @@ -637,10 +653,14 @@ async def translate_and_relay_brokerd_events( # cancelled by the ems controlling client before we # received this ack, in which case we relay that cancel # signal **asap** to the backend broker - action = getattr(entry, 'action', None) + action = flow.get('action') + # action = getattr(entry, 'action', None) if action and action == 'cancel': # assign newly providerd broker backend request id - entry.reqid = reqid + flow['reqid'] = reqid + # entry.reqid = reqid + + entry = flow.maps[0] # tell broker to cancel immediately await brokerd_trades_stream.send(entry) @@ -649,7 +669,11 @@ async def translate_and_relay_brokerd_events( # our book -> registered as live flow else: # update the flow with the ack msg - book._ems_entries[oid] = BrokerdOrderAck(**brokerd_msg) + # book._ems_entries[oid] = BrokerdOrderAck(**brokerd_msg) + flow.maps.insert( + 0, + BrokerdOrderAck(**brokerd_msg).to_dict() + ) # no msg to client necessary continue @@ -666,6 +690,7 @@ async def translate_and_relay_brokerd_events( msg = BrokerdError(**brokerd_msg) resp = 'broker_errored' log.error(pformat(msg)) # XXX make one when it's blank? + book._msgflows[oid].maps.insert(0, msg.to_dict()) # TODO: figure out how this will interact with EMS clients # for ex. on an error do we react with a dark orders @@ -686,6 +711,9 @@ async def translate_and_relay_brokerd_events( } if ( oid := book._ems2brokerd_ids.inverse.get(reqid) ): + + # ack = book._ems_entries[oid] + # ack = book._msgflows[oid].maps[0] msg = BrokerdStatus(**brokerd_msg) # TODO: should we flatten out these cases and/or should @@ -704,6 +732,9 @@ async def translate_and_relay_brokerd_events( # since the order dialogue should be done. log.info(f'Execution for {oid} is complete!') + # remove from active flows + book._msgflows.pop(oid) + # just log it else: log.info(f'{broker} filled {msg}') @@ -712,7 +743,21 @@ async def translate_and_relay_brokerd_events( # one of {submitted, cancelled} resp = 'broker_' + msg.status - # unknown valid BrokerdStatus + + # book._ems_entries[oid] = msg + book._msgflows[oid].maps.insert(0, msg.to_dict()) + + # TODO: i wonder if we should just support receiving an + # actual ``BrokerdOrder`` msg here? Is it a bad idea to + # presume that inbound orders on the backend dialog can be + # used to drive order tracking/tracing in the EMS *over* + # a set of backends from some other non-ems owner? + # this will likely feel better once we get open_msg_scope() + # or wtv finished. + + # BrokerdStatus containing an embedded order msg which + # should be loaded as a "pre-existing open order" from the + # brokerd backend. case { 'name': 'status', 'status': status, @@ -724,7 +769,18 @@ async def translate_and_relay_brokerd_events( # to be able to more formally handle multi-player # trading... - if status == 'submitted': + if status != 'submitted': + log.error( + f'Unknown status msg:\n' + f'{pformat(brokerd_msg)}\n' + 'Unable to relay message to client side!?' + ) + + else: + # existing open backend order which we broadcast to + # all currently connected clients. + order_dict = brokerd_msg['broker_details'].pop('order') + order = BrokerdOrder(**order_dict) msg = BrokerdStatus(**brokerd_msg) log.info( f'Relaying existing open order:\n {brokerd_msg}' @@ -734,22 +790,49 @@ async def translate_and_relay_brokerd_events( # may end up with collisions? broker = details['name'] oid = str(reqid) - book._ems_entries[oid] = msg - # attempt to avoid collisions msg.reqid = oid - resp = 'broker_submitted' + + # XXX: MEGA HACK ALERT FOR the dialog entries delivery + # on client connect... + # TODO: fix this garbage.. + msg.broker_details['resp'] = resp = 'broker_submitted' # register this existing broker-side dialog book._ems2brokerd_ids[oid] = reqid + # book._ems_entries[oid] = msg - else: - log.error( - f'Unknown status msg:\n' - f'{pformat(brokerd_msg)}\n' - 'Unable to relay message to client side!?' - ) - continue + # fill in approximate msg flow history + flow = book._msgflows[oid] + flow.maps.insert(0, order.to_dict()) + flow.maps.insert(0, msg.to_dict()) + flow.maps.insert(0, details) + flattened = dict(flow) + # await tractor.breakpoint() + + # fan-out-relay position msgs immediately by + # broadcasting updates on all client streams + for client_stream in router.clients.copy(): + try: + await client_stream.send(flattened) + # Status( + # oid=oid, + # resp=resp, + # time_ns=time.time_ns(), + # broker_reqid=reqid, + # brokerd_msg=flattened, + # ) + # ) + except( + trio.ClosedResourceError, + trio.BrokenResourceError, + ): + router.clients.remove(client_stream) + log.warning( + f'client for {client_stream} was already closed?') + + # don't fall through + continue # BrokerdFill case { @@ -768,58 +851,31 @@ async def translate_and_relay_brokerd_events( raise ValueError(f'Brokerd message {brokerd_msg} is invalid') # retrieve existing live flow - entry = book._ems_entries[oid] + # entry = book._ems_entries[oid] + # assert entry.oid == oid # from when we only stored the first ack + # old_reqid = entry.reqid + # if old_reqid and old_reqid != reqid: + # log.warning( + # f'Brokerd order id change for {oid}:\n' + # f'{old_reqid} -> {reqid}' + # ) - if getattr(entry, 'oid', None): - assert entry.oid == oid - old_reqid = entry.reqid - if old_reqid and old_reqid != reqid: - log.warning( - f'Brokerd order id change for {oid}:\n' - f'{old_reqid} -> {reqid}' + # Create and relay response status message + # to requesting EMS client + try: + ems_client_order_stream = router.dialogues[oid] + await ems_client_order_stream.send( + Status( + oid=oid, + resp=resp, + time_ns=time.time_ns(), + broker_reqid=reqid, + brokerd_msg=msg, ) - - # Create and relay response status message - # to requesting EMS client - try: - ems_client_order_stream = router.dialogues[oid] - await ems_client_order_stream.send( - Status( - oid=oid, - resp=resp, - time_ns=time.time_ns(), - broker_reqid=reqid, - brokerd_msg=msg, - ) - ) - except KeyError: - log.error( - f'Received `brokerd` msg for unknown client oid: {oid}') - - else: - # existing open order relay - assert oid == entry.reqid - - # fan-out-relay position msgs immediately by - # broadcasting updates on all client streams - for client_stream in router.clients.copy(): - try: - await client_stream.send( - Status( - oid=oid, - resp=resp, - time_ns=time.time_ns(), - broker_reqid=reqid, - brokerd_msg=msg, - ) - ) - except( - trio.ClosedResourceError, - trio.BrokenResourceError, - ): - router.clients.remove(client_stream) - log.warning( - f'client for {client_stream} was already closed?') + ) + except KeyError: + log.error( + f'Received `brokerd` msg for unknown client oid: {oid}') # TODO: do we want this to keep things cleaned up? # it might require a special status from brokerd to affirm the @@ -854,7 +910,8 @@ async def process_client_order_cmds( # others who are registered for such order affiliated msgs). client_dialogues[oid] = client_order_stream reqid = dark_book._ems2brokerd_ids.inverse.get(oid) - live_entry = dark_book._ems_entries.get(oid) + # live_entry = dark_book._ems_entries.get(oid) + live_entry = dark_book._msgflows.get(oid) match cmd: # existing live-broker order cancel @@ -862,12 +919,14 @@ async def process_client_order_cmds( 'action': 'cancel', 'oid': oid, } if live_entry: - reqid = live_entry.reqid + # reqid = live_entry.reqid + reqid = live_entry['reqid'] msg = BrokerdCancel( oid=oid, reqid=reqid, time_ns=time.time_ns(), - account=live_entry.account, + # account=live_entry.account, + account=live_entry['account'], ) # NOTE: cancel response will be relayed back in messages @@ -885,6 +944,7 @@ async def process_client_order_cmds( # the order ack does show up later such that the brokerd # order request can be cancelled at that time. dark_book._ems_entries[oid] = msg + live_entry.maps.insert(0, msg.to_dict()) # dark trigger cancel case { @@ -936,7 +996,8 @@ async def process_client_order_cmds( # the only msg will be a ``BrokerdStatus`` # assert live_entry.oid == oid - reqid = live_entry.reqid + # reqid = live_entry.reqid + reqid = live_entry['reqid'] # if we already had a broker order id then # this is likely an order update commmand. log.info(f"Modifying live {broker} order: {reqid}") @@ -971,7 +1032,8 @@ async def process_client_order_cmds( # client, before that ack, when the ack does arrive we # immediately take the reqid from the broker and cancel # that live order asap. - dark_book._ems_entries[oid] = msg + # dark_book._ems_entries[oid] = msg + dark_book._msgflows[oid].maps.insert(0, msg.to_dict()) # dark-order / alert submission case { @@ -1144,12 +1206,35 @@ async def _emsd_main( brokerd_stream = relay.brokerd_dialogue # .clone() + # convert dialogs to status msgs for client delivery + statuses = {} + # for oid, msg in book._ems_entries.items(): + for oid, msgflow in book._msgflows.items(): + # we relay to the client side a msg that contains + # all data flattened from the message history. + # status = msgflow['status'] + flattened = dict(msgflow) + # status = flattened['status'] + flattened.pop('brokerd_msg', None) + statuses[oid] = flattened + # Status( + # oid=oid, + # time_ns=flattened['time_ns'], + # # time_ns=msg.time_ns, + # # resp=f'broker_{msg.status}', + # resp=f'broker_{status}', + # # trigger_price=msg.order.price, + # trigger_price=flattened['price'], + # brokerd_msg=flattened, + # ) + # await tractor.breakpoint() + # signal to client that we're started and deliver # all known pps and accounts for this ``brokerd``. await ems_ctx.started(( relay.positions, list(relay.accounts), - book._ems_entries, + statuses, )) # establish 2-way stream with requesting order-client and diff --git a/piker/clearing/_messages.py b/piker/clearing/_messages.py index eb94b147..ffd46ff2 100644 --- a/piker/clearing/_messages.py +++ b/piker/clearing/_messages.py @@ -67,7 +67,7 @@ class Order(Struct): # determines whether the create execution # will be submitted to the ems or directly to # the backend broker - exec_mode: str # {'dark', 'live', 'paper'} + exec_mode: str # {'dark', 'live'} # -------------- @@ -136,11 +136,14 @@ class BrokerdCancel(Struct): class BrokerdOrder(Struct): - action: str # {buy, sell} oid: str account: str time_ns: int + # TODO: if we instead rely on a +ve/-ve size to determine + # the action we more or less don't need this field right? + action: str = '' # {buy, sell} + # "broker request id": broker specific/internal order id if this is # None, creates a new order otherwise if the id is valid the backend # api must modify the existing matching order. If the broker allows @@ -149,7 +152,7 @@ class BrokerdOrder(Struct): # field reqid: Optional[Union[int, str]] = None - symbol: str # symbol. ? + symbol: str # fqsn price: float size: float @@ -183,25 +186,21 @@ class BrokerdStatus(Struct): reqid: Union[int, str] time_ns: int - # XXX: should be best effort set for every update - account: str = '' - # TODO: instead (ack, pending, open, fill, clos(ed), cancelled) # { - # 'submitted', - # 'cancelled', - # 'filled', + # 'submitted', # open + # 'cancelled', # canceled + # 'filled', # closed # } status: str - - # +ve is buy, -ve is sell - size: float = 0.0 - price: float = 0.0 - + account: str filled: float = 0.0 reason: str = '' remaining: float = 0.0 + external: bool = False + # order: Optional[BrokerdOrder] = None + # XXX: better design/name here? # flag that can be set to indicate a message for an order # event that wasn't originated by piker's emsd (eg. some external diff --git a/piker/ui/order_mode.py b/piker/ui/order_mode.py index b8dd37f9..b9d23ab3 100644 --- a/piker/ui/order_mode.py +++ b/piker/ui/order_mode.py @@ -49,9 +49,14 @@ from ._position import ( SettingsPane, ) from ._forms import FieldsForm -# from ._label import FormatLabel from ._window import MultiStatus -from ..clearing._messages import Order, BrokerdPosition +from ..clearing._messages import ( + Order, + Status, + # BrokerdOrder, + # BrokerdStatus, + BrokerdPosition, +) from ._forms import open_form_input_handling @@ -519,22 +524,36 @@ class OrderMode: def load_unknown_dialog_from_msg( self, + # status: Status, msg: dict, ) -> OrderDialog: oid = str(msg['oid']) - size = msg['brokerd_msg']['size'] + # oid = str(status.oid) + + # bstatus = BrokerdStatus(**msg.brokerd_msg) + # NOTE: the `.order` attr **must** be set with the + # equivalent order msg in order to be loaded. + # border = BrokerdOrder(**bstatus.broker_details['order']) + # msg = msg['brokerd_msg'] + + # size = border.size + size = msg['size'] if size >= 0: action = 'buy' else: action = 'sell' - acct = msg['brokerd_msg']['account'] - price = msg['brokerd_msg']['price'] - deats = msg['brokerd_msg']['broker_details'] + # acct = border.account + # price = border.price + # price = msg['brokerd_msg']['price'] + symbol = msg['symbol'] + deats = msg['broker_details'] + brokername = deats['name'] fqsn = ( - deats['fqsn'] + '.' + deats['name'] + # deats['fqsn'] + '.' + deats['name'] + symbol + '.' + brokername ) symbol = Symbol.from_fqsn( fqsn=fqsn, @@ -543,11 +562,11 @@ class OrderMode: # map to order composite-type order = Order( action=action, - price=price, - account=acct, + price=msg['price'], + account=msg['account'], size=size, symbol=symbol, - brokers=symbol.brokers, + brokers=[brokername], oid=oid, exec_mode='live', # dark or live ) @@ -808,8 +827,8 @@ async def open_order_mode( # HACK ALERT: ensure a resp field is filled out since # techincally the call below expects a ``Status``. TODO: # parse into proper ``Status`` equivalents ems-side? - msg.setdefault('resp', msg['broker_details']['resp']) - msg.setdefault('oid', msg['broker_details']['oid']) + # msg.setdefault('resp', msg['broker_details']['resp']) + # msg.setdefault('oid', msg['broker_details']['oid']) msg['brokerd_msg'] = msg await process_trade_msg( @@ -892,6 +911,7 @@ async def process_trade_msg( log.warning( f'received msg for untracked dialog:\n{fmsg}' ) + # dialog = mode.load_unknown_dialog_from_msg(Status(**msg)) dialog = mode.load_unknown_dialog_from_msg(msg) # record message to dialog tracking From 7fa9dbf86940106796793c56156804a9e7aa181d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 10 Aug 2022 00:16:08 -0400 Subject: [PATCH 09/33] Add full EMS order-dialog (re-)load support! This includes darks, lives and alerts with all connecting clients being broadcast all existing order-flow dialog states. Obviously for now darks and alerts only live as long as the `emsd` actor lifetime (though we will store these in local state eventually) and "live" orders have lifetimes managed by their respective backend broker. The details of this change-set is extensive, so here we go.. Messaging schema: - change the messaging `Status` status-key set to: `resp: Literal['pending', 'open', 'dark_open', 'triggered', 'closed', 'fill', 'canceled', 'error']` which better reflects the semantics of order lifetimes and was partially inspired by the status keys `kraken` provides for their order-entry API. The prior key set was based on `ib`'s horrible semantics which sound like they're right out of the 80s.. Also, we reflect this same set in the `BrokerdStatus` msg and likely we'll just get rid of the separate brokerd-dialog side type eventually. - use `Literal` type annots for statuses where applicable and as they are supported by `msgspec`. - add additional optional `Status` fields: -`req: Order` to allow each status msg to optionally ref its commanding order-request msg allowing at least a request-response style implicit tracing in all response msgs. -`src: str` tag string to show the source of the msg. -`reqid: str | int` such that the ems can relay the `brokerd` request id both to the client side and have one spot to look up prior status msgs and - draft a (unused/commented) `Dialog` type which can be eventually used at all EMS endpoints to track msg-flow states EMS engine adjustments/rework: - use the new status key set throughout and expect `BrokerdStatus` msgs to use the same new schema as `Status`. - add a `_DarkBook._active: dict[str, Status]` table which is now used for all per-leg-dialog associations and order flow state tracking allowing for the both the brokerd-relay and client-request handler loops to read/write the same msg-table and provides for delivering the overall EMS-active-orders state to newly/re-connecting clients with minimal processing; this table replaces what the `._ems_entries` table from prior. - add `Router.client_broadcast()` to send a msg to all currently connected peers. - a variety of msg handler block logic tweaks including more `case:` blocks to be both flatter and improve explicitness: - for the relay loop move all `Status` msg update and sending to within each block instead of a fallthrough case plus hard-to-follow state logic. - add a specific case for unhandled backend status keys and just log them. - pop alerts from `._active` immediately once triggered. - where possible mutate status msgs fields over instantiating new ones. - insert and expect `Order` instances in the dark clearing loop and adjust `case:` blocks accordingly. - tag `dark_open` and `triggered` statuses as sourced from the ems. - drop all the `ChainMap` stuff for now; we're going to make our own `Dialog` type for this purpose.. Order mode rework: - always parse the `Status` msg and use match syntax cases with object patterns, hackily assign the `.req` in many blocks to work around not yet having proper on-the-wire decoding yet. - make `.load_unknown_dialog_from_msg()` expect a `Status` with boxed `.req: Order` as input. - change `OrderDialog` -> `Dialog` in prep for a general purpose type of the same name. `ib` backend order loading support: - do "closed" status detection inside the msg-relay loop instead of expecting the ems to do this.. - add an attempt to cancel inactive orders by scheduling cancel submissions continually (no idea if this works). - add a status map to go from the 80s keys to our new set. - deliver `Status` msgs with an embedded `Order` for existing live order loading and make sure to try an get the source exchange info (instead of SMART). Paper engine ported to match: - use new status keys in `BrokerdStatus` msgs - use `match:` syntax in request handler loop --- piker/brokers/ib/broker.py | 96 ++++--- piker/clearing/_ems.py | 485 +++++++++++++++----------------- piker/clearing/_messages.py | 151 ++++++---- piker/clearing/_paper_engine.py | 163 ++++++----- piker/ui/order_mode.py | 264 +++++++++-------- 5 files changed, 605 insertions(+), 554 deletions(-) diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index 96f8572d..b7c121e7 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -60,6 +60,8 @@ from piker.pp import ( ) from piker.log import get_console_log from piker.clearing._messages import ( + Order, + Status, BrokerdOrder, BrokerdOrderAck, BrokerdStatus, @@ -184,7 +186,7 @@ async def handle_order_requests( ) ) - elif action == 'cancel': + if action == 'cancel': msg = BrokerdCancel(**request_msg) client.submit_cancel(reqid=int(msg.reqid)) @@ -491,43 +493,43 @@ async def trades_dialogue( order = trade.order quant = trade.order.totalQuantity + action = order.action.lower() size = { - 'SELL': -1, - 'BUY': 1, - }[order.action] * quant - fqsn, _ = con2fqsn(trade.contract) + 'sell': -1, + 'buy': 1, + }[action] * quant + con = trade.contract + + # TODO: in the case of the SMART venue (aka ib's + # router-clearing sys) we probably should handle + # showing such orders overtop of the fqsn for the + # primary exchange, how to map this easily is going + # to be a bit tricky though? + deats = await proxy.con_deats(contracts=[con]) + fqsn = list(deats)[0] + reqid = order.orderId # TODO: maybe embed a ``BrokerdOrder`` instead # since then we can directly load it on the client # side in the order mode loop? - msg = BrokerdStatus( + msg = Status( + time_ns=time.time_ns(), + resp='open', + oid=str(reqid), reqid=reqid, - time_ns=(ts := time.time_ns()), - status='submitted', - account=accounts_def.inverse[order.account], - filled=0, - reason='Existing live order', - # this seems to not be necessarily up to date in - # the execDetails event.. so we have to send it - # here I guess? - remaining=quant, - broker_details={ - 'name': 'ib', - 'fqsn': fqsn, - # this is a embedded/boxed order - # msg that can be loaded by the ems - # and for relay to clients. - 'order': BrokerdOrder( - symbol=fqsn, - account=accounts_def.inverse[order.account], - oid=reqid, - time_ns=ts, - size=size, - price=order.lmtPrice, - ), - }, + # embedded order info + req=Order( + action=action, + exec_mode='live', + oid=str(reqid), + symbol=fqsn, + account=accounts_def.inverse[order.account], + price=order.lmtPrice, + size=size, + ), + src='ib', ) order_msgs.append(msg) @@ -686,6 +688,7 @@ async def trades_dialogue( # allocate event relay tasks for each client connection n.start_soon( deliver_trade_events, + n, trade_event_stream, ems_stream, accounts_def, @@ -779,6 +782,7 @@ _statuses: dict[str, str] = { async def deliver_trade_events( + nurse: trio.Nursery, trade_event_stream: trio.MemoryReceiveChannel, ems_stream: tractor.MsgStream, accounts_def: dict[str, str], # eg. `'ib.main'` -> `'DU999999'` @@ -834,14 +838,35 @@ async def deliver_trade_events( # unwrap needed data from ib_insync internal types trade: Trade = item status: OrderStatus = trade.orderStatus - status_key = status.status.lower() + ib_status_key = status.status.lower() + + acctid = accounts_def.inverse[trade.order.account] # double check there is no error when # cancelling.. gawwwd - if status_key == 'cancelled': + if ib_status_key == 'cancelled': last_log = trade.log[-1] if last_log.message: - status_key = trade.log[-2].status + ib_status_key = trade.log[-2].status + + elif ib_status_key == 'inactive': + async def sched_cancel(): + log.warning( + 'OH GAWD an inactive order..scheduling a cancel\n' + f'{pformat(item)}' + ) + proxy = proxies[acctid] + await proxy.submit_cancel(reqid=trade.order.orderId) + await trio.sleep(1) + nurse.start_soon(sched_cancel) + + nurse.start_soon(sched_cancel) + + status_key = _statuses.get(ib_status_key) or ib_status_key + + remaining = status.remaining + if remaining == 0: + status_key = 'closed' # skip duplicate filled updates - we get the deats # from the execution details event @@ -859,7 +884,7 @@ async def deliver_trade_events( # this seems to not be necessarily up to date in the # execDetails event.. so we have to send it here I guess? - remaining=status.remaining, + remaining=remaining, broker_details={'name': 'ib'}, ) @@ -1002,9 +1027,8 @@ async def deliver_trade_events( cid, msg = pack_position(item) log.info(f'New IB position msg: {msg}') - # acctid = msg.account = accounts_def.inverse[msg.account] # cuck ib and it's shitty fifo sys for pps! - # await ems_stream.send(msg) + continue case 'event': diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 857460d6..0411f026 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -27,6 +27,7 @@ from typing import ( AsyncIterator, Any, Callable, + Optional, ) from bidict import bidict @@ -43,9 +44,10 @@ from . import _paper_engine as paper from ._messages import ( Order, Status, + # Cancel, BrokerdCancel, BrokerdOrder, - BrokerdOrderAck, + # BrokerdOrderAck, BrokerdStatus, BrokerdFill, BrokerdError, @@ -130,6 +132,7 @@ class _DarkBook(Struct): ] = {} # _ems_entries: dict[str, str] = {} + _active: dict = {} # mapping of ems dialog ids to msg flow history _msgflows: defaultdict[ @@ -192,6 +195,7 @@ async def clear_dark_triggers( for oid, ( pred, tf, + # TODO: send this msg instead? cmd, percent_away, abs_diff_away @@ -211,30 +215,29 @@ async def clear_dark_triggers( # majority of iterations will be non-matches continue + brokerd_msg: Optional[BrokerdOrder] = None match cmd: # alert: nothing to do but relay a status # back to the requesting ems client - case { - 'action': 'alert', - }: - resp = 'alert_triggered' + case Order(action='alert'): + resp = 'triggered' # executable order submission - case { - 'action': action, - 'symbol': symbol, - 'account': account, - 'size': size, - }: + case Order( + action=action, + symbol=symbol, + account=account, + size=size, + ): bfqsn: str = symbol.replace(f'.{broker}', '') submit_price = price + abs_diff_away - resp = 'dark_triggered' # hidden on client-side + resp = 'triggered' # hidden on client-side log.info( f'Dark order triggered for price {price}\n' f'Submitting order @ price {submit_price}') - live_req = BrokerdOrder( + brokerd_msg = BrokerdOrder( action=action, oid=oid, account=account, @@ -243,7 +246,8 @@ async def clear_dark_triggers( price=submit_price, size=size, ) - await brokerd_orders_stream.send(live_req) + + await brokerd_orders_stream.send(brokerd_msg) # mark this entry as having sent an order # request. the entry will be replaced once the @@ -252,18 +256,18 @@ async def clear_dark_triggers( # allocated unique ``BrokerdOrderAck.reqid`` key # generated by the broker's own systems. # book._ems_entries[oid] = live_req - book._msgflows[oid].append(live_req) + # book._msgflows[oid].maps.insert(0, live_req) case _: raise ValueError(f'Invalid dark book entry: {cmd}') # fallthrough logic - resp = Status( + status = Status( oid=oid, # ems dialog id time_ns=time.time_ns(), resp=resp, - trigger_price=price, - brokerd_msg=cmd, + req=cmd, + brokerd_msg=brokerd_msg, ) # remove exec-condition from set @@ -274,9 +278,18 @@ async def clear_dark_triggers( f'pred for {oid} was already removed!?' ) + # update actives + if cmd.action == 'alert': + # don't register the alert status (so it won't + # be reloaded by clients) since it's now + # complete / closed. + book._active.pop(oid) + else: + book._active[oid] = status + # send response to client-side try: - await ems_client_order_stream.send(resp) + await ems_client_order_stream.send(status) except ( trio.ClosedResourceError, ): @@ -396,6 +409,22 @@ class Router(Struct): relay.consumers -= 1 + async def client_broadcast( + self, + msg: dict, + + ) -> None: + for client_stream in self.clients.copy(): + try: + await client_stream.send(msg) + except( + trio.ClosedResourceError, + trio.BrokenResourceError, + ): + self.clients.remove(client_stream) + log.warning( + f'client for {client_stream} was already closed?') + _router: Router = None @@ -570,8 +599,7 @@ async def translate_and_relay_brokerd_events( broker ems 'error' -> log it locally (for now) - 'status' -> relabel as 'broker_', if complete send 'executed' - 'fill' -> 'broker_filled' + ('status' | 'fill'} -> relayed through see ``Status`` msg type. Currently handled status values from IB: {'presubmitted', 'submitted', 'cancelled', 'inactive'} @@ -610,31 +638,16 @@ async def translate_and_relay_brokerd_events( # fan-out-relay position msgs immediately by # broadcasting updates on all client streams - for client_stream in router.clients.copy(): - try: - await client_stream.send(pos_msg) - except( - trio.ClosedResourceError, - trio.BrokenResourceError, - ): - router.clients.remove(client_stream) - log.warning( - f'client for {client_stream} was already closed?') - + await router.client_broadcast(pos_msg) continue # BrokerdOrderAck + # initial response to brokerd order request case { 'name': 'ack', 'reqid': reqid, # brokerd generated order-request id 'oid': oid, # ems order-dialog id - } if ( - # entry := book._ems_entries.get(oid) - flow := book._msgflows.get(oid) - ): - # initial response to brokerd order request - # if name == 'ack': - + }: # register the brokerd request id (that was generated # / created internally by the broker backend) with our # local ems order id for reverse lookup later. @@ -649,31 +662,24 @@ async def translate_and_relay_brokerd_events( # new order which has not yet be registered into the # local ems book, insert it now and handle 2 cases: - # - the order has previously been requested to be + # 1. the order has previously been requested to be # cancelled by the ems controlling client before we # received this ack, in which case we relay that cancel # signal **asap** to the backend broker - action = flow.get('action') - # action = getattr(entry, 'action', None) - if action and action == 'cancel': + # status = book._active.get(oid) + status = book._active[oid] + req = status.req + if req and req.action == 'cancel': # assign newly providerd broker backend request id - flow['reqid'] = reqid - # entry.reqid = reqid + # and tell broker to cancel immediately + status.reqid = reqid + await brokerd_trades_stream.send(req) - entry = flow.maps[0] - - # tell broker to cancel immediately - await brokerd_trades_stream.send(entry) - - # - the order is now active and will be mirrored in + # 2. the order is now active and will be mirrored in # our book -> registered as live flow else: - # update the flow with the ack msg - # book._ems_entries[oid] = BrokerdOrderAck(**brokerd_msg) - flow.maps.insert( - 0, - BrokerdOrderAck(**brokerd_msg).to_dict() - ) + # TODO: should we relay this ack state? + status.resp = 'pending' # no msg to client necessary continue @@ -684,13 +690,10 @@ async def translate_and_relay_brokerd_events( 'oid': oid, # ems order-dialog id 'reqid': reqid, # brokerd generated order-request id 'symbol': sym, - 'broker_details': details, - # 'reason': reason, - }: + } if status_msg := book._active.get(oid): + msg = BrokerdError(**brokerd_msg) - resp = 'broker_errored' log.error(pformat(msg)) # XXX make one when it's blank? - book._msgflows[oid].maps.insert(0, msg.to_dict()) # TODO: figure out how this will interact with EMS clients # for ex. on an error do we react with a dark orders @@ -699,141 +702,132 @@ async def translate_and_relay_brokerd_events( # some unexpected failure - something we need to think more # about. In most default situations, with composed orders # (ex. brackets), most brokers seem to use a oca policy. + ems_client_order_stream = router.dialogues[oid] + status_msg.resp = 'error' + status_msg.brokerd_msg = msg + book._active[oid] = status_msg + await ems_client_order_stream.send(status_msg) # BrokerdStatus case { 'name': 'status', 'status': status, 'reqid': reqid, # brokerd generated order-request id - # TODO: feels like the wrong msg for this field? - 'remaining': remaining, } if ( - oid := book._ems2brokerd_ids.inverse.get(reqid) + (oid := book._ems2brokerd_ids.inverse.get(reqid)) + and status in ( + 'canceled', + 'open', + 'closed', + ) ): - - # ack = book._ems_entries[oid] - # ack = book._msgflows[oid].maps[0] msg = BrokerdStatus(**brokerd_msg) - # TODO: should we flatten out these cases and/or should - # they maybe even eventually be separate messages? - if status == 'cancelled': + # TODO: maybe pack this into a composite type that + # contains both the IPC stream as well the + # msg-chain/dialog. + ems_client_order_stream = router.dialogues[oid] + status_msg = book._active[oid] + status_msg.resp = status + + # retrieve existing live flow + old_reqid = status_msg.reqid + if old_reqid and old_reqid != reqid: + log.warning( + f'Brokerd order id change for {oid}:\n' + f'{old_reqid} -> {reqid}' + ) + + status_msg.reqid = reqid # THIS LINE IS CRITICAL! + status_msg.brokerd_msg = msg + status_msg.src = msg.broker_details['name'] + await ems_client_order_stream.send(status_msg) + + if status == 'closed': + log.info(f'Execution for {oid} is complete!') + status_msg = book._active.pop(oid) + + elif status == 'canceled': log.info(f'Cancellation for {oid} is complete!') - if status == 'filled': - # conditional execution is fully complete, no more - # fills for the noted order - if not remaining: - - resp = 'broker_executed' - - # be sure to pop this stream from our dialogue set - # since the order dialogue should be done. - log.info(f'Execution for {oid} is complete!') - - # remove from active flows - book._msgflows.pop(oid) - + else: # open + # relayed from backend but probably not handled so # just log it - else: - log.info(f'{broker} filled {msg}') + log.info(f'{broker} opened order {msg}') - else: - # one of {submitted, cancelled} - resp = 'broker_' + msg.status - - - # book._ems_entries[oid] = msg - book._msgflows[oid].maps.insert(0, msg.to_dict()) - - # TODO: i wonder if we should just support receiving an - # actual ``BrokerdOrder`` msg here? Is it a bad idea to - # presume that inbound orders on the backend dialog can be - # used to drive order tracking/tracing in the EMS *over* - # a set of backends from some other non-ems owner? - # this will likely feel better once we get open_msg_scope() - # or wtv finished. - - # BrokerdStatus containing an embedded order msg which + # ``Status`` containing an embedded order msg which # should be loaded as a "pre-existing open order" from the # brokerd backend. case { 'name': 'status', - 'status': status, + 'resp': status, 'reqid': reqid, # brokerd generated order-request id - 'broker_details': details, }: - # TODO: we probably want some kind of "tagging" system - # for external order submissions like this eventually - # to be able to more formally handle multi-player - # trading... - - if status != 'submitted': + if ( + status != 'open' + ): + # TODO: check for an oid we might know since it was + # registered from a previous order/status load? log.error( - f'Unknown status msg:\n' + f'Unknown/transient status msg:\n' f'{pformat(brokerd_msg)}\n' 'Unable to relay message to client side!?' ) + # TODO: we probably want some kind of "tagging" system + # for external order submissions like this eventually + # to be able to more formally handle multi-player + # trading... else: # existing open backend order which we broadcast to # all currently connected clients. - order_dict = brokerd_msg['broker_details'].pop('order') - order = BrokerdOrder(**order_dict) - msg = BrokerdStatus(**brokerd_msg) log.info( f'Relaying existing open order:\n {brokerd_msg}' ) # use backend request id as our ems id though this # may end up with collisions? - broker = details['name'] - oid = str(reqid) - # attempt to avoid collisions - msg.reqid = oid + status_msg = Status(**brokerd_msg) + order = Order(**status_msg.req) + assert order.price and order.size + status_msg.req = order - # XXX: MEGA HACK ALERT FOR the dialog entries delivery - # on client connect... - # TODO: fix this garbage.. - msg.broker_details['resp'] = resp = 'broker_submitted' + assert status_msg.src # source tag? + oid = str(status_msg.reqid) + + # attempt to avoid collisions + status_msg.reqid = oid + assert status_msg.resp == 'open' # register this existing broker-side dialog book._ems2brokerd_ids[oid] = reqid - # book._ems_entries[oid] = msg - - # fill in approximate msg flow history - flow = book._msgflows[oid] - flow.maps.insert(0, order.to_dict()) - flow.maps.insert(0, msg.to_dict()) - flow.maps.insert(0, details) - flattened = dict(flow) - # await tractor.breakpoint() + book._active[oid] = status_msg # fan-out-relay position msgs immediately by # broadcasting updates on all client streams - for client_stream in router.clients.copy(): - try: - await client_stream.send(flattened) - # Status( - # oid=oid, - # resp=resp, - # time_ns=time.time_ns(), - # broker_reqid=reqid, - # brokerd_msg=flattened, - # ) - # ) - except( - trio.ClosedResourceError, - trio.BrokenResourceError, - ): - router.clients.remove(client_stream) - log.warning( - f'client for {client_stream} was already closed?') + await router.client_broadcast(status_msg) # don't fall through continue + # TOO FAST ``BrokerdStatus`` that arrives + # before the ``BrokerdAck``. + case { + # XXX: sometimes there is a race with the backend (like + # `ib` where the pending stauts will be related before + # the ack, in which case we just ignore the faster + # pending msg and wait for our expected ack to arrive + # later (i.e. the first block below should enter). + 'name': 'status', + 'status': status, + 'reqid': reqid, + }: + log.warning( + 'Unhandled broker status:\n' + f'{pformat(brokerd_msg)}\n' + ) + # BrokerdFill case { 'name': 'fill', @@ -843,40 +837,18 @@ async def translate_and_relay_brokerd_events( oid := book._ems2brokerd_ids.inverse.get(reqid) ): # proxy through the "fill" result(s) + log.info(f'Fill for {oid} cleared with:\n{pformat(msg)}') msg = BrokerdFill(**brokerd_msg) - resp = 'broker_filled' - log.info(f'\nFill for {oid} cleared with:\n{pformat(resp)}') + ems_client_order_stream = router.dialogues[oid] + status_msg = book._active[oid] + status_msg.resp = 'fill' + status_msg.reqid = reqid + status_msg.brokerd_msg = msg + await ems_client_order_stream.send(status_msg) case _: raise ValueError(f'Brokerd message {brokerd_msg} is invalid') - # retrieve existing live flow - # entry = book._ems_entries[oid] - # assert entry.oid == oid # from when we only stored the first ack - # old_reqid = entry.reqid - # if old_reqid and old_reqid != reqid: - # log.warning( - # f'Brokerd order id change for {oid}:\n' - # f'{old_reqid} -> {reqid}' - # ) - - # Create and relay response status message - # to requesting EMS client - try: - ems_client_order_stream = router.dialogues[oid] - await ems_client_order_stream.send( - Status( - oid=oid, - resp=resp, - time_ns=time.time_ns(), - broker_reqid=reqid, - brokerd_msg=msg, - ) - ) - except KeyError: - log.error( - f'Received `brokerd` msg for unknown client oid: {oid}') - # TODO: do we want this to keep things cleaned up? # it might require a special status from brokerd to affirm the # flow is complete? @@ -910,23 +882,27 @@ async def process_client_order_cmds( # others who are registered for such order affiliated msgs). client_dialogues[oid] = client_order_stream reqid = dark_book._ems2brokerd_ids.inverse.get(oid) - # live_entry = dark_book._ems_entries.get(oid) - live_entry = dark_book._msgflows.get(oid) + + # any dark/live status which is current + status = dark_book._active.get(oid) match cmd: # existing live-broker order cancel case { 'action': 'cancel', 'oid': oid, - } if live_entry: - # reqid = live_entry.reqid - reqid = live_entry['reqid'] - msg = BrokerdCancel( + } if ( + (status := dark_book._active.get(oid)) + and status.resp in ('open', 'pending') + ): + reqid = status.reqid + order = status.req + to_brokerd_msg = BrokerdCancel( oid=oid, reqid=reqid, time_ns=time.time_ns(), # account=live_entry.account, - account=live_entry['account'], + account=order.account, ) # NOTE: cancel response will be relayed back in messages @@ -936,39 +912,52 @@ async def process_client_order_cmds( log.info( f'Submitting cancel for live order {reqid}' ) - await brokerd_order_stream.send(msg) + await brokerd_order_stream.send(to_brokerd_msg) else: # this might be a cancel for an order that hasn't been # acked yet by a brokerd, so register a cancel for when # the order ack does show up later such that the brokerd # order request can be cancelled at that time. - dark_book._ems_entries[oid] = msg - live_entry.maps.insert(0, msg.to_dict()) + # dark_book._ems_entries[oid] = msg + # special case for now.. + status.req = to_brokerd_msg # dark trigger cancel case { 'action': 'cancel', 'oid': oid, - } if not live_entry: - # try: + } if ( + status and status.resp == 'dark_open' + # or status and status.req + ): # remove from dark book clearing - dark_book.orders[symbol].pop(oid, None) + entry = dark_book.orders[symbol].pop(oid, None) + if entry: + ( + pred, + tickfilter, + cmd, + percent_away, + abs_diff_away + ) = entry - # tell client side that we've cancelled the - # dark-trigger order - await client_order_stream.send( - Status( - resp='dark_cancelled', - oid=oid, - time_ns=time.time_ns(), - ) - ) - # de-register this client dialogue - router.dialogues.pop(oid) + # tell client side that we've cancelled the + # dark-trigger order + status.resp = 'canceled' + status.req = cmd - # except KeyError: - # log.exception(f'No dark order for {symbol}?') + await client_order_stream.send(status) + # de-register this client dialogue + router.dialogues.pop(oid) + dark_book._active.pop(oid) + + else: + log.exception(f'No dark order for {symbol}?') + + # TODO: eventually we should be receiving + # this struct on the wire unpacked in a scoped protocol + # setup with ``tractor``. # live order submission case { @@ -977,11 +966,9 @@ async def process_client_order_cmds( 'price': trigger_price, 'size': size, 'action': ('buy' | 'sell') as action, - 'exec_mode': 'live', + 'exec_mode': ('live' | 'paper'), }: - # TODO: eventually we should be receiving - # this struct on the wire unpacked in a scoped protocol - # setup with ``tractor``. + # TODO: relay this order msg directly? req = Order(**cmd) broker = req.brokers[0] @@ -990,17 +977,13 @@ async def process_client_order_cmds( # aren't expectig their own name, but should they? sym = fqsn.replace(f'.{broker}', '') - if live_entry is not None: - # sanity check on emsd id, but it won't work - # for pre-existing orders that we load since - # the only msg will be a ``BrokerdStatus`` - # assert live_entry.oid == oid - - # reqid = live_entry.reqid - reqid = live_entry['reqid'] + if status is not None: # if we already had a broker order id then # this is likely an order update commmand. log.info(f"Modifying live {broker} order: {reqid}") + reqid = status.reqid + status.req = req + status.resp = 'pending' msg = BrokerdOrder( oid=oid, # no ib support for oids... @@ -1017,6 +1000,18 @@ async def process_client_order_cmds( account=req.account, ) + if status is None: + status = Status( + oid=oid, + reqid=reqid, + resp='pending', + time_ns=time.time_ns(), + brokerd_msg=msg, + req=req, + ) + + dark_book._active[oid] = status + # send request to backend # XXX: the trades data broker response loop # (``translate_and_relay_brokerd_events()`` above) will @@ -1032,8 +1027,7 @@ async def process_client_order_cmds( # client, before that ack, when the ack does arrive we # immediately take the reqid from the broker and cancel # that live order asap. - # dark_book._ems_entries[oid] = msg - dark_book._msgflows[oid].maps.insert(0, msg.to_dict()) + # dark_book._msgflows[oid].maps.insert(0, msg.to_dict()) # dark-order / alert submission case { @@ -1049,9 +1043,11 @@ async def process_client_order_cmds( # submit order to local EMS book and scan loop, # effectively a local clearing engine, which # scans for conditions and triggers matching executions - exec_mode in ('dark', 'paper') + exec_mode in ('dark',) or action == 'alert' ): + req = Order(**cmd) + # Auto-gen scanner predicate: # we automatically figure out what the alert check # condition should be based on the current first @@ -1098,23 +1094,25 @@ async def process_client_order_cmds( )[oid] = ( pred, tickfilter, - cmd, + req, percent_away, abs_diff_away ) - resp = 'dark_submitted' + resp = 'dark_open' # alerts have special msgs to distinguish - if action == 'alert': - resp = 'alert_submitted' + # if action == 'alert': + # resp = 'open' - await client_order_stream.send( - Status( - resp=resp, - oid=oid, - time_ns=time.time_ns(), - ) + status = Status( + resp=resp, + oid=oid, + time_ns=time.time_ns(), + req=req, + src='dark', ) + dark_book._active[oid] = status + await client_order_stream.send(status) @tractor.context @@ -1206,35 +1204,12 @@ async def _emsd_main( brokerd_stream = relay.brokerd_dialogue # .clone() - # convert dialogs to status msgs for client delivery - statuses = {} - # for oid, msg in book._ems_entries.items(): - for oid, msgflow in book._msgflows.items(): - # we relay to the client side a msg that contains - # all data flattened from the message history. - # status = msgflow['status'] - flattened = dict(msgflow) - # status = flattened['status'] - flattened.pop('brokerd_msg', None) - statuses[oid] = flattened - # Status( - # oid=oid, - # time_ns=flattened['time_ns'], - # # time_ns=msg.time_ns, - # # resp=f'broker_{msg.status}', - # resp=f'broker_{status}', - # # trigger_price=msg.order.price, - # trigger_price=flattened['price'], - # brokerd_msg=flattened, - # ) - # await tractor.breakpoint() - # signal to client that we're started and deliver # all known pps and accounts for this ``brokerd``. await ems_ctx.started(( relay.positions, list(relay.accounts), - statuses, + book._active, )) # establish 2-way stream with requesting order-client and diff --git a/piker/clearing/_messages.py b/piker/clearing/_messages.py index ffd46ff2..2c0d95c8 100644 --- a/piker/clearing/_messages.py +++ b/piker/clearing/_messages.py @@ -18,56 +18,99 @@ Clearing sub-system message and protocols. """ -from typing import Optional, Union +from collections import ( + ChainMap, + deque, +) +from typing import ( + Optional, + Literal, + Union, +) from ..data._source import Symbol from ..data.types import Struct +# TODO: a composite for tracking msg flow on 2-legged +# dialogs. +# class Dialog(ChainMap): +# ''' +# Msg collection abstraction to easily track the state changes of +# a msg flow in one high level, query-able and immutable construct. + +# The main use case is to query data from a (long-running) +# msg-transaction-sequence + + +# ''' +# def update( +# self, +# msg, +# ) -> None: +# self.maps.insert(0, msg.to_dict()) + +# def flatten(self) -> dict: +# return dict(self) + + # TODO: ``msgspec`` stuff worth paying attention to: # - schema evolution: https://jcristharif.com/msgspec/usage.html#schema-evolution +# - for eg. ``BrokerdStatus``, instead just have separate messages? # - use literals for a common msg determined by diff keys? # - https://jcristharif.com/msgspec/usage.html#literal -# - for eg. ``BrokerdStatus``, instead just have separate messages? # -------------- # Client -> emsd # -------------- -class Cancel(Struct): - '''Cancel msg for removing a dark (ems triggered) or - broker-submitted (live) trigger/order. - - ''' - action: str = 'cancel' - oid: str # uuid4 - symbol: str - - class Order(Struct): - # TODO: use ``msgspec.Literal`` + # TODO: ideally we can combine these 2 fields into + # 1 and just use the size polarity to determine a buy/sell. + # i would like to see this become more like # https://jcristharif.com/msgspec/usage.html#literal - action: str # {'buy', 'sell', 'alert'} + # action: Literal[ + # 'live', + # 'dark', + # 'alert', + # ] + + action: Literal[ + 'buy', + 'sell', + 'alert', + ] + # determines whether the create execution + # will be submitted to the ems or directly to + # the backend broker + exec_mode: Literal[ + 'dark', + 'live', + # 'paper', no right? + ] + # internal ``emdsd`` unique "order id" oid: str # uuid4 symbol: Union[str, Symbol] account: str # should we set a default as '' ? price: float - # TODO: could we drop the ``.action`` field above and instead just - # use +/- values here? Would make the msg smaller at the sake of a - # teensie fp precision? - size: float - brokers: list[str] + size: float # -ve is "sell", +ve is "buy" - # Assigned once initial ack is received - # ack_time_ns: Optional[int] = None + brokers: Optional[list[str]] = [] - # determines whether the create execution - # will be submitted to the ems or directly to - # the backend broker - exec_mode: str # {'dark', 'live'} + +class Cancel(Struct): + ''' + Cancel msg for removing a dark (ems triggered) or + broker-submitted (live) trigger/order. + + ''' + action: str = 'cancel' + oid: str # uuid4 + symbol: str + req: Optional[Order] = None # -------------- @@ -79,35 +122,30 @@ class Order(Struct): class Status(Struct): name: str = 'status' - oid: str # uuid4 time_ns: int - # { - # 'dark_submitted', - # 'dark_cancelled', - # 'dark_triggered', + resp: Literal[ + 'pending', # acked but not yet open + 'open', + 'dark_open', # live in dark loop + 'triggered', # dark-submitted to brokerd-backend + 'closed', # fully cleared all size/units + 'fill', # partial execution + 'canceled', + 'error', + ] - # 'broker_submitted', - # 'broker_cancelled', - # 'broker_executed', - # 'broker_filled', - # 'broker_errored', - - # 'alert_submitted', - # 'alert_triggered', - - # } - resp: str # "response", see above - - # trigger info - trigger_price: Optional[float] = None - # price: float - - # broker: Optional[str] = None + oid: str # uuid4 # this maps normally to the ``BrokerdOrder.reqid`` below, an id # normally allocated internally by the backend broker routing system - broker_reqid: Optional[Union[int, str]] = None + reqid: Optional[Union[int, str]] = None + + # the (last) source order/request msg if provided + # (eg. the Order/Cancel which causes this msg) + req: Optional[Union[Order, Cancel]] = None + + src: Optional[str] = None # for relaying backend msg data "through" the ems layer brokerd_msg: dict = {} @@ -185,20 +223,19 @@ class BrokerdStatus(Struct): name: str = 'status' reqid: Union[int, str] time_ns: int + status: Literal[ + 'open', + 'canceled', + 'fill', + 'pending', + ] - # TODO: instead (ack, pending, open, fill, clos(ed), cancelled) - # { - # 'submitted', # open - # 'cancelled', # canceled - # 'filled', # closed - # } - status: str account: str filled: float = 0.0 reason: str = '' remaining: float = 0.0 - external: bool = False + # external: bool = False # order: Optional[BrokerdOrder] = None # XXX: better design/name here? @@ -206,7 +243,7 @@ class BrokerdStatus(Struct): # event that wasn't originated by piker's emsd (eg. some external # trading system which does it's own order control but that you # might want to "track" using piker UIs/systems). - external: bool = False + # external: bool = False # XXX: not required schema as of yet broker_details: dict = { diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 2160bfca..e4fbb1bb 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -45,8 +45,13 @@ from ..data._normalize import iterticks from ..data._source import unpack_fqsn from ..log import get_logger from ._messages import ( - BrokerdCancel, BrokerdOrder, BrokerdOrderAck, BrokerdStatus, - BrokerdFill, BrokerdPosition, BrokerdError + BrokerdCancel, + BrokerdOrder, + BrokerdOrderAck, + BrokerdStatus, + BrokerdFill, + BrokerdPosition, + BrokerdError, ) @@ -94,6 +99,10 @@ class PaperBoi: ''' is_modify: bool = False + if action == 'alert': + # bypass all fill simulation + return reqid + entry = self._reqids.get(reqid) if entry: # order is already existing, this is a modify @@ -104,10 +113,6 @@ class PaperBoi: # register order internally self._reqids[reqid] = (oid, symbol, action, price) - if action == 'alert': - # bypass all fill simulation - return reqid - # TODO: net latency model # we checkpoint here quickly particulalry # for dark orders since we want the dark_executed @@ -119,7 +124,9 @@ class PaperBoi: size = -size msg = BrokerdStatus( - status='submitted', + status='open', + # account=f'paper_{self.broker}', + account='paper', reqid=reqid, time_ns=time.time_ns(), filled=0.0, @@ -136,7 +143,14 @@ class PaperBoi: ) or ( action == 'sell' and (clear_price := self.last_bid[0]) >= price ): - await self.fake_fill(symbol, clear_price, size, action, reqid, oid) + await self.fake_fill( + symbol, + clear_price, + size, + action, + reqid, + oid, + ) else: # register this submissions as a paper live order @@ -178,7 +192,9 @@ class PaperBoi: await trio.sleep(0.05) msg = BrokerdStatus( - status='cancelled', + status='canceled', + # account=f'paper_{self.broker}', + account='paper', reqid=reqid, time_ns=time.time_ns(), broker_details={'name': 'paperboi'}, @@ -230,25 +246,23 @@ class PaperBoi: self._trade_ledger.update(fill_msg.to_dict()) if order_complete: - msg = BrokerdStatus( - reqid=reqid, time_ns=time.time_ns(), - - status='filled', + # account=f'paper_{self.broker}', + account='paper', + status='closed', filled=size, remaining=0 if order_complete else remaining, - - broker_details={ - 'paper_info': { - 'oid': oid, - }, - 'action': action, - 'size': size, - 'price': price, - 'name': self.broker, - }, + # broker_details={ + # 'paper_info': { + # 'oid': oid, + # }, + # 'action': action, + # 'size': size, + # 'price': price, + # 'name': self.broker, + # }, ) await self.ems_trades_stream.send(msg) @@ -393,69 +407,72 @@ async def handle_order_requests( # order_request: dict async for request_msg in ems_order_stream: - action = request_msg['action'] - - if action in {'buy', 'sell'}: - - account = request_msg['account'] - if account != 'paper': - log.error( - 'This is a paper account,' - ' only a `paper` selection is valid' - ) - await ems_order_stream.send(BrokerdError( - oid=request_msg['oid'], - symbol=request_msg['symbol'], - reason=f'Paper only. No account found: `{account}` ?', - )) - continue + # action = request_msg['action'] + match request_msg: + # if action in {'buy', 'sell'}: + case {'action': ('buy' | 'sell')}: + order = BrokerdOrder(**request_msg) + account = order.account + if account != 'paper': + log.error( + 'This is a paper account,' + ' only a `paper` selection is valid' + ) + await ems_order_stream.send(BrokerdError( + # oid=request_msg['oid'], + oid=order.oid, + # symbol=request_msg['symbol'], + symbol=order.symbol, + reason=f'Paper only. No account found: `{account}` ?', + )) + continue # validate - order = BrokerdOrder(**request_msg) + # order = BrokerdOrder(**request_msg) - if order.reqid is None: - reqid = str(uuid.uuid4()) - else: - reqid = order.reqid + # if order.reqid is None: + # reqid = + # else: + reqid = order.reqid or str(uuid.uuid4()) - # deliver ack that order has been submitted to broker routing - await ems_order_stream.send( - BrokerdOrderAck( + # deliver ack that order has been submitted to broker routing + await ems_order_stream.send( + BrokerdOrderAck( - # ems order request id - oid=order.oid, + # ems order request id + oid=order.oid, - # broker specific request id - reqid=reqid, + # broker specific request id + reqid=reqid, + ) ) - ) - # call our client api to submit the order - reqid = await client.submit_limit( + # call our client api to submit the order + reqid = await client.submit_limit( - oid=order.oid, - symbol=order.symbol, - price=order.price, - action=order.action, - size=order.size, + oid=order.oid, + symbol=order.symbol, + price=order.price, + action=order.action, + size=order.size, - # XXX: by default 0 tells ``ib_insync`` methods that - # there is no existing order so ask the client to create - # a new one (which it seems to do by allocating an int - # counter - collision prone..) - reqid=reqid, - ) + # XXX: by default 0 tells ``ib_insync`` methods that + # there is no existing order so ask the client to create + # a new one (which it seems to do by allocating an int + # counter - collision prone..) + reqid=reqid, + ) - elif action == 'cancel': - msg = BrokerdCancel(**request_msg) + # elif action == 'cancel': + case {'action': 'cancel'}: + msg = BrokerdCancel(**request_msg) + await client.submit_cancel( + reqid=msg.reqid + ) - await client.submit_cancel( - reqid=msg.reqid - ) - - else: - log.error(f'Unknown order command: {request_msg}') + case _: + log.error(f'Unknown order command: {request_msg}') @tractor.context diff --git a/piker/ui/order_mode.py b/piker/ui/order_mode.py index b9d23ab3..4e8d9e66 100644 --- a/piker/ui/order_mode.py +++ b/piker/ui/order_mode.py @@ -63,7 +63,7 @@ from ._forms import open_form_input_handling log = get_logger(__name__) -class OrderDialog(Struct): +class Dialog(Struct): ''' Trade dialogue meta-data describing the lifetime of an order submission to ``emsd`` from a chart. @@ -146,7 +146,7 @@ class OrderMode: current_pp: Optional[PositionTracker] = None active: bool = False name: str = 'order' - dialogs: dict[str, OrderDialog] = field(default_factory=dict) + dialogs: dict[str, Dialog] = field(default_factory=dict) _colors = { 'alert': 'alert_yellow', @@ -163,6 +163,7 @@ class OrderMode: ) -> LevelLine: level = order.price + print(f'SIZE: {order.size}') line = order_line( self.chart, @@ -175,7 +176,8 @@ class OrderMode: color=self._colors[order.action], dotted=True if ( - order.exec_mode == 'dark' and order.action != 'alert' + order.exec_mode == 'dark' + and order.action != 'alert' ) else False, **line_kwargs, @@ -265,7 +267,7 @@ class OrderMode: send_msg: bool = True, order: Optional[Order] = None, - ) -> OrderDialog: + ) -> Dialog: ''' Send execution order to EMS return a level line to represent the order on a chart. @@ -304,7 +306,7 @@ class OrderMode: uuid=order.oid, ) - dialog = OrderDialog( + dialog = Dialog( uuid=order.oid, order=order, symbol=order.symbol, @@ -373,7 +375,7 @@ class OrderMode: self, uuid: str - ) -> OrderDialog: + ) -> Dialog: ''' Order submitted status event handler. @@ -428,7 +430,7 @@ class OrderMode: self, uuid: str, - msg: Dict[str, Any], + msg: Status, ) -> None: @@ -452,7 +454,7 @@ class OrderMode: # TODO: add in standard fill/exec info that maybe we # pack in a broker independent way? - f'{msg["resp"]}: {msg["trigger_price"]}', + f'{msg.resp}: {msg.req.price}', ], ) log.runtime(result) @@ -524,53 +526,36 @@ class OrderMode: def load_unknown_dialog_from_msg( self, - # status: Status, - msg: dict, + msg: Status, - ) -> OrderDialog: + ) -> Dialog: - oid = str(msg['oid']) - # oid = str(status.oid) - - # bstatus = BrokerdStatus(**msg.brokerd_msg) # NOTE: the `.order` attr **must** be set with the # equivalent order msg in order to be loaded. - # border = BrokerdOrder(**bstatus.broker_details['order']) - # msg = msg['brokerd_msg'] + order = Order(**msg.req) + oid = str(msg.oid) + symbol = order.symbol - # size = border.size - size = msg['size'] - if size >= 0: - action = 'buy' + # TODO: MEGA UGGG ZONEEEE! + src = msg.src + if ( + src + and src != 'dark' + and src not in symbol + ): + fqsn = symbol + '.' + src + brokername = src else: - action = 'sell' + fqsn = symbol + *head, brokername = fqsn.rsplit('.') - # acct = border.account - # price = border.price - # price = msg['brokerd_msg']['price'] - symbol = msg['symbol'] - deats = msg['broker_details'] - brokername = deats['name'] - fqsn = ( - # deats['fqsn'] + '.' + deats['name'] - symbol + '.' + brokername - ) - symbol = Symbol.from_fqsn( + # fill out complex fields + order.oid = str(order.oid) + order.brokers = [brokername] + order.symbol = Symbol.from_fqsn( fqsn=fqsn, info={}, ) - # map to order composite-type - order = Order( - action=action, - price=msg['price'], - account=msg['account'], - size=size, - symbol=symbol, - brokers=[brokername], - oid=oid, - exec_mode='live', # dark or live - ) - dialog = self.submit_order( send_msg=False, order=order, @@ -770,7 +755,7 @@ async def open_order_mode( order_pane.order_mode = mode # select a pp to track - tracker = trackers[pp_account] + tracker: PositionTracker = trackers[pp_account] mode.current_pp = tracker tracker.show() tracker.hide_info() @@ -870,12 +855,13 @@ async def process_trade_msg( book: OrderBook, msg: dict, -) -> None: +) -> tuple[Dialog, Status]: get_index = mode.chart.get_index fmsg = pformat(msg) log.info(f'Received order msg:\n{fmsg}') name = msg['name'] + if name in ( 'position', ): @@ -901,105 +887,117 @@ async def process_trade_msg( # short circuit to next msg to avoid # unnecessary msg content lookups return - # continue - resp = msg['resp'] - oid = str(msg['oid']) - dialog = mode.dialogs.get(oid) + msg = Status(**msg) + resp = msg.resp + oid = msg.oid + dialog: Dialog = mode.dialogs.get(oid) - if dialog is None: - log.warning( - f'received msg for untracked dialog:\n{fmsg}' - ) - # dialog = mode.load_unknown_dialog_from_msg(Status(**msg)) - dialog = mode.load_unknown_dialog_from_msg(msg) + match msg: + case Status(resp='dark_open' | 'open'): - # record message to dialog tracking - dialog.msgs[oid] = msg + if dialog is not None: + # show line label once order is live + mode.on_submit(oid) - # response to 'action' request (buy/sell) - if resp in ( - 'dark_submitted', - 'broker_submitted' - ): - # show line label once order is live - mode.on_submit(oid) + else: + log.warning( + f'received msg for untracked dialog:\n{fmsg}' + ) + assert msg.resp in ('open', 'dark_open'), f'Unknown msg: {msg}' - # resp to 'cancel' request or error condition - # for action request - elif resp in ( - 'broker_inactive', - 'broker_errored', - ): - # delete level line from view - mode.on_cancel(oid) - broker_msg = msg['brokerd_msg'] - log.error( - f'Order {oid}->{resp} with:\n{pformat(broker_msg)}' - ) + sym = mode.chart.linked.symbol + fqsn = sym.front_fqsn() + order = Order(**msg.req) + if ( + ((order.symbol + f'.{msg.src}') == fqsn) - elif resp in ( - 'broker_cancelled', - 'dark_cancelled' - ): - # delete level line from view - mode.on_cancel(oid) - broker_msg = msg['brokerd_msg'] - log.cancel( - f'Order {oid}->{resp} with:\n{pformat(broker_msg)}' - ) + # a existing dark order for the same symbol + or ( + order.symbol == fqsn + and (msg.src == 'dark') or (msg.src in fqsn) + ) + ): + dialog = mode.load_unknown_dialog_from_msg(msg) + mode.on_submit(oid) - elif resp in ( - 'dark_triggered' - ): - log.info(f'Dark order triggered for {fmsg}') + case Status(resp='error'): + # delete level line from view + mode.on_cancel(oid) + broker_msg = msg.brokerd_msg + log.error( + f'Order {oid}->{resp} with:\n{pformat(broker_msg)}' + ) - elif resp in ( - 'alert_triggered' - ): - # should only be one "fill" for an alert - # add a triangle and remove the level line - mode.on_fill( - oid, - price=msg['trigger_price'], - arrow_index=get_index(time.time()), - ) - mode.lines.remove_line(uuid=oid) - await mode.on_exec(oid, msg) + case Status(resp='canceled'): + # delete level line from view + mode.on_cancel(oid) + req = msg.req + log.cancel( + f'Canceled order {oid}:\n{pformat(req)}' + ) - # response to completed 'action' request for buy/sell - elif resp in ( - 'broker_executed', - ): - # right now this is just triggering a system alert - await mode.on_exec(oid, msg) + case Status( + resp='triggered', + # req=Order(exec_mode='dark') # TODO: + req={'exec_mode': 'dark'}, + ): + log.info(f'Dark order triggered for {fmsg}') - if msg['brokerd_msg']['remaining'] == 0: + case Status( + resp='triggered', + # req=Order(exec_mode='live', action='alert') as req, # TODO + req={'exec_mode': 'live', 'action': 'alert'} as req, + ): + # should only be one "fill" for an alert + # add a triangle and remove the level line + mode.on_fill( + oid, + price=req.price, + arrow_index=get_index(time.time()), + ) + mode.lines.remove_line(uuid=oid) + msg.req = Order(**req) + await mode.on_exec(oid, msg) + + # response to completed 'dialog' for order request + case Status( + resp='closed', + # req=Order() as req, # TODO + req=req, + ): + # right now this is just triggering a system alert + msg.req = Order(**req) + await mode.on_exec(oid, msg) mode.lines.remove_line(uuid=oid) - # each clearing tick is responded individually - elif resp in ( - 'broker_filled', - ): - known_order = book._sent_orders.get(oid) - if not known_order: - log.warning(f'order {oid} is unknown') - return - # continue + # each clearing tick is responded individually + case Status(resp='fill'): + known_order = book._sent_orders.get(oid) + if not known_order: + log.warning(f'order {oid} is unknown') + return + # continue - action = known_order.action - details = msg['brokerd_msg'] + action = known_order.action + details = msg.brokerd_msg - # TODO: some kinda progress system - mode.on_fill( - oid, - price=details['price'], - pointing='up' if action == 'buy' else 'down', + # TODO: some kinda progress system + mode.on_fill( + oid, + price=details['price'], + pointing='up' if action == 'buy' else 'down', - # TODO: put the actual exchange timestamp - arrow_index=get_index(details['broker_time']), - ) + # TODO: put the actual exchange timestamp + arrow_index=get_index(details['broker_time']), + ) - # TODO: how should we look this up? - # tracker = mode.trackers[msg['account']] - # tracker.live_pp.fills.append(msg) + # TODO: how should we look this up? + # tracker = mode.trackers[msg['account']] + # tracker.live_pp.fills.append(msg) + + # record message to dialog tracking + if dialog: + dialog.msgs[oid] = msg + + return dialog, msg From 176b230a463c4b1eecee166a0d83741a7c31029e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 10 Aug 2022 16:41:00 -0400 Subject: [PATCH 10/33] Use modern `Union` pipe op syntax for msg fields --- piker/clearing/_messages.py | 60 ++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/piker/clearing/_messages.py b/piker/clearing/_messages.py index 2c0d95c8..6296c3bd 100644 --- a/piker/clearing/_messages.py +++ b/piker/clearing/_messages.py @@ -18,14 +18,13 @@ Clearing sub-system message and protocols. """ -from collections import ( - ChainMap, - deque, -) +# from collections import ( +# ChainMap, +# deque, +# ) from typing import ( Optional, Literal, - Union, ) from ..data._source import Symbol @@ -55,7 +54,8 @@ from ..data.types import Struct # TODO: ``msgspec`` stuff worth paying attention to: -# - schema evolution: https://jcristharif.com/msgspec/usage.html#schema-evolution +# - schema evolution: +# https://jcristharif.com/msgspec/usage.html#schema-evolution # - for eg. ``BrokerdStatus``, instead just have separate messages? # - use literals for a common msg determined by diff keys? # - https://jcristharif.com/msgspec/usage.html#literal @@ -92,7 +92,7 @@ class Order(Struct): # internal ``emdsd`` unique "order id" oid: str # uuid4 - symbol: Union[str, Symbol] + symbol: str | Symbol account: str # should we set a default as '' ? price: float @@ -110,7 +110,6 @@ class Cancel(Struct): action: str = 'cancel' oid: str # uuid4 symbol: str - req: Optional[Order] = None # -------------- @@ -123,31 +122,38 @@ class Status(Struct): name: str = 'status' time_ns: int + oid: str # uuid4 ems-order dialog id resp: Literal[ - 'pending', # acked but not yet open + 'pending', # acked by broker but not yet open 'open', - 'dark_open', # live in dark loop - 'triggered', # dark-submitted to brokerd-backend + 'dark_open', # dark/algo triggered order is open in ems clearing loop + 'triggered', # above triggered order sent to brokerd, or an alert closed 'closed', # fully cleared all size/units 'fill', # partial execution 'canceled', 'error', ] - oid: str # uuid4 - # this maps normally to the ``BrokerdOrder.reqid`` below, an id # normally allocated internally by the backend broker routing system - reqid: Optional[Union[int, str]] = None + reqid: Optional[int | str] = None # the (last) source order/request msg if provided - # (eg. the Order/Cancel which causes this msg) - req: Optional[Union[Order, Cancel]] = None + # (eg. the Order/Cancel which causes this msg) and + # acts as a back-reference to the corresponding + # request message which was the source of this msg. + req: Optional[Order | Cancel] = None + # XXX: better design/name here? + # flag that can be set to indicate a message for an order + # event that wasn't originated by piker's emsd (eg. some external + # trading system which does it's own order control but that you + # might want to "track" using piker UIs/systems). src: Optional[str] = None - # for relaying backend msg data "through" the ems layer + # for relaying a boxed brokerd-dialog-side msg data "through" the + # ems layer to clients. brokerd_msg: dict = {} @@ -169,7 +175,7 @@ class BrokerdCancel(Struct): # for setting a unique order id then this value will be relayed back # on the emsd order request stream as the ``BrokerdOrderAck.reqid`` # field - reqid: Optional[Union[int, str]] = None + reqid: Optional[int | str] = None class BrokerdOrder(Struct): @@ -188,7 +194,7 @@ class BrokerdOrder(Struct): # for setting a unique order id then this value will be relayed back # on the emsd order request stream as the ``BrokerdOrderAck.reqid`` # field - reqid: Optional[Union[int, str]] = None + reqid: Optional[int | str] = None symbol: str # fqsn price: float @@ -211,7 +217,7 @@ class BrokerdOrderAck(Struct): name: str = 'ack' # defined and provided by backend - reqid: Union[int, str] + reqid: int | str # emsd id originally sent in matching request msg oid: str @@ -221,7 +227,7 @@ class BrokerdOrderAck(Struct): class BrokerdStatus(Struct): name: str = 'status' - reqid: Union[int, str] + reqid: int | str time_ns: int status: Literal[ 'open', @@ -235,14 +241,6 @@ class BrokerdStatus(Struct): reason: str = '' remaining: float = 0.0 - # external: bool = False - # order: Optional[BrokerdOrder] = None - - # XXX: better design/name here? - # flag that can be set to indicate a message for an order - # event that wasn't originated by piker's emsd (eg. some external - # trading system which does it's own order control but that you - # might want to "track" using piker UIs/systems). # external: bool = False # XXX: not required schema as of yet @@ -258,7 +256,7 @@ class BrokerdFill(Struct): ''' name: str = 'fill' - reqid: Union[int, str] + reqid: int | str time_ns: int # order exeuction related @@ -288,7 +286,7 @@ class BrokerdError(Struct): # if no brokerd order request was actually submitted (eg. we errored # at the ``pikerd`` layer) then there will be ``reqid`` allocated. - reqid: Optional[Union[int, str]] = None + reqid: Optional[int | str] = None symbol: str reason: str From 2b61672723382fa8591556d35a97cd6b91fa748c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 10 Aug 2022 17:17:47 -0400 Subject: [PATCH 11/33] Handle 'closed' vs. 'fill` race case.. `ib` is super good not being reliable with order event sequence order and duplication of fill info. This adds some guards to try and avoid popping the last status status too early if we end up receiving a `'closed'` before the expected `'fill`' event(s). Further delete the `status_msg` ref on each iteration to avoid stale reference lookups in the relay task/loop. --- piker/clearing/_ems.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 0411f026..24a78420 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -667,19 +667,19 @@ async def translate_and_relay_brokerd_events( # received this ack, in which case we relay that cancel # signal **asap** to the backend broker # status = book._active.get(oid) - status = book._active[oid] - req = status.req + status_msg = book._active[oid] + req = status_msg.req if req and req.action == 'cancel': # assign newly providerd broker backend request id # and tell broker to cancel immediately - status.reqid = reqid + status_msg.reqid = reqid await brokerd_trades_stream.send(req) # 2. the order is now active and will be mirrored in # our book -> registered as live flow else: # TODO: should we relay this ack state? - status.resp = 'pending' + status_msg.resp = 'pending' # no msg to client necessary continue @@ -729,6 +729,7 @@ async def translate_and_relay_brokerd_events( # msg-chain/dialog. ems_client_order_stream = router.dialogues[oid] status_msg = book._active[oid] + old_resp = status_msg.resp status_msg.resp = status # retrieve existing live flow @@ -746,7 +747,11 @@ async def translate_and_relay_brokerd_events( if status == 'closed': log.info(f'Execution for {oid} is complete!') - status_msg = book._active.pop(oid) + + # only if we already rxed a fill then probably + # this clear is fully complete? (frickin ib..) + if old_resp == 'fill': + status_msg = book._active.pop(oid) elif status == 'canceled': log.info(f'Cancellation for {oid} is complete!') @@ -823,6 +828,7 @@ async def translate_and_relay_brokerd_events( 'status': status, 'reqid': reqid, }: + status_msg = book._active[oid] log.warning( 'Unhandled broker status:\n' f'{pformat(brokerd_msg)}\n' @@ -837,10 +843,19 @@ async def translate_and_relay_brokerd_events( oid := book._ems2brokerd_ids.inverse.get(reqid) ): # proxy through the "fill" result(s) - log.info(f'Fill for {oid} cleared with:\n{pformat(msg)}') msg = BrokerdFill(**brokerd_msg) + log.info(f'Fill for {oid} cleared with:\n{pformat(msg)}') + ems_client_order_stream = router.dialogues[oid] + + # wtf a fill can come after 'closed' from ib? status_msg = book._active[oid] + + # only if we already rxed a 'closed' + # this clear is fully complete? (frickin ib..) + if status_msg.resp == 'closed': + status_msg = book._active.pop(oid) + status_msg.resp = 'fill' status_msg.reqid = reqid status_msg.brokerd_msg = msg @@ -849,6 +864,10 @@ async def translate_and_relay_brokerd_events( case _: raise ValueError(f'Brokerd message {brokerd_msg} is invalid') + # XXX: ugh sometimes we don't access it? + if status_msg: + del status_msg + # TODO: do we want this to keep things cleaned up? # it might require a special status from brokerd to affirm the # flow is complete? From 16012f6f02df1daa83948a8afa291123a62f8047 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 10 Aug 2022 17:59:27 -0400 Subject: [PATCH 12/33] Include both symbols in error msg when a mismatch --- piker/clearing/_client.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/piker/clearing/_client.py b/piker/clearing/_client.py index 3e87ab96..11eb9b69 100644 --- a/piker/clearing/_client.py +++ b/piker/clearing/_client.py @@ -83,7 +83,13 @@ class OrderBook: """Cancel an order (or alert) in the EMS. """ - cmd = self._sent_orders[uuid] + cmd = self._sent_orders.get(uuid) + if not cmd: + log.error( + f'Unknown order {uuid}!?\n' + f'Maybe there is a stale entry or line?\n' + f'You should report this as a bug!' + ) msg = Cancel( oid=uuid, symbol=cmd.symbol, @@ -156,7 +162,10 @@ async def relay_order_cmds_from_sync_code( # send msg over IPC / wire await to_ems_stream.send(cmd) else: - log.warning(f'Ignoring unmatched order cmd for {sym}: {msg}') + log.warning( + f'Ignoring unmatched order cmd for {sym} != {symbol_key}:' + f'\n{msg}' + ) @acm From b59ed74bc1651f5ea57ca6afbaa11f7bbea8851b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 11 Aug 2022 14:18:53 -0400 Subject: [PATCH 13/33] 'Only send `'closed'` on Filled events, lowercase all statues' --- piker/brokers/ib/broker.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index b7c121e7..da1b03c8 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -846,8 +846,12 @@ async def deliver_trade_events( # cancelling.. gawwwd if ib_status_key == 'cancelled': last_log = trade.log[-1] - if last_log.message: + if ( + last_log.message + and 'Error' not in last_log.message + ): ib_status_key = trade.log[-2].status + print(ib_status_key) elif ib_status_key == 'inactive': async def sched_cancel(): @@ -862,10 +866,16 @@ async def deliver_trade_events( nurse.start_soon(sched_cancel) - status_key = _statuses.get(ib_status_key) or ib_status_key + status_key = ( + _statuses.get(ib_status_key) + or ib_status_key.lower() + ) remaining = status.remaining - if remaining == 0: + if ( + status_key == 'filled' + and remaining == 0 + ): status_key = 'closed' # skip duplicate filled updates - we get the deats @@ -1019,9 +1029,18 @@ async def deliver_trade_events( if err['reqid'] == -1: log.error(f'TWS external order error:\n{pformat(err)}') - # TODO: what schema for this msg if we're going to make it - # portable across all backends? - # msg = BrokerdError(**err) + # TODO: we don't want to relay data feed / lookup errors + # so we need some further filtering logic here.. + # for most cases the 'status' block above should take + # care of this. + # await ems_stream.send(BrokerdStatus( + # status='error', + # reqid=err['reqid'], + # reason=err['reason'], + # time_ns=time.time_ns(), + # account=accounts_def.inverse[trade.order.account], + # broker_details={'name': 'ib'}, + # )) case 'position': From 91fdc7c5c7d371411f49fa33362c5c4ee01a68c1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 11 Aug 2022 14:20:23 -0400 Subject: [PATCH 14/33] Load boxed `.req` values as `Order`s in mode loop --- piker/ui/order_mode.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/piker/ui/order_mode.py b/piker/ui/order_mode.py index 4e8d9e66..cecb8787 100644 --- a/piker/ui/order_mode.py +++ b/piker/ui/order_mode.py @@ -163,7 +163,6 @@ class OrderMode: ) -> LevelLine: level = order.price - print(f'SIZE: {order.size}') line = order_line( self.chart, @@ -859,7 +858,7 @@ async def process_trade_msg( get_index = mode.chart.get_index fmsg = pformat(msg) - log.info(f'Received order msg:\n{fmsg}') + log.debug(f'Received order msg:\n{fmsg}') name = msg['name'] if name in ( @@ -920,6 +919,7 @@ async def process_trade_msg( ): dialog = mode.load_unknown_dialog_from_msg(msg) mode.on_submit(oid) + # return dialog, msg case Status(resp='error'): # delete level line from view @@ -932,16 +932,15 @@ async def process_trade_msg( case Status(resp='canceled'): # delete level line from view mode.on_cancel(oid) - req = msg.req - log.cancel( - f'Canceled order {oid}:\n{pformat(req)}' - ) + req = Order(**msg.req) + log.cancel(f'Canceled {req.action}:{oid}') case Status( resp='triggered', # req=Order(exec_mode='dark') # TODO: req={'exec_mode': 'dark'}, ): + # TODO: UX for a "pending" clear/live order log.info(f'Dark order triggered for {fmsg}') case Status( @@ -951,13 +950,14 @@ async def process_trade_msg( ): # should only be one "fill" for an alert # add a triangle and remove the level line + req = Order(**req) mode.on_fill( oid, price=req.price, arrow_index=get_index(time.time()), ) mode.lines.remove_line(uuid=oid) - msg.req = Order(**req) + msg.req = req await mode.on_exec(oid, msg) # response to completed 'dialog' for order request @@ -966,18 +966,18 @@ async def process_trade_msg( # req=Order() as req, # TODO req=req, ): - # right now this is just triggering a system alert msg.req = Order(**req) await mode.on_exec(oid, msg) mode.lines.remove_line(uuid=oid) # each clearing tick is responded individually case Status(resp='fill'): + + # handle out-of-piker fills reporting? known_order = book._sent_orders.get(oid) if not known_order: log.warning(f'order {oid} is unknown') return - # continue action = known_order.action details = msg.brokerd_msg From bec32956a86c1d4b33e83feebf5b8381a3973ce0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 11 Aug 2022 14:26:12 -0400 Subject: [PATCH 15/33] Move fill case-block earlier, log broker errors --- piker/clearing/_ems.py | 67 +++++++++++++++++++++---------------- piker/clearing/_messages.py | 1 + 2 files changed, 39 insertions(+), 29 deletions(-) diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 24a78420..0d8693f9 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -754,13 +754,40 @@ async def translate_and_relay_brokerd_events( status_msg = book._active.pop(oid) elif status == 'canceled': - log.info(f'Cancellation for {oid} is complete!') + log.cancel(f'Cancellation for {oid} is complete!') else: # open # relayed from backend but probably not handled so # just log it log.info(f'{broker} opened order {msg}') + # BrokerdFill + case { + 'name': 'fill', + 'reqid': reqid, # brokerd generated order-request id + # 'symbol': sym, # paper engine doesn't have this, nbd? + } if ( + oid := book._ems2brokerd_ids.inverse.get(reqid) + ): + # proxy through the "fill" result(s) + msg = BrokerdFill(**brokerd_msg) + log.info(f'Fill for {oid} cleared with:\n{pformat(msg)}') + + ems_client_order_stream = router.dialogues[oid] + + # wtf a fill can come after 'closed' from ib? + status_msg = book._active[oid] + + # only if we already rxed a 'closed' + # this clear is fully complete? (frickin ib..) + # if status_msg.resp == 'closed': + # status_msg = book._active.pop(oid) + + status_msg.resp = 'fill' + status_msg.reqid = reqid + status_msg.brokerd_msg = msg + await ems_client_order_stream.send(status_msg) + # ``Status`` containing an embedded order msg which # should be loaded as a "pre-existing open order" from the # brokerd backend. @@ -816,6 +843,14 @@ async def translate_and_relay_brokerd_events( # don't fall through continue + # brokerd error + case { + 'name': 'status', + 'status': 'error', + }: + log.error(f'Broker error:\n{pformat(brokerd_msg)}') + # XXX: we presume the brokerd cancels its own order + # TOO FAST ``BrokerdStatus`` that arrives # before the ``BrokerdAck``. case { @@ -830,37 +865,11 @@ async def translate_and_relay_brokerd_events( }: status_msg = book._active[oid] log.warning( - 'Unhandled broker status:\n' + 'Unhandled broker status for dialog:\n' + f'{pformat(status_msg)}\n' f'{pformat(brokerd_msg)}\n' ) - # BrokerdFill - case { - 'name': 'fill', - 'reqid': reqid, # brokerd generated order-request id - # 'symbol': sym, # paper engine doesn't have this, nbd? - } if ( - oid := book._ems2brokerd_ids.inverse.get(reqid) - ): - # proxy through the "fill" result(s) - msg = BrokerdFill(**brokerd_msg) - log.info(f'Fill for {oid} cleared with:\n{pformat(msg)}') - - ems_client_order_stream = router.dialogues[oid] - - # wtf a fill can come after 'closed' from ib? - status_msg = book._active[oid] - - # only if we already rxed a 'closed' - # this clear is fully complete? (frickin ib..) - if status_msg.resp == 'closed': - status_msg = book._active.pop(oid) - - status_msg.resp = 'fill' - status_msg.reqid = reqid - status_msg.brokerd_msg = msg - await ems_client_order_stream.send(status_msg) - case _: raise ValueError(f'Brokerd message {brokerd_msg} is invalid') diff --git a/piker/clearing/_messages.py b/piker/clearing/_messages.py index 6296c3bd..f8fd6937 100644 --- a/piker/clearing/_messages.py +++ b/piker/clearing/_messages.py @@ -234,6 +234,7 @@ class BrokerdStatus(Struct): 'canceled', 'fill', 'pending', + 'error', ] account: str From 2aec1c5f1dd76f4dcc69c71142d7ff38497791e1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 11 Aug 2022 15:56:28 -0400 Subject: [PATCH 16/33] Only pprint our struct when we detect a py REPL --- piker/data/types.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/piker/data/types.py b/piker/data/types.py index d8926610..9b08b432 100644 --- a/piker/data/types.py +++ b/piker/data/types.py @@ -18,6 +18,7 @@ Built-in (extension) types. """ +import sys from typing import Optional from pprint import pformat @@ -42,7 +43,12 @@ class Struct( } def __repr__(self): - return f'Struct({pformat(self.to_dict())})' + # only turn on pprint when we detect a python REPL + # at runtime B) + if hasattr(sys, 'ps1'): + return f'Struct({pformat(self.to_dict())})' + + return super().__repr__() def copy( self, From c8bff81220b968672df0cd5d865dbd1d772ade62 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 11 Aug 2022 19:58:53 -0400 Subject: [PATCH 17/33] Add runtime guards around feed pausing during interaction --- piker/ui/_interaction.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index d8f65dd9..71797a33 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -221,6 +221,7 @@ async def handle_viewmode_kb_inputs( # TODO: show pp config mini-params in status bar widget # mode.pp_config.show() + trigger_type: str = 'dark' if ( # 's' for "submit" to activate "live" order Qt.Key_S in pressed or @@ -228,9 +229,6 @@ async def handle_viewmode_kb_inputs( ): trigger_type: str = 'live' - else: - trigger_type: str = 'dark' - # order mode trigger "actions" if Qt.Key_D in pressed: # for "damp eet" action = 'sell' @@ -397,8 +395,11 @@ class ChartView(ViewBox): ''' if self._ic is None: - self.chart.pause_all_feeds() - self._ic = trio.Event() + try: + self.chart.pause_all_feeds() + self._ic = trio.Event() + except RuntimeError: + pass def signal_ic( self, @@ -411,9 +412,12 @@ class ChartView(ViewBox): ''' if self._ic: - self._ic.set() - self._ic = None - self.chart.resume_all_feeds() + try: + self._ic.set() + self._ic = None + self.chart.resume_all_feeds() + except RuntimeError: + pass @asynccontextmanager async def open_async_input_handler( @@ -669,7 +673,10 @@ class ChartView(ViewBox): # XXX: WHY ev.accept() - self.start_ic() + try: + self.start_ic() + except RuntimeError: + pass # if self._ic is None: # self.chart.pause_all_feeds() # self._ic = trio.Event() From e2cd8c4aefe058a130d3c79243a611aae3a4c143 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 11 Aug 2022 21:30:32 -0400 Subject: [PATCH 18/33] Add initial `kraken` live order loading --- piker/brokers/kraken/broker.py | 90 ++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/piker/brokers/kraken/broker.py b/piker/brokers/kraken/broker.py index 3641934a..6bb3be17 100644 --- a/piker/brokers/kraken/broker.py +++ b/piker/brokers/kraken/broker.py @@ -39,7 +39,6 @@ from bidict import bidict import pendulum import trio import tractor -import wsproto from piker.pp import ( Position, @@ -49,6 +48,8 @@ from piker.pp import ( open_pps, ) from piker.clearing._messages import ( + Order, + Status, BrokerdCancel, BrokerdError, BrokerdFill, @@ -126,7 +127,7 @@ async def handle_order_requests( oid=msg['oid'], symbol=msg['symbol'], reason=( - f'TooFastEdit reqid:{reqid}, could not cancelling..' + f'Edit too fast:{reqid}, cancelling..' ), ) @@ -249,7 +250,7 @@ async def handle_order_requests( @acm async def subscribe( - ws: wsproto.WSConnection, + ws: NoBsWs, token: str, subs: list[tuple[str, dict]] = [ ('ownTrades', { @@ -632,8 +633,6 @@ async def handle_order_updates( # to do all fill/status/pp updates in that sub and just use # this one for ledger syncs? - # XXX: ASK SUPPORT ABOUT THIS! - # For eg. we could take the "last 50 trades" and do a diff # with the ledger and then only do a re-sync if something # seems amiss? @@ -696,7 +695,6 @@ async def handle_order_updates( status_msg = BrokerdStatus( reqid=reqid, time_ns=time.time_ns(), - account=acc_name, status='filled', filled=size, @@ -870,38 +868,56 @@ async def handle_order_updates( f'{update_msg}\n' 'Cancelling order for now!..' ) + # call ws api to cancel: + # https://docs.kraken.com/websockets/#message-cancelOrder + await ws.send_msg({ + 'event': 'cancelOrder', + 'token': token, + 'reqid': reqid or 0, + 'txid': [txid], + }) + continue - elif noid: # a non-ems-active order - # TODO: handle these and relay them - # through the EMS to the client / UI - # side! - log.cancel( - f'Rx unknown active order {txid}:\n' - f'{update_msg}\n' - 'Cancelling order for now!..' + # a non-ems-active order, emit live + # order embedded in status msg. + elif noid: + # parse out existing live order + descr = rest['descr'] + fqsn = descr['pair'].replace( + '/', '').lower() + price = float(descr['price']) + size = float(rest['vol']) + action = descr['type'] + + # register the userref value from + # kraken (usually an `int` staring + # at 1?) as our reqid. + reqids2txids[reqid] = txid + oid = str(reqid) + ids[oid] = reqid # NOTE!: str -> int + + # fill out ``Status`` + boxed ``Order`` + status_msg = Status( + time_ns=time.time_ns(), + resp='open', + oid=oid, + reqid=reqid, + + # embedded order info + req=Order( + action=action, + exec_mode='live', + oid=oid, + symbol=fqsn, + account=acc_name, + price=price, + size=size, + ), + src='kraken', ) - - # call ws api to cancel: - # https://docs.kraken.com/websockets/#message-cancelOrder - await ws.send_msg({ - 'event': 'cancelOrder', - 'token': token, - 'reqid': reqid or 0, - 'txid': [txid], - }) - continue - - # remap statuses to ems set. - ems_status = { - 'open': 'submitted', - 'closed': 'filled', - 'canceled': 'cancelled', - # do we even need to forward - # this state to the ems? - 'pending': 'pending', - }[status] - # TODO: i like the open / closed semantics - # more we should consider them for internals + apiflows[reqid].maps.append(status_msg) + await ems_stream.send(status_msg) + continue # send BrokerdStatus messages for all # order state updates @@ -912,7 +928,7 @@ async def handle_order_updates( account=f'kraken.{acctid}', # everyone doin camel case.. - status=ems_status, # force lower case + status=status, # force lower case filled=vlm, reason='', # why held? From f6ba95a6c7711a89aa9358caa7fdf72fd60a9e45 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 11 Aug 2022 22:48:01 -0400 Subject: [PATCH 19/33] Split existing live-open case into its own block --- piker/brokers/kraken/broker.py | 190 +++++++++++++++++---------------- 1 file changed, 96 insertions(+), 94 deletions(-) diff --git a/piker/brokers/kraken/broker.py b/piker/brokers/kraken/broker.py index 6bb3be17..0d50c078 100644 --- a/piker/brokers/kraken/broker.py +++ b/piker/brokers/kraken/broker.py @@ -741,6 +741,57 @@ async def handle_order_updates( txid, update_msg = list(order_msg.items())[0] match update_msg: + # EMS-unknown live order that needs to be + # delivered and loaded on the client-side. + case { + 'userref': reqid, + + # during a fill this field is **not** + # provided! but, it is always avail on + # actual status updates.. see case above. + 'status': status, + **rest, + } if ( + ids.inverse.get(reqid) is None + ): + # parse out existing live order + descr = rest['descr'] + fqsn = descr['pair'].replace( + '/', '').lower() + price = float(descr['price']) + size = float(rest['vol']) + action = descr['type'] + + # register the userref value from + # kraken (usually an `int` staring + # at 1?) as our reqid. + reqids2txids[reqid] = txid + oid = str(reqid) + ids[oid] = reqid # NOTE!: str -> int + + # fill out ``Status`` + boxed ``Order`` + status_msg = Status( + time_ns=time.time_ns(), + resp='open', + oid=oid, + reqid=reqid, + + # embedded order info + req=Order( + action=action, + exec_mode='live', + oid=oid, + symbol=fqsn, + account=acc_name, + price=price, + size=size, + ), + src='kraken', + ) + apiflows[reqid].maps.append(status_msg) + await ems_stream.send(status_msg) + continue + # XXX: eg. of full msg schema: # {'avg_price': _, # 'cost': _, @@ -819,105 +870,56 @@ async def handle_order_updates( ) oid = ids.inverse.get(reqid) + # XXX: too fast edit handled by the + # request handler task: this + # scenario occurs when ems side + # requests are coming in too quickly + # such that there is no known txid + # yet established for the ems + # dialog's last reqid when the + # request handler task is already + # receceiving a new update for that + # reqid. In this case we simply mark + # the reqid as being "too fast" and + # then when we get the next txid + # update from kraken's backend, and + # thus the new txid, we simply + # cancel the order for now. + # TODO: Ideally we eventually + # instead make the client side of + # the ems block until a submission + # is confirmed by the backend + # instead of this hacky throttle + # style approach and avoid requests + # coming in too quickly on the other + # side of the ems, aka the client + # <-> ems dialog. if ( status == 'open' - and ( - # XXX: too fast edit handled by the - # request handler task: this - # scenario occurs when ems side - # requests are coming in too quickly - # such that there is no known txid - # yet established for the ems - # dialog's last reqid when the - # request handler task is already - # receceiving a new update for that - # reqid. In this case we simply mark - # the reqid as being "too fast" and - # then when we get the next txid - # update from kraken's backend, and - # thus the new txid, we simply - # cancel the order for now. - - # TODO: Ideally we eventually - # instead make the client side of - # the ems block until a submission - # is confirmed by the backend - # instead of this hacky throttle - # style approach and avoid requests - # coming in too quickly on the other - # side of the ems, aka the client - # <-> ems dialog. - (toofast := isinstance( - reqids2txids.get(reqid), - TooFastEdit - )) - - # pre-existing open order NOT from - # this EMS session. - or (noid := oid is None) + and isinstance( + reqids2txids.get(reqid), + TooFastEdit ) ): - if toofast: - # TODO: don't even allow this case - # by not moving the client side line - # until an edit confirmation - # arrives... - log.cancel( - f'Received too fast edit {txid}:\n' - f'{update_msg}\n' - 'Cancelling order for now!..' - ) - # call ws api to cancel: - # https://docs.kraken.com/websockets/#message-cancelOrder - await ws.send_msg({ - 'event': 'cancelOrder', - 'token': token, - 'reqid': reqid or 0, - 'txid': [txid], - }) - continue - - # a non-ems-active order, emit live - # order embedded in status msg. - elif noid: - # parse out existing live order - descr = rest['descr'] - fqsn = descr['pair'].replace( - '/', '').lower() - price = float(descr['price']) - size = float(rest['vol']) - action = descr['type'] - - # register the userref value from - # kraken (usually an `int` staring - # at 1?) as our reqid. - reqids2txids[reqid] = txid - oid = str(reqid) - ids[oid] = reqid # NOTE!: str -> int - - # fill out ``Status`` + boxed ``Order`` - status_msg = Status( - time_ns=time.time_ns(), - resp='open', - oid=oid, - reqid=reqid, - - # embedded order info - req=Order( - action=action, - exec_mode='live', - oid=oid, - symbol=fqsn, - account=acc_name, - price=price, - size=size, - ), - src='kraken', - ) - apiflows[reqid].maps.append(status_msg) - await ems_stream.send(status_msg) - continue + # TODO: don't even allow this case + # by not moving the client side line + # until an edit confirmation + # arrives... + log.cancel( + f'Received too fast edit {txid}:\n' + f'{update_msg}\n' + 'Cancelling order for now!..' + ) + # call ws api to cancel: + # https://docs.kraken.com/websockets/#message-cancelOrder + await ws.send_msg({ + 'event': 'cancelOrder', + 'token': token, + 'reqid': reqid or 0, + 'txid': [txid], + }) + continue # send BrokerdStatus messages for all # order state updates From 665bb183f76b2b292fd9644009aa32f18c200a9b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 11 Aug 2022 23:02:33 -0400 Subject: [PATCH 20/33] Unpack existing live order params in case statement --- piker/brokers/kraken/broker.py | 66 ++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/piker/brokers/kraken/broker.py b/piker/brokers/kraken/broker.py index 0d50c078..3c4f8479 100644 --- a/piker/brokers/kraken/broker.py +++ b/piker/brokers/kraken/broker.py @@ -739,12 +739,44 @@ async def handle_order_updates( f'{pformat(order_msg)}' ) txid, update_msg = list(order_msg.items())[0] + + # XXX: eg. of full msg schema: + # {'avg_price': _, + # 'cost': _, + # 'descr': { + # 'close': None, + # 'leverage': None, + # 'order': descr, + # 'ordertype': 'limit', + # 'pair': 'XMR/EUR', + # 'price': '74.94000000', + # 'price2': '0.00000000', + # 'type': 'buy' + # }, + # 'expiretm': None, + # 'fee': '0.00000000', + # 'limitprice': '0.00000000', + # 'misc': '', + # 'oflags': 'fciq', + # 'opentm': '1656966131.337344', + # 'refid': None, + # 'starttm': None, + # 'stopprice': '0.00000000', + # 'timeinforce': 'GTC', + # 'vol': submit_vlm, # '13.34400854', + # 'vol_exec': exec_vlm} # 0.0000 match update_msg: # EMS-unknown live order that needs to be # delivered and loaded on the client-side. case { 'userref': reqid, + 'descr': { + 'pair': pair, + 'price': price, + 'type': action, + }, + 'vol': vol, # during a fill this field is **not** # provided! but, it is always avail on @@ -755,12 +787,9 @@ async def handle_order_updates( ids.inverse.get(reqid) is None ): # parse out existing live order - descr = rest['descr'] - fqsn = descr['pair'].replace( - '/', '').lower() - price = float(descr['price']) - size = float(rest['vol']) - action = descr['type'] + fqsn = pair.replace('/', '').lower() + price = float(price) + size = float(vol) # register the userref value from # kraken (usually an `int` staring @@ -792,31 +821,6 @@ async def handle_order_updates( await ems_stream.send(status_msg) continue - # XXX: eg. of full msg schema: - # {'avg_price': _, - # 'cost': _, - # 'descr': { - # 'close': None, - # 'leverage': None, - # 'order': descr, - # 'ordertype': 'limit', - # 'pair': 'XMR/EUR', - # 'price': '74.94000000', - # 'price2': '0.00000000', - # 'type': 'buy' - # }, - # 'expiretm': None, - # 'fee': '0.00000000', - # 'limitprice': '0.00000000', - # 'misc': '', - # 'oflags': 'fciq', - # 'opentm': '1656966131.337344', - # 'refid': None, - # 'starttm': None, - # 'stopprice': '0.00000000', - # 'timeinforce': 'GTC', - # 'vol': submit_vlm, # '13.34400854', - # 'vol_exec': exec_vlm} # 0.0000 case { 'userref': reqid, From c4af706d51031cae644eeae02bd3ba430b51d059 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 14 Aug 2022 16:16:48 -0400 Subject: [PATCH 21/33] Make order-book-vars globals to persist across ems-dialog connections --- piker/clearing/_paper_engine.py | 40 ++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index e4fbb1bb..04b52870 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -33,10 +33,10 @@ from bidict import bidict import pendulum import trio import tractor -from dataclasses import dataclass from .. import data from ..data._source import Symbol +from ..data.types import Struct from ..pp import ( Position, Transaction, @@ -58,8 +58,7 @@ from ._messages import ( log = get_logger(__name__) -@dataclass -class PaperBoi: +class PaperBoi(Struct): """ Emulates a broker order client providing the same API and delivering an order-event response stream but with methods for @@ -73,8 +72,8 @@ class PaperBoi: # map of paper "live" orders which be used # to simulate fills based on paper engine settings - _buys: bidict - _sells: bidict + _buys: dict + _sells: dict _reqids: bidict _positions: dict[str, Position] _trade_ledger: dict[str, Any] @@ -409,7 +408,6 @@ async def handle_order_requests( # action = request_msg['action'] match request_msg: - # if action in {'buy', 'sell'}: case {'action': ('buy' | 'sell')}: order = BrokerdOrder(**request_msg) account = order.account @@ -427,12 +425,6 @@ async def handle_order_requests( )) continue - # validate - # order = BrokerdOrder(**request_msg) - - # if order.reqid is None: - # reqid = - # else: reqid = order.reqid or str(uuid.uuid4()) # deliver ack that order has been submitted to broker routing @@ -475,6 +467,23 @@ async def handle_order_requests( log.error(f'Unknown order command: {request_msg}') +_reqids: bidict[str, tuple] = {} +_buys: dict[ + str, + dict[ + tuple[str, float], + tuple[float, str, str], + ] +] = {} +_sells: dict[ + str, + dict[ + tuple[str, float], + tuple[float, str, str], + ] +] = {} + + @tractor.context async def trades_dialogue( @@ -484,6 +493,7 @@ async def trades_dialogue( loglevel: str = None, ) -> None: + tractor.log.get_console_log(loglevel) async with ( @@ -505,10 +515,10 @@ async def trades_dialogue( client = PaperBoi( broker, ems_stream, - _buys={}, - _sells={}, + _buys=_buys, + _sells=_sells, - _reqids={}, + _reqids=_reqids, # TODO: load paper positions from ``positions.toml`` _positions={}, From 317610e00ae4098aa43a125169cb4bf2acb8ed80 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 14 Aug 2022 16:39:35 -0400 Subject: [PATCH 22/33] Store positions globally and deliver on ctx connects --- piker/clearing/_paper_engine.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 04b52870..609db6fb 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -482,6 +482,7 @@ _sells: dict[ tuple[float, str, str], ] ] = {} +_positions: dict[str, Position] = {} @tractor.context @@ -503,10 +504,22 @@ async def trades_dialogue( ) as feed, ): + pp_msgs: list[BrokerdPosition] = [] + pos: Position + token: str # f'{symbol}.{self.broker}' + for token, pos in _positions.items(): + pp_msgs.append(BrokerdPosition( + broker=broker, + account='paper', + symbol=fqsn, + size=pos.size, + avg_price=pos.ppu, + )) + # TODO: load paper positions per broker from .toml config file # and pass as symbol to position data mapping: ``dict[str, dict]`` # await ctx.started(all_positions) - await ctx.started(({}, ['paper'])) + await ctx.started((pp_msgs, ['paper'])) async with ( ctx.open_stream() as ems_stream, @@ -521,7 +534,7 @@ async def trades_dialogue( _reqids=_reqids, # TODO: load paper positions from ``positions.toml`` - _positions={}, + _positions=_positions, # TODO: load postions from ledger file _trade_ledger={}, From a602c47d47433bb2ccc700d7841155b19d5b44e6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 14 Aug 2022 16:42:29 -0400 Subject: [PATCH 23/33] Support loading paper engine live orders --- piker/ui/order_mode.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/piker/ui/order_mode.py b/piker/ui/order_mode.py index cecb8787..10ae8866 100644 --- a/piker/ui/order_mode.py +++ b/piker/ui/order_mode.py @@ -539,7 +539,7 @@ class OrderMode: src = msg.src if ( src - and src != 'dark' + and src not in ('dark', 'paperboi') and src not in symbol ): fqsn = symbol + '.' + src @@ -900,6 +900,7 @@ async def process_trade_msg( mode.on_submit(oid) else: + # await tractor.breakpoint() log.warning( f'received msg for untracked dialog:\n{fmsg}' ) @@ -914,7 +915,11 @@ async def process_trade_msg( # a existing dark order for the same symbol or ( order.symbol == fqsn - and (msg.src == 'dark') or (msg.src in fqsn) + and ( + msg.src in ('dark', 'paperboi') + or (msg.src in fqsn) + + ) ) ): dialog = mode.load_unknown_dialog_from_msg(msg) From 7379dc03affb89e51722d1c5538ba8b69837ea9c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 15 Aug 2022 13:36:32 -0400 Subject: [PATCH 24/33] The `ps1` check doesn't work for `pdb`.. --- piker/data/types.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/piker/data/types.py b/piker/data/types.py index 9b08b432..c23f6266 100644 --- a/piker/data/types.py +++ b/piker/data/types.py @@ -45,7 +45,10 @@ class Struct( def __repr__(self): # only turn on pprint when we detect a python REPL # at runtime B) - if hasattr(sys, 'ps1'): + if ( + hasattr(sys, 'ps1') + # TODO: check if we're in pdb + ): return f'Struct({pformat(self.to_dict())})' return super().__repr__() From ee8c00684b084abe7ab07ed92b2094048a447a57 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 15 Aug 2022 15:24:25 -0400 Subject: [PATCH 25/33] Add actor-global "broker client" for tracking reqids --- piker/brokers/kraken/broker.py | 36 ++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/piker/brokers/kraken/broker.py b/piker/brokers/kraken/broker.py index 3c4f8479..8503e049 100644 --- a/piker/brokers/kraken/broker.py +++ b/piker/brokers/kraken/broker.py @@ -31,6 +31,7 @@ import time from typing import ( Any, AsyncIterator, + Iterable, Union, ) @@ -86,6 +87,33 @@ class TooFastEdit(Exception): 'Edit requests faster then api submissions' +# TODO: make this wrap the `Client` and `ws` instances +# and give it methods to submit cancel vs. add vs. edit +# requests? +class BrokerClient: + ''' + Actor global, client-unique order manager API. + + For now provides unique ``brokerd`` defined "request ids" + and "user reference" values to track ``kraken`` ws api order + dialogs. + + ''' + counter: Iterable = count(1) + _table: set[int] = set() + + @classmethod + def new_reqid(cls) -> int: + for reqid in cls.counter: + if reqid not in cls._table: + cls._table.add(reqid) + return reqid + + @classmethod + def add_reqid(cls, reqid: int) -> None: + cls._table.add(reqid) + + async def handle_order_requests( ws: NoBsWs, @@ -105,7 +133,6 @@ async def handle_order_requests( # XXX: UGH, let's unify this.. with ``msgspec``. msg: dict[str, Any] order: BrokerdOrder - counter = count(1) async for msg in ems_order_stream: log.info(f'Rx order msg:\n{pformat(msg)}') @@ -178,7 +205,8 @@ async def handle_order_requests( else: ep = 'addOrder' - reqid = next(counter) + + reqid = BrokerClient.new_reqid() ids[order.oid] = reqid log.debug( f"Adding order {reqid}\n" @@ -798,6 +826,10 @@ async def handle_order_updates( oid = str(reqid) ids[oid] = reqid # NOTE!: str -> int + # ensure wtv reqid they give us we don't re-use on + # new order submissions to this actor's client. + BrokerClient.add_reqid(reqid) + # fill out ``Status`` + boxed ``Order`` status_msg = Status( time_ns=time.time_ns(), From be8fd32e7d6d6fddb00eb896d8b556d083da0fd6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 16 Aug 2022 09:21:47 -0400 Subject: [PATCH 26/33] Only emit ems fill msgs for 'status' events from ib Fills seems to be dual emitted from both the `status` and `fill` events in `ib_insync` internals and more or less contain the same data nested inside their `Trade` type. We started handling the 'fill' case to deal with a race issue in commissions/cost report tracking but we don't really want to leak that same race to incremental fills vs. order-"closed" tracking.. So go back to only emitting the fill msgs on statuses and a "closed" on `.remaining == 0`. --- piker/brokers/ib/broker.py | 53 ++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index da1b03c8..9679203d 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -807,8 +807,9 @@ async def deliver_trade_events( log.info(f'ib sending {event_name}:\n{pformat(item)}') match event_name: - # TODO: templating the ib statuses in comparison with other - # brokers is likely the way to go: + # NOTE: we remap statuses to the ems set via the + # ``_statuses: dict`` above. + # https://interactivebrokers.github.io/tws-api/interfaceIBApi_1_1EWrapper.html#a17f2a02d6449710b6394d0266a353313 # short list: # - PendingSubmit @@ -839,7 +840,6 @@ async def deliver_trade_events( trade: Trade = item status: OrderStatus = trade.orderStatus ib_status_key = status.status.lower() - acctid = accounts_def.inverse[trade.order.account] # double check there is no error when @@ -851,9 +851,9 @@ async def deliver_trade_events( and 'Error' not in last_log.message ): ib_status_key = trade.log[-2].status - print(ib_status_key) elif ib_status_key == 'inactive': + async def sched_cancel(): log.warning( 'OH GAWD an inactive order..scheduling a cancel\n' @@ -874,14 +874,34 @@ async def deliver_trade_events( remaining = status.remaining if ( status_key == 'filled' - and remaining == 0 ): - status_key = 'closed' + fill: Fill = trade.fills[-1] + execu: Execution = fill.execution + # execdict = asdict(execu) + # execdict.pop('acctNumber') + + msg = BrokerdFill( + # should match the value returned from + # `.submit_limit()` + reqid=execu.orderId, + time_ns=time.time_ns(), # cuz why not + action=action_map[execu.side], + size=execu.shares, + price=execu.price, + # broker_details=execdict, + # XXX: required by order mode currently + broker_time=execu.time, + ) + await ems_stream.send(msg) + + if remaining == 0: + # emit a closed status on filled statuses where + # all units were cleared. + status_key = 'closed' # skip duplicate filled updates - we get the deats # from the execution details event msg = BrokerdStatus( - reqid=trade.order.orderId, time_ns=time.time_ns(), # cuz why not account=accounts_def.inverse[trade.order.account], @@ -899,6 +919,7 @@ async def deliver_trade_events( broker_details={'name': 'ib'}, ) await ems_stream.send(msg) + continue case 'fill': @@ -914,8 +935,6 @@ async def deliver_trade_events( # https://www.python.org/dev/peps/pep-0526/#global-and-local-variable-annotations trade: Trade fill: Fill - - # TODO: maybe we can use matching to better handle these cases. trade, fill = item execu: Execution = fill.execution execid = execu.execId @@ -944,22 +963,6 @@ async def deliver_trade_events( } ) - msg = BrokerdFill( - # should match the value returned from `.submit_limit()` - reqid=execu.orderId, - time_ns=time.time_ns(), # cuz why not - - action=action_map[execu.side], - size=execu.shares, - price=execu.price, - - broker_details=trade_entry, - # XXX: required by order mode currently - broker_time=trade_entry['broker_time'], - - ) - await ems_stream.send(msg) - # 2 cases: # - fill comes first or # - comms report comes first From bafd2cb44f8d5ad0c47315c0c4d9923000dfade1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 16 Aug 2022 11:18:49 -0400 Subject: [PATCH 27/33] Only relay fills if dialog still alive --- piker/clearing/_ems.py | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 0d8693f9..542b7c4d 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -249,12 +249,6 @@ async def clear_dark_triggers( await brokerd_orders_stream.send(brokerd_msg) - # mark this entry as having sent an order - # request. the entry will be replaced once the - # target broker replies back with - # a ``BrokerdOrderAck`` msg including the - # allocated unique ``BrokerdOrderAck.reqid`` key - # generated by the broker's own systems. # book._ems_entries[oid] = live_req # book._msgflows[oid].maps.insert(0, live_req) @@ -279,6 +273,12 @@ async def clear_dark_triggers( ) # update actives + # mark this entry as having sent an order + # request. the entry will be replaced once the + # target broker replies back with + # a ``BrokerdOrderAck`` msg including the + # allocated unique ``BrokerdOrderAck.reqid`` key + # generated by the broker's own systems. if cmd.action == 'alert': # don't register the alert status (so it won't # be reloaded by clients) since it's now @@ -666,7 +666,6 @@ async def translate_and_relay_brokerd_events( # cancelled by the ems controlling client before we # received this ack, in which case we relay that cancel # signal **asap** to the backend broker - # status = book._active.get(oid) status_msg = book._active[oid] req = status_msg.req if req and req.action == 'cancel': @@ -747,14 +746,11 @@ async def translate_and_relay_brokerd_events( if status == 'closed': log.info(f'Execution for {oid} is complete!') - - # only if we already rxed a fill then probably - # this clear is fully complete? (frickin ib..) - if old_resp == 'fill': - status_msg = book._active.pop(oid) + status_msg = book._active.pop(oid) elif status == 'canceled': log.cancel(f'Cancellation for {oid} is complete!') + status_msg = book._active.pop(oid) else: # open # relayed from backend but probably not handled so @@ -775,18 +771,15 @@ async def translate_and_relay_brokerd_events( ems_client_order_stream = router.dialogues[oid] - # wtf a fill can come after 'closed' from ib? - status_msg = book._active[oid] - - # only if we already rxed a 'closed' - # this clear is fully complete? (frickin ib..) - # if status_msg.resp == 'closed': - # status_msg = book._active.pop(oid) - - status_msg.resp = 'fill' - status_msg.reqid = reqid - status_msg.brokerd_msg = msg - await ems_client_order_stream.send(status_msg) + # XXX: bleh, a fill can come after 'closed' from `ib`? + # only send a late fill event we haven't already closed + # out the dialog status locally. + status_msg = book._active.get(oid) + if status_msg: + status_msg.resp = 'fill' + status_msg.reqid = reqid + status_msg.brokerd_msg = msg + await ems_client_order_stream.send(status_msg) # ``Status`` containing an embedded order msg which # should be loaded as a "pre-existing open order" from the From 43bdd4d022076aeff0f6edaa07248ea2830d684f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 16 Aug 2022 11:19:21 -0400 Subject: [PATCH 28/33] Pass correct instrument symbol in position msgs --- piker/clearing/_paper_engine.py | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 609db6fb..2936ff59 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -253,15 +253,6 @@ class PaperBoi(Struct): status='closed', filled=size, remaining=0 if order_complete else remaining, - # broker_details={ - # 'paper_info': { - # 'oid': oid, - # }, - # 'action': action, - # 'size': size, - # 'price': price, - # 'name': self.broker, - # }, ) await self.ems_trades_stream.send(msg) @@ -270,7 +261,10 @@ class PaperBoi(Struct): pp = self._positions.setdefault( token, Position( - Symbol(key=symbol), + Symbol( + key=symbol, + broker_info={self.broker: {}}, + ), size=size, ppu=price, bsuid=symbol, @@ -403,10 +397,8 @@ async def handle_order_requests( ) -> None: - # order_request: dict + request_msg: dict async for request_msg in ems_order_stream: - - # action = request_msg['action'] match request_msg: case {'action': ('buy' | 'sell')}: order = BrokerdOrder(**request_msg) @@ -417,9 +409,7 @@ async def handle_order_requests( ' only a `paper` selection is valid' ) await ems_order_stream.send(BrokerdError( - # oid=request_msg['oid'], oid=order.oid, - # symbol=request_msg['symbol'], symbol=order.symbol, reason=f'Paper only. No account found: `{account}` ?', )) @@ -430,25 +420,18 @@ async def handle_order_requests( # deliver ack that order has been submitted to broker routing await ems_order_stream.send( BrokerdOrderAck( - - # ems order request id oid=order.oid, - - # broker specific request id reqid=reqid, - ) ) # call our client api to submit the order reqid = await client.submit_limit( - oid=order.oid, symbol=order.symbol, price=order.price, action=order.action, size=order.size, - # XXX: by default 0 tells ``ib_insync`` methods that # there is no existing order so ask the client to create # a new one (which it seems to do by allocating an int @@ -511,7 +494,7 @@ async def trades_dialogue( pp_msgs.append(BrokerdPosition( broker=broker, account='paper', - symbol=fqsn, + symbol=pos.symbol.front_fqsn(), size=pos.size, avg_price=pos.ppu, )) From 06845e5504fa325e37e5dc72281eebb3489b4589 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Aug 2022 09:37:29 -0400 Subject: [PATCH 29/33] `kraken`: drop `make_sub()` and inline sub defs in `subscribe()` --- piker/brokers/kraken/feed.py | 46 +++++++++++++++--------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/piker/brokers/kraken/feed.py b/piker/brokers/kraken/feed.py index 7c589d85..e67d204c 100644 --- a/piker/brokers/kraken/feed.py +++ b/piker/brokers/kraken/feed.py @@ -34,7 +34,6 @@ import pendulum from trio_typing import TaskStatus import tractor import trio -import wsproto from piker._cacheables import open_cached_client from piker.brokers._util import ( @@ -243,22 +242,6 @@ def normalize( return topic, quote -def make_sub(pairs: list[str], data: dict[str, Any]) -> dict[str, str]: - ''' - Create a request subscription packet dict. - - https://docs.kraken.com/websockets/#message-subscribe - - ''' - # eg. specific logic for this in kraken's sync client: - # https://github.com/krakenfx/kraken-wsclient-py/blob/master/kraken_wsclient_py/kraken_wsclient_py.py#L188 - return { - 'pair': pairs, - 'event': 'subscribe', - 'subscription': data, - } - - @acm async def open_history_client( symbol: str, @@ -381,15 +364,20 @@ async def stream_quotes( } @acm - async def subscribe(ws: wsproto.WSConnection): + async def subscribe(ws: NoBsWs): + # XXX: setup subs # https://docs.kraken.com/websockets/#message-subscribe - # specific logic for this in kraken's shitty sync client: + # specific logic for this in kraken's sync client: # https://github.com/krakenfx/kraken-wsclient-py/blob/master/kraken_wsclient_py/kraken_wsclient_py.py#L188 - ohlc_sub = make_sub( - list(ws_pairs.values()), - {'name': 'ohlc', 'interval': 1} - ) + ohlc_sub = { + 'event': 'subscribe', + 'pair': list(ws_pairs.values()), + 'subscription': { + 'name': 'ohlc', + 'interval': 1, + }, + } # TODO: we want to eventually allow unsubs which should # be completely fine to request from a separate task @@ -398,10 +386,14 @@ async def stream_quotes( await ws.send_msg(ohlc_sub) # trade data (aka L1) - l1_sub = make_sub( - list(ws_pairs.values()), - {'name': 'spread'} # 'depth': 10} - ) + l1_sub = { + 'event': 'subscribe', + 'pair': list(ws_pairs.values()), + 'subscription': { + 'name': 'spread', + # 'depth': 10} + }, + } # pull a first quote and deliver await ws.send_msg(l1_sub) From 5861839783c26804ac59a7c34fc13454cd5a6f5f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Aug 2022 13:05:02 -0400 Subject: [PATCH 30/33] Fix multi-account order loading.. We were overwriting the existing loaded orders list in the per client loop (lul) so move the def above all that. Comment out the "try-to-cancel-inactive-orders-via-task-after-timeout" stuff pertaining to https://github.com/erdewit/ib_insync/issues/363 for now since we don't have a mechanism in place to cancel the re-cancel task once the order is cancelled - plus who knows if this is even the best way to do it.. --- piker/brokers/ib/broker.py | 57 +++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index 9679203d..f9dd91ea 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -186,7 +186,7 @@ async def handle_order_requests( ) ) - if action == 'cancel': + elif action == 'cancel': msg = BrokerdCancel(**request_msg) client.submit_cancel(reqid=int(msg.reqid)) @@ -465,6 +465,7 @@ async def trades_dialogue( # TODO: we probably want to generalize this into a "ledgers" api.. ledgers: dict[str, dict] = {} tables: dict[str, PpTable] = {} + order_msgs: list[Status] = [] with ( ExitStack() as lstack, ): @@ -486,11 +487,8 @@ async def trades_dialogue( for account, proxy in proxies.items(): client = aioclients[account] - trades: list[Trade] = client.ib.openTrades() - order_msgs = [] for trade in trades: - order = trade.order quant = trade.order.totalQuantity action = order.action.lower() @@ -840,34 +838,37 @@ async def deliver_trade_events( trade: Trade = item status: OrderStatus = trade.orderStatus ib_status_key = status.status.lower() - acctid = accounts_def.inverse[trade.order.account] - # double check there is no error when - # cancelling.. gawwwd - if ib_status_key == 'cancelled': - last_log = trade.log[-1] - if ( - last_log.message - and 'Error' not in last_log.message - ): - ib_status_key = trade.log[-2].status + # TODO: try out cancelling inactive orders after delay: + # https://github.com/erdewit/ib_insync/issues/363 + # acctid = accounts_def.inverse[trade.order.account] - elif ib_status_key == 'inactive': + # # double check there is no error when + # # cancelling.. gawwwd + # if ib_status_key == 'cancelled': + # last_log = trade.log[-1] + # if ( + # last_log.message + # and 'Error' not in last_log.message + # ): + # ib_status_key = trade.log[-2].status - async def sched_cancel(): - log.warning( - 'OH GAWD an inactive order..scheduling a cancel\n' - f'{pformat(item)}' - ) - proxy = proxies[acctid] - await proxy.submit_cancel(reqid=trade.order.orderId) - await trio.sleep(1) - nurse.start_soon(sched_cancel) + # elif ib_status_key == 'inactive': - nurse.start_soon(sched_cancel) + # async def sched_cancel(): + # log.warning( + # 'OH GAWD an inactive order.scheduling a cancel\n' + # f'{pformat(item)}' + # ) + # proxy = proxies[acctid] + # await proxy.submit_cancel(reqid=trade.order.orderId) + # await trio.sleep(1) + # nurse.start_soon(sched_cancel) + + # nurse.start_soon(sched_cancel) status_key = ( - _statuses.get(ib_status_key) + _statuses.get(ib_status_key.lower()) or ib_status_key.lower() ) @@ -880,7 +881,7 @@ async def deliver_trade_events( # execdict = asdict(execu) # execdict.pop('acctNumber') - msg = BrokerdFill( + fill_msg = BrokerdFill( # should match the value returned from # `.submit_limit()` reqid=execu.orderId, @@ -892,7 +893,7 @@ async def deliver_trade_events( # XXX: required by order mode currently broker_time=execu.time, ) - await ems_stream.send(msg) + await ems_stream.send(fill_msg) if remaining == 0: # emit a closed status on filled statuses where From 973bf87e67449a2dd50560a870ade6a54d2d7f73 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 18 Aug 2022 11:27:12 -0400 Subject: [PATCH 31/33] Don't log aboout unknown status msg if no oid --- piker/clearing/_ems.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 542b7c4d..ae54615b 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -728,7 +728,6 @@ async def translate_and_relay_brokerd_events( # msg-chain/dialog. ems_client_order_stream = router.dialogues[oid] status_msg = book._active[oid] - old_resp = status_msg.resp status_msg.resp = status # retrieve existing live flow @@ -736,7 +735,8 @@ async def translate_and_relay_brokerd_events( if old_reqid and old_reqid != reqid: log.warning( f'Brokerd order id change for {oid}:\n' - f'{old_reqid} -> {reqid}' + f'{old_reqid}:{type(old_reqid)} ->' + f' {reqid}{type(reqid)}' ) status_msg.reqid = reqid # THIS LINE IS CRITICAL! @@ -856,12 +856,16 @@ async def translate_and_relay_brokerd_events( 'status': status, 'reqid': reqid, }: - status_msg = book._active[oid] - log.warning( - 'Unhandled broker status for dialog:\n' - f'{pformat(status_msg)}\n' - f'{pformat(brokerd_msg)}\n' - ) + oid = book._ems2brokerd_ids.inverse.get(reqid) + msg = f'Unhandled broker status for dialog {reqid}:\n' + if oid: + status_msg = book._active[oid] + msg += ( + f'last status msg: {pformat(status_msg)}\n\n' + f'this msg:{pformat(brokerd_msg)}\n' + ) + + log.warning(msg) case _: raise ValueError(f'Brokerd message {brokerd_msg} is invalid') From 4d2e23b5ced752e466030fb99bc97d0a6f20e75b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 18 Aug 2022 16:00:41 -0400 Subject: [PATCH 32/33] Expose level line marker via property --- piker/ui/_lines.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/piker/ui/_lines.py b/piker/ui/_lines.py index 421d4ec8..697e889f 100644 --- a/piker/ui/_lines.py +++ b/piker/ui/_lines.py @@ -421,6 +421,10 @@ class LevelLine(pg.InfiniteLine): return path + @property + def marker(self) -> LevelMarker: + return self._marker + def hoverEvent(self, ev): ''' Mouse hover callback. From b9dba48306e8a43e9234b85f680d8242ec254035 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 18 Aug 2022 16:04:44 -0400 Subject: [PATCH 33/33] Show correct account label on loaded order lines Quite a simple fix, we just assign the account-specific `PositionTracker` to the level line's `._on_level_change()` handler instead of whatever the current `OrderMode.current_pp` is set to. Further this adds proper pane switching support such that when a user modifies an order line from an account which is not the currently selected one, the settings pane is changed to reflect the account and thus corresponding position info for that account and instrument B) --- piker/ui/order_mode.py | 85 +++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/piker/ui/order_mode.py b/piker/ui/order_mode.py index 10ae8866..cbe1bf9f 100644 --- a/piker/ui/order_mode.py +++ b/piker/ui/order_mode.py @@ -79,38 +79,6 @@ class Dialog(Struct): fills: Dict[str, Any] = {} -def on_level_change_update_next_order_info( - - level: float, - - # these are all ``partial``-ed in at callback assignment time. - line: LevelLine, - order: Order, - tracker: PositionTracker, - -) -> None: - ''' - A callback applied for each level change to the line - which will recompute the order size based on allocator - settings. this is assigned inside - ``OrderMode.line_from_order()`` - - ''' - # NOTE: the ``Order.account`` is set at order stage time - # inside ``OrderMode.line_from_order()``. - order_info = tracker.alloc.next_order_info( - startup_pp=tracker.startup_pp, - live_pp=tracker.live_pp, - price=level, - action=order.action, - ) - line.update_labels(order_info) - - # update bound-in staged order - order.price = level - order.size = order_info['size'] - - @dataclass class OrderMode: ''' @@ -155,6 +123,42 @@ class OrderMode: } _staged_order: Optional[Order] = None + def on_level_change_update_next_order_info( + self, + level: float, + + # these are all ``partial``-ed in at callback assignment time. + line: LevelLine, + order: Order, + tracker: PositionTracker, + + ) -> None: + ''' + A callback applied for each level change to the line + which will recompute the order size based on allocator + settings. this is assigned inside + ``OrderMode.line_from_order()`` + + ''' + # NOTE: the ``Order.account`` is set at order stage time inside + # ``OrderMode.line_from_order()`` or is inside ``Order`` msg + # field for loaded orders. + order_info = tracker.alloc.next_order_info( + startup_pp=tracker.startup_pp, + live_pp=tracker.live_pp, + price=level, + action=order.action, + ) + line.update_labels(order_info) + + # update bound-in staged order + order.price = level + order.size = order_info['size'] + + # when an order is changed we flip the settings side-pane to + # reflect the corresponding account and pos info. + self.pane.on_ui_settings_change('account', order.account) + def line_from_order( self, order: Order, @@ -186,10 +190,12 @@ class OrderMode: # immediately if order.action != 'alert': line._on_level_change = partial( - on_level_change_update_next_order_info, + self.on_level_change_update_next_order_info, line=line, order=order, - tracker=self.current_pp, + # use the corresponding position tracker for the + # order's account. + tracker=self.trackers[order.account], ) else: @@ -238,7 +244,6 @@ class OrderMode: line = self.line_from_order( order, - show_markers=True, # just for the stage line to avoid # flickering while moving the cursor @@ -250,7 +255,6 @@ class OrderMode: # prevent flickering of marker while moving/tracking cursor only_show_markers_on_hover=False, ) - line = self.lines.stage_line(line) # hide crosshair y-line and label @@ -274,10 +278,8 @@ class OrderMode: ''' if not order: staged = self._staged_order + # apply order fields for ems oid = str(uuid.uuid4()) - # symbol: Symbol = staged.symbol - - # format order data for ems order = staged.copy() order.oid = oid @@ -528,10 +530,9 @@ class OrderMode: msg: Status, ) -> Dialog: - # NOTE: the `.order` attr **must** be set with the # equivalent order msg in order to be loaded. - order = Order(**msg.req) + order = msg.req oid = str(msg.oid) symbol = order.symbol @@ -900,7 +901,6 @@ async def process_trade_msg( mode.on_submit(oid) else: - # await tractor.breakpoint() log.warning( f'received msg for untracked dialog:\n{fmsg}' ) @@ -922,6 +922,7 @@ async def process_trade_msg( ) ) ): + msg.req = order dialog = mode.load_unknown_dialog_from_msg(msg) mode.on_submit(oid) # return dialog, msg