`_ems`: tolerate and warn on already popped execs

In the `translate_and_relay_brokerd_events()` loop task that is, such
that we never crash on a `status_msg = book._active.pop(oid)` in the
'closed' status handler whenever a double removal happens.

Turns out there were unforeseen races here when a benign backend error
would cause an order-mode dialog to be cancelled (incorrectly) and then
a UI side `.on_cancel()` would trigger too-early removal from the
`book._active` table despite the backend sending an actual 'closed'
event (much) later, this would crash on the now missing entry..

So instead we now,
- obviously use `book._active.pop(oid, None)`
- emit a `log.warning()` (not info lol) on a null-read and with a less
  "one-line-y" message explaining the double removal and maybe *why*.
qt_w_graceful_SIGINT
Tyler Goodlet 2025-09-29 13:21:11 -04:00
parent 62f27bf509
commit 0462415491
1 changed files with 19 additions and 9 deletions

View File

@ -388,6 +388,7 @@ async def open_brokerd_dialog(
for ep_name in [
'open_trade_dialog', # probably final name?
'trades_dialogue', # legacy
# ^!TODO, rm this since all backends ported no ?!?
]:
trades_endpoint = getattr(
brokermod,
@ -1019,8 +1020,18 @@ async def translate_and_relay_brokerd_events(
)
if status == 'closed':
log.info(f'Execution for {oid} is complete!')
status_msg = book._active.pop(oid)
log.info(
f'Execution is complete!\n'
f'oid: {oid!r}\n'
)
status_msg = book._active.pop(oid, None)
if status_msg is None:
log.warning(
f'Order was already cleared from book ??\n'
f'oid: {oid!r}\n'
f'\n'
f'Maybe the order cancelled before submitted ??\n'
)
elif status == 'canceled':
log.cancel(f'Cancellation for {oid} is complete!')
@ -1544,19 +1555,18 @@ async def maybe_open_trade_relays(
@tractor.context
async def _emsd_main(
ctx: tractor.Context,
ctx: tractor.Context, # becomes `ems_ctx` below
fqme: str,
exec_mode: str, # ('paper', 'live')
loglevel: str|None = None,
) -> tuple[
dict[
# brokername, acctid
tuple[str, str],
) -> tuple[ # `ctx.started()` value!
dict[ # positions
tuple[str, str], # brokername, acctid
list[BrokerdPosition],
],
list[str],
dict[str, Status],
list[str], # accounts
dict[str, Status], # dialogs
]:
'''
EMS (sub)actor entrypoint providing the execution management