Drop status event processing at large

Since we figured out how to pass through ems dialog ids to the
`openOrders` sub we don't really need to do much with status updates
other then error handling. This drops `process_status()` and moves the
error handling logic into a status handler sub-block; we now just
info-log status updates for troubleshooting purposes.
kraken_userref_hackzin
Tyler Goodlet 2022-08-01 14:08:45 -04:00
parent 1cbf45b4c4
commit 69e501764a
2 changed files with 59 additions and 143 deletions

View File

@ -140,11 +140,8 @@ async def handle_order_requests(
try: try:
txid = reqids2txids[reqid] txid = reqids2txids[reqid]
except KeyError: except KeyError:
assert 0
# XXX: not sure if this block ever gets hit now? # XXX: not sure if this block ever gets hit now?
log.error('TOO FAST EDIT') log.error('TOO FAST EDIT')
reqids2txids[reqid] = TooFastEdit(reqid) reqids2txids[reqid] = TooFastEdit(reqid)
await ems_order_stream.send( await ems_order_stream.send(
BrokerdError( BrokerdError(
@ -971,68 +968,15 @@ async def handle_order_updates(
chain = apiflows[reqid] chain = apiflows[reqid]
chain.maps.append(event) chain.maps.append(event)
resps, errored = process_status( if status == 'error':
event,
oid,
token,
chain,
reqids2txids,
)
if resps:
for resp in resps:
await ems_stream.send(resp)
txid = txid or lasttxid
if (
# errored likely on a rate limit or bad input
errored
and txid
# we throttle too-fast-requests on the ems side
or (txid and isinstance(txid, TooFastEdit))
):
# client was editting too quickly
# so we instead cancel this order
log.cancel(
f'Cancelling {reqid}@{txid} due to error:\n {event}')
await ws.send_msg({
'event': 'cancelOrder',
'token': token,
'reqid': reqid or 0,
'txid': [txid],
})
case _:
log.warning(f'Unhandled trades update msg: {msg}')
def process_status(
event: dict[str, str],
oid: str,
token: str,
chain: ChainMap,
reqids2txids: dict[int, str],
) -> tuple[list[MsgUnion], bool]:
'''
Process `'[add/edit/cancel]OrderStatus'` events by translating to
and returning the equivalent EMS-msg responses.
'''
match event:
case {
'event': etype,
'status': 'error',
'reqid': reqid,
'errorMessage': errmsg,
}:
# any of ``{'add', 'edit', 'cancel'}`` # any of ``{'add', 'edit', 'cancel'}``
action = etype.removesuffix('OrderStatus') action = etype.removesuffix('OrderStatus')
errmsg = rest['errorMessage']
log.error( log.error(
f'Failed to {action} order {reqid}:\n' f'Failed to {action} order {reqid}:\n'
f'{errmsg}' f'{errmsg}'
) )
resp = BrokerdError( await ems_stream.send(BrokerdError(
oid=oid, oid=oid,
# XXX: use old reqid in case it changed? # XXX: use old reqid in case it changed?
reqid=reqid, reqid=reqid,
@ -1040,68 +984,28 @@ def process_status(
reason=f'Failed {action}:\n{errmsg}', reason=f'Failed {action}:\n{errmsg}',
broker_details=event broker_details=event
) ))
return [resp], True
# successful request cases txid = txid or lasttxid
case { if (
'event': 'addOrderStatus', txid
'status': "ok",
'reqid': reqid, # oid from ems side
'txid': txid,
'descr': descr, # only on success?
}:
log.info(
f'Submitted order: {descr}\n'
f'ems oid: {oid}\n'
f'brokerd reqid: {reqid}\n'
f'txid: {txid}\n'
)
return [], False
case { # we throttle too-fast-requests on the ems side
'event': 'editOrderStatus', or (txid and isinstance(txid, TooFastEdit))
'status': "ok", ):
'reqid': reqid, # oid from ems side # client was editting too quickly
'descr': descr, # so we instead cancel this order
log.cancel(
f'Cancelling {reqid}@{txid} due to:\n {event}')
await ws.send_msg({
'event': 'cancelOrder',
'token': token,
'reqid': reqid or 0,
'txid': [txid],
})
# NOTE: for edit request this is a new value case _:
'txid': txid, log.warning(f'Unhandled trades update msg: {msg}')
'originaltxid': origtxid,
}:
log.info(
f'Editting order {oid}[requid={reqid}]:\n'
f'brokerd reqid: {reqid}\n'
f'txid: {origtxid} -> {txid}\n'
f'{descr}'
)
# XXX: update the expected txid since the ``openOrders`` sub
# doesn't relay through the ``userref`` value..
# (hopefully kraken will fix this so we don't need this
# line.)
# reqids2txids[reqid] = txid
# deliver another ack to update the ems-side `.reqid`.
return [], False
case {
"event": "cancelOrderStatus",
"status": "ok",
'reqid': reqid,
# XXX: sometimes this isn't provided!?
# 'txid': txids,
**rest,
}:
for txid in rest.get('txid', [chain['reqid']]):
log.info(
f'Cancelling order {oid}[requid={reqid}]:\n'
f'brokerd reqid: {reqid}\n'
)
# if txid == reqids2txids[reqid]:
# reqids2txids.pop(reqid)
return [], False
def norm_trade_records( def norm_trade_records(

View File

@ -39,7 +39,11 @@ from docker.errors import (
APIError, APIError,
# ContainerError, # ContainerError,
) )
from requests.exceptions import ConnectionError, ReadTimeout import requests
from requests.exceptions import (
ConnectionError,
ReadTimeout,
)
from ..log import get_logger, get_console_log from ..log import get_logger, get_console_log
from .. import config from .. import config
@ -188,13 +192,12 @@ class Container:
def hard_kill(self, start: float) -> None: def hard_kill(self, start: float) -> None:
delay = time.time() - start delay = time.time() - start
log.error(
f'Failed to kill container {self.cntr.id} after {delay}s\n'
'sending SIGKILL..'
)
# get out the big guns, bc apparently marketstore # get out the big guns, bc apparently marketstore
# doesn't actually know how to terminate gracefully # doesn't actually know how to terminate gracefully
# :eyeroll:... # :eyeroll:...
log.error(
f'SIGKILL-ing: {self.cntr.id} after {delay}s\n'
)
self.try_signal('SIGKILL') self.try_signal('SIGKILL')
self.cntr.wait( self.cntr.wait(
timeout=3, timeout=3,
@ -218,21 +221,26 @@ class Container:
self.try_signal('SIGINT') self.try_signal('SIGINT')
start = time.time() start = time.time()
for _ in range(30): for _ in range(6):
with trio.move_on_after(0.5) as cs: with trio.move_on_after(0.5) as cs:
cs.shield = True
log.cancel('polling for CNTR logs...') log.cancel('polling for CNTR logs...')
try: try:
await self.process_logs_until(stop_msg) await self.process_logs_until(stop_msg)
except ApplicationLogError: except ApplicationLogError:
hard_kill = True hard_kill = True
else:
# if we aren't cancelled on above checkpoint then we # if we aren't cancelled on above checkpoint then we
# assume we read the expected stop msg and terminated. # assume we read the expected stop msg and
# terminated.
break break
if cs.cancelled_caught:
# on timeout just try a hard kill after
# a quick container sync-wait.
hard_kill = True
try: try:
log.info(f'Polling for container shutdown:\n{cid}') log.info(f'Polling for container shutdown:\n{cid}')
@ -254,9 +262,16 @@ class Container:
except ( except (
docker.errors.APIError, docker.errors.APIError,
ConnectionError, ConnectionError,
requests.exceptions.ConnectionError,
trio.Cancelled,
): ):
log.exception('Docker connection failure') log.exception('Docker connection failure')
self.hard_kill(start) self.hard_kill(start)
raise
except trio.Cancelled:
log.exception('trio cancelled...')
self.hard_kill(start)
else: else:
hard_kill = True hard_kill = True
@ -305,15 +320,12 @@ async def open_ahabd(
)) ))
try: try:
# TODO: we might eventually want a proxy-style msg-prot here # TODO: we might eventually want a proxy-style msg-prot here
# to allow remote control of containers without needing # to allow remote control of containers without needing
# callers to have root perms? # callers to have root perms?
await trio.sleep_forever() await trio.sleep_forever()
finally: finally:
# needed?
with trio.CancelScope(shield=True):
await cntr.cancel(stop_msg) await cntr.cancel(stop_msg)