pikers
/
piker
3
0
Fork 0
Code Issues 4 Pull Requests 6 Packages Projects Releases Wiki Activity

Merge pull request #190 from pikers/ems_to_bidir_streaming 

Ems to bidir streaming
ems_hotfixes
goodboy 2021-06-10 08:45:44 -04:00 committed by GitHub
parent ee65382d97 a9cc3210d8
commit 689bc0cde0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 1349 additions and 677 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
piker/_daemon.py
View File

@ -102,7 +102,9 @@ async def open_pikerd(
assert _services is None
# XXX: this may open a root actor as well
async with tractor.open_root_actor(
async with (
tractor.open_root_actor(
# passed through to ``open_root_actor``
arbiter_addr=_tractor_kwargs['arbiter_addr'],
name=_root_dname,
@ -113,10 +115,10 @@ async def open_pikerd(
# TODO: eventually we should be able to avoid
# having the root have more then permissions to
# spawn other specialized daemons I think?
# enable_modules=[__name__],
enable_modules=_root_modules,
) as _, tractor.open_nursery() as actor_nursery:
) as _,
tractor.open_nursery() as actor_nursery,
):
async with trio.open_nursery() as service_nursery:
# setup service mngr singleton instance
@ -137,6 +139,7 @@ async def open_pikerd(
async def maybe_open_runtime(
loglevel: Optional[str] = None,
**kwargs,
) -> None:
"""
Start the ``tractor`` runtime (a root actor) if none exists.
@ -159,6 +162,7 @@ async def maybe_open_runtime(
async def maybe_open_pikerd(
loglevel: Optional[str] = None,
**kwargs,
) -> Union[tractor._portal.Portal, Services]:
"""If no ``pikerd`` daemon-root-actor can be found start it and
yield up (we should probably figure out returning a portal to self
@ -197,6 +201,66 @@ _data_mods = [
]
class Brokerd:
locks = defaultdict(trio.Lock)
@asynccontextmanager
async def maybe_spawn_daemon(
service_name: str,
spawn_func: Callable,
spawn_args: dict[str, Any],
loglevel: Optional[str] = None,
**kwargs,
) -> tractor.Portal:
"""
If no ``service_name`` daemon-actor can be found,
spawn one in a local subactor and return a portal to it.
"""
if loglevel:
get_console_log(loglevel)
# serialize access to this section to avoid
# 2 or more tasks racing to create a daemon
lock = Brokerd.locks[service_name]
await lock.acquire()
# attach to existing brokerd if possible
async with tractor.find_actor(service_name) as portal:
if portal is not None:
lock.release()
yield portal
return
# ask root ``pikerd`` daemon to spawn the daemon we need if
# pikerd is not live we now become the root of the
# process tree
async with maybe_open_pikerd(
loglevel=loglevel,
**kwargs,
) as pikerd_portal:
if pikerd_portal is None:
# we are root so spawn brokerd directly in our tree
# the root nursery is accessed through process global state
await spawn_func(**spawn_args)
else:
await pikerd_portal.run(
spawn_func,
**spawn_args,
)
async with tractor.wait_for_actor(service_name) as portal:
lock.release()
yield portal
async def spawn_brokerd(
brokername: str,
@ -205,8 +269,6 @@ async def spawn_brokerd(
) -> tractor._portal.Portal:
from .data import _setup_persistent_brokerd
log.info(f'Spawning {brokername} broker daemon')
brokermod = get_brokermod(brokername)
@ -226,13 +288,9 @@ async def spawn_brokerd(
**tractor_kwargs
)
# TODO: so i think this is the perfect use case for supporting
# a cross-actor async context manager api instead of this
# shoort-and-forget task spawned in the root nursery, we'd have an
# async exit stack that we'd register the `portal.open_context()`
# call with and then have the ability to unwind the call whenevs.
# non-blocking setup of brokerd service nursery
from .data import _setup_persistent_brokerd
await _services.open_remote_ctx(
portal,
_setup_persistent_brokerd,
@ -242,10 +300,6 @@ async def spawn_brokerd(
return dname
class Brokerd:
locks = defaultdict(trio.Lock)
@asynccontextmanager
async def maybe_spawn_brokerd(
@ -253,57 +307,24 @@ async def maybe_spawn_brokerd(
loglevel: Optional[str] = None,
**kwargs,
) -> tractor._portal.Portal:
"""
If no ``brokerd.{brokername}`` daemon-actor can be found,
spawn one in a local subactor and return a portal to it.
) -> tractor.Portal:
'''Helper to spawn a brokerd service.
"""
if loglevel:
get_console_log(loglevel)
'''
async with maybe_spawn_daemon(
dname = f'brokerd.{brokername}'
# serialize access to this section to avoid
# 2 or more tasks racing to create a daemon
lock = Brokerd.locks[brokername]
await lock.acquire()
# attach to existing brokerd if possible
async with tractor.find_actor(dname) as portal:
if portal is not None:
lock.release()
yield portal
return
# ask root ``pikerd`` daemon to spawn the daemon we need if
# pikerd is not live we now become the root of the
# process tree
async with maybe_open_pikerd(
f'brokerd.{brokername}',
spawn_func=spawn_brokerd,
spawn_args={'brokername': brokername, 'loglevel': loglevel},
loglevel=loglevel,
**kwargs,
) as pikerd_portal:
if pikerd_portal is None:
# we are root so spawn brokerd directly in our tree
# the root nursery is accessed through process global state
await spawn_brokerd(brokername, loglevel=loglevel)
else:
await pikerd_portal.run(
spawn_brokerd,
brokername=brokername,
loglevel=loglevel,
)
async with tractor.wait_for_actor(dname) as portal:
lock.release()
) as portal:
yield portal
async def spawn_emsd(
brokername: str,
loglevel: Optional[str] = None,
**extra_tractor_kwargs
@ -314,10 +335,10 @@ async def spawn_emsd(
"""
log.info('Spawning emsd')
# TODO: raise exception when _services == None?
global _services
assert _services
await _services.actor_n.start_actor(
portal = await _services.actor_n.start_actor(
'emsd',
enable_modules=[
'piker.clearing._ems',
@ -327,4 +348,34 @@ async def spawn_emsd(
debug_mode=_services.debug_mode, # set by pikerd flag
**extra_tractor_kwargs
)
# non-blocking setup of clearing service
from .clearing._ems import _setup_persistent_emsd
await _services.open_remote_ctx(
portal,
_setup_persistent_emsd,
)
return 'emsd'
@asynccontextmanager
async def maybe_open_emsd(
brokername: str,
loglevel: Optional[str] = None,
**kwargs,
) -> tractor._portal.Portal: # noqa
async with maybe_spawn_daemon(
'emsd',
spawn_func=spawn_emsd,
spawn_args={'loglevel': loglevel},
loglevel=loglevel,
**kwargs,
) as portal:
yield portal

243
piker/brokers/ib.py
View File

@ -25,7 +25,7 @@ from contextlib import asynccontextmanager
from dataclasses import asdict
from datetime import datetime
from functools import partial
from typing import List, Dict, Any, Tuple, Optional, AsyncIterator, Callable
from typing import List, Dict, Any, Tuple, Optional, AsyncIterator
import asyncio
from pprint import pformat
import inspect
@ -39,7 +39,8 @@ import tractor
from async_generator import aclosing
from ib_insync.wrapper import RequestError
from ib_insync.contract import Contract, ContractDetails, Option
from ib_insync.order import Order
from ib_insync.order import Order, Trade, OrderStatus
from ib_insync.objects import Fill, Execution
from ib_insync.ticker import Ticker
from ib_insync.objects import Position
import ib_insync as ibis
@ -53,6 +54,12 @@ from .._daemon import maybe_spawn_brokerd
from ..data._source import from_df
from ..data._sharedmem import ShmArray
from ._util import SymbolNotFound, NoData
from ..clearing._messages import (
BrokerdOrder, BrokerdOrderAck, BrokerdStatus,
BrokerdPosition, BrokerdCancel,
BrokerdFill,
# BrokerdError,
)
log = get_logger(__name__)
@ -472,7 +479,7 @@ class Client:
# XXX: by default 0 tells ``ib_insync`` methods that there is no
# existing order so ask the client to create a new one (which it
# seems to do by allocating an int counter - collision prone..)
brid: int = None,
reqid: int = None,
) -> int:
"""Place an order and return integer request id provided by client.
@ -488,7 +495,7 @@ class Client:
trade = self.ib.placeOrder(
contract,
Order(
orderId=brid or 0, # stupid api devs..
orderId=reqid or 0, # stupid api devs..
action=action.upper(), # BUY/SELL
orderType='LMT',
lmtPrice=price,
@ -582,6 +589,7 @@ class Client:
self,
to_trio: trio.abc.SendChannel,
) -> None:
# connect error msgs
def push_err(
reqId: int,
@ -589,13 +597,16 @@ class Client:
errorString: str,
contract: Contract,
) -> None:
log.error(errorString)
try:
to_trio.send_nowait((
'error',
# error "object"
{'reqid': reqId,
'message': errorString,
'reason': errorString,
'contract': contract}
))
except trio.BrokenResourceError:
@ -635,6 +646,8 @@ async def _aio_get_client(
"""Return an ``ib_insync.IB`` instance wrapped in our client API.
Client instances are cached for later use.
TODO: consider doing this with a ctx mngr eventually?
"""
# first check cache for existing client
@ -738,7 +751,7 @@ async def _trio_run_client_method(
class _MethodProxy:
def __init__(
self,
portal: tractor._portal.Portal
portal: tractor.Portal
) -> None:
self._portal = portal
@ -755,7 +768,12 @@ class _MethodProxy:
)
def get_client_proxy(portal, target=Client) -> _MethodProxy:
def get_client_proxy(
portal: tractor.Portal,
target=Client,
) -> _MethodProxy:
proxy = _MethodProxy(portal)
@ -843,7 +861,7 @@ async def get_bars(
end_dt: str = "",
) -> (dict, np.ndarray):
_err = None
_err: Optional[Exception] = None
fails = 0
for _ in range(2):
@ -880,12 +898,12 @@ async def get_bars(
raise NoData(f'Symbol: {sym}')
break
else:
log.exception(
"Data query rate reached: Press `ctrl-alt-f`"
"in TWS"
)
print(_err)
# TODO: should probably create some alert on screen
# and then somehow get that to trigger an event here
@ -932,7 +950,7 @@ async def backfill_bars(
if fails is None or fails > 1:
break
if out is (None, None):
if out == (None, None):
# could be trying to retreive bars over weekend
# TODO: add logic here to handle tradable hours and only grab
# valid bars in the range
@ -1183,112 +1201,213 @@ def pack_position(pos: Position) -> Dict[str, Any]:
else:
symbol = con.symbol
return {
'broker': 'ib',
'account': pos.account,
'symbol': symbol,
'currency': con.currency,
'size': float(pos.position),
'avg_price': float(pos.avgCost) / float(con.multiplier or 1.0),
}
@tractor.msg.pub(
send_on_connect={'local_trades': 'start'}
return BrokerdPosition(
broker='ib',
account=pos.account,
symbol=symbol,
currency=con.currency,
size=float(pos.position),
avg_price=float(pos.avgCost) / float(con.multiplier or 1.0),
)
async def stream_trades(
async def handle_order_requests(
ems_order_stream: tractor.MsgStream,
) -> None:
# request_msg: dict
async for request_msg in ems_order_stream:
log.info(f'Received order request {request_msg}')
action = request_msg['action']
if action in {'buy', 'sell'}:
# validate
order = BrokerdOrder(**request_msg)
# call our client api to submit the order
reqid = await _trio_run_client_method(
method='submit_limit',
oid=order.oid,
symbol=order.symbol,
price=order.price,
action=order.action,
size=order.size,
# XXX: by default 0 tells ``ib_insync`` methods that
# there is no existing order so ask the client to create
# a new one (which it seems to do by allocating an int
# counter - collision prone..)
reqid=order.reqid,
)
# deliver ack that order has been submitted to broker routing
await ems_order_stream.send(
BrokerdOrderAck(
# ems order request id
oid=order.oid,
# broker specific request id
reqid=reqid,
time_ns=time.time_ns(),
).dict()
)
elif action == 'cancel':
msg = BrokerdCancel(**request_msg)
await _trio_run_client_method(
method='submit_cancel',
reqid=msg.reqid
)
else:
log.error(f'Unknown order command: {request_msg}')
@tractor.context
async def trades_dialogue(
ctx: tractor.Context,
loglevel: str = None,
get_topics: Callable = None,
) -> AsyncIterator[Dict[str, Any]]:
# XXX: required to propagate ``tractor`` loglevel to piker logging
get_console_log(loglevel or tractor.current_actor().loglevel)
stream = await _trio_run_client_method(
ib_trade_events_stream = await _trio_run_client_method(
method='recv_trade_updates',
)
# deliver positions to subscriber before anything else
positions = await _trio_run_client_method(method='positions')
all_positions = {}
for pos in positions:
yield {'local_trades': ('position', pack_position(pos))}
msg = pack_position(pos)
all_positions[msg.symbol] = msg.dict()
await ctx.started(all_positions)
action_map = {'BOT': 'buy', 'SLD': 'sell'}
async for event_name, item in stream:
async with (
ctx.open_stream() as ems_stream,
trio.open_nursery() as n,
):
# start order request handler **before** local trades event loop
n.start_soon(handle_order_requests, ems_stream)
async for event_name, item in ib_trade_events_stream:
# XXX: begin normalization of nonsense ib_insync internal
# object-state tracking representations...
if event_name == 'status':
# unwrap needed data from ib_insync internal objects
trade = item
status = trade.orderStatus
# unwrap needed data from ib_insync internal types
trade: Trade = item
status: OrderStatus = trade.orderStatus
# skip duplicate filled updates - we get the deats
# from the execution details event
msg = {
'reqid': trade.order.orderId,
'status': status.status,
'filled': status.filled,
'reason': status.whyHeld,
msg = BrokerdStatus(
reqid=trade.order.orderId,
time_ns=time.time_ns(), # cuz why not
status=status.status.lower(), # force lower case
filled=status.filled,
reason=status.whyHeld,
# this seems to not be necessarily up to date in the
# execDetails event.. so we have to send it here I guess?
'remaining': status.remaining,
}
remaining=status.remaining,
broker_details={'name': 'ib'},
)
elif event_name == 'fill':
# for wtv reason this is a separate event type
# from IB, not sure why it's needed other then for extra
# complexity and over-engineering :eyeroll:.
# we may just end up dropping these events (or
# translating them to ``Status`` msgs) if we can
# show the equivalent status events are no more latent.
# unpack ib_insync types
# pep-0526 style:
# https://www.python.org/dev/peps/pep-0526/#global-and-local-variable-annotations
trade: Trade
fill: Fill
trade, fill = item
execu = fill.execution
execu: Execution = fill.execution
msg = {
'reqid': execu.orderId,
'execid': execu.execId,
# supposedly IB server fill time
'broker_time': execu.time, # converted to float by us
# ns from main TCP handler by us inside ``ib_insync`` override
'time': fill.time,
'time_ns': time.time_ns(), # cuz why not
'action': action_map[execu.side],
'size': execu.shares,
'price': execu.price,
# TODO: normalize out commissions details?
details = {
'contract': asdict(fill.contract),
'execution': asdict(fill.execution),
'commissions': asdict(fill.commissionReport),
'broker_time': execu.time, # supposedly IB server fill time
'name': 'ib',
}
msg = BrokerdFill(
# should match the value returned from `.submit_limit()`
reqid=execu.orderId,
time_ns=time.time_ns(), # cuz why not
action=action_map[execu.side],
size=execu.shares,
price=execu.price,
broker_details=details,
# XXX: required by order mode currently
broker_time=details['broker_time'],
)
elif event_name == 'error':
msg = item
err: dict = item
# f$#$% gawd dammit insync..
con = msg['contract']
con = err['contract']
if isinstance(con, Contract):
msg['contract'] = asdict(con)
err['contract'] = asdict(con)
if msg['reqid'] == -1:
log.error(pformat(msg))
if err['reqid'] == -1:
log.error(f'TWS external order error:\n{pformat(err)}')
# don't forward, it's pointless..
# don't forward for now, it's unecessary.. but if we wanted to,
# msg = BrokerdError(**err)
continue
elif event_name == 'position':
msg = pack_position(item)
if msg.get('reqid', 0) < -1:
if getattr(msg, 'reqid', 0) < -1:
# it's a trade event generated by TWS usage.
log.warning(f"TWS triggered trade:\n{pformat(msg)}")
msg['reqid'] = 'tws-' + str(-1 * msg['reqid'])
msg.reqid = 'tws-' + str(-1 * msg.reqid)
# mark msg as from "external system"
# TODO: probably something better then this..
# TODO: probably something better then this.. and start
# considering multiplayer/group trades tracking
msg['external'] = True
yield {'remote_trades': (event_name, msg)}
continue
yield {'local_trades': (event_name, msg)}
# XXX: we always serialize to a dict for msgpack
# translations, ideally we can move to an msgspec (or other)
# encoder # that can be enabled in ``tractor`` ahead of
# time so we can pass through the message types directly.
await ems_stream.send(msg.dict())
@tractor.context

143
piker/clearing/_client.py
View File

@ -19,32 +19,23 @@ Orders and execution client API.
"""
from contextlib import asynccontextmanager
from typing import Dict, Tuple, List
from typing import Dict
from pprint import pformat
from dataclasses import dataclass, field
import trio
import tractor
# import msgspec
from ..data._source import Symbol
from ..log import get_logger
from ._ems import _emsd_main
from .._daemon import maybe_open_emsd
from ._messages import Order, Cancel
log = get_logger(__name__)
# class Order(msgspec.Struct):
# action: str
# price: float
# size: float
# symbol: str
# brokers: List[str]
# oid: str
# exec_mode: str
@dataclass
class OrderBook:
"""Buy-side (client-side ?) order book ctl and tracking.
@ -62,31 +53,34 @@ class OrderBook:
_to_ems: trio.abc.SendChannel
_from_order_book: trio.abc.ReceiveChannel
_sent_orders: Dict[str, dict] = field(default_factory=dict)
_sent_orders: Dict[str, Order] = field(default_factory=dict)
_ready_to_receive: trio.Event = trio.Event()
def send(
self,
uuid: str,
symbol: str,
brokers: List[str],
brokers: list[str],
price: float,
size: float,
action: str,
exec_mode: str,
) -> dict:
cmd = {
'action': action,
'price': price,
'size': size,
'symbol': symbol,
'brokers': brokers,
'oid': uuid,
'exec_mode': exec_mode, # dark or live
}
self._sent_orders[uuid] = cmd
self._to_ems.send_nowait(cmd)
return cmd
msg = Order(
action=action,
price=price,
size=size,
symbol=symbol,
brokers=brokers,
oid=uuid,
exec_mode=exec_mode, # dark or live
)
self._sent_orders[uuid] = msg
self._to_ems.send_nowait(msg.dict())
return msg
def update(
self,
@ -94,29 +88,29 @@ class OrderBook:
**data: dict,
) -> dict:
cmd = self._sent_orders[uuid]
cmd.update(data)
self._sent_orders[uuid] = cmd
self._to_ems.send_nowait(cmd)
msg = cmd.dict()
msg.update(data)
self._sent_orders[uuid] = Order(**msg)
self._to_ems.send_nowait(msg)
return cmd
def cancel(self, uuid: str) -> bool:
"""Cancel an order (or alert) from the EMS.
"""Cancel an order (or alert) in the EMS.
"""
cmd = self._sent_orders[uuid]
msg = {
'action': 'cancel',
'oid': uuid,
'symbol': cmd['symbol'],
}
self._to_ems.send_nowait(msg)
msg = Cancel(
oid=uuid,
symbol=cmd.symbol,
)
self._to_ems.send_nowait(msg.dict())
_orders: OrderBook = None
def get_orders(
emsd_uid: Tuple[str, str] = None
emsd_uid: tuple[str, str] = None
) -> OrderBook:
""""
OrderBook singleton factory per actor.
@ -136,7 +130,14 @@ def get_orders(
return _orders
async def send_order_cmds(symbol_key: str):
# TODO: we can get rid of this relay loop once we move
# order_mode inputs to async code!
async def relay_order_cmds_from_sync_code(
symbol_key: str,
to_ems_stream: tractor.MsgStream,
) -> None:
"""
Order streaming task: deliver orders transmitted from UI
to downstream consumers.
@ -156,16 +157,15 @@ async def send_order_cmds(symbol_key: str):
book = get_orders()
orders_stream = book._from_order_book
# signal that ems connection is up and ready
book._ready_to_receive.set()
async for cmd in orders_stream:
print(cmd)
if cmd['symbol'] == symbol_key:
# send msg over IPC / wire
log.info(f'Send order cmd:\n{pformat(cmd)}')
yield cmd
await to_ems_stream.send(cmd)
else:
# XXX BRUTAL HACKZORZES !!!
# re-insert for another consumer
@ -174,36 +174,12 @@ async def send_order_cmds(symbol_key: str):
book._to_ems.send_nowait(cmd)
@asynccontextmanager
async def maybe_open_emsd(
brokername: str,
) -> tractor._portal.Portal: # noqa
async with tractor.find_actor('emsd') as portal:
if portal is not None:
yield portal
return
# ask remote daemon tree to spawn it
from .._daemon import spawn_emsd
async with tractor.find_actor('pikerd') as portal:
assert portal
name = await portal.run(
spawn_emsd,
brokername=brokername,
)
async with tractor.wait_for_actor(name) as portal:
yield portal
@asynccontextmanager
async def open_ems(
broker: str,
symbol: Symbol,
) -> None:
) -> (OrderBook, tractor.MsgStream, dict):
"""Spawn an EMS daemon and begin sending orders and receiving
alerts.
@ -237,32 +213,31 @@ async def open_ems(
- 'broker_filled'
"""
actor = tractor.current_actor()
# wait for service to connect back to us signalling
# ready for order commands
book = get_orders()
async with maybe_open_emsd(broker) as portal:
async with portal.open_stream_from(
async with (
# connect to emsd
portal.open_context(
_emsd_main,
client_actor_name=actor.name,
broker=broker,
symbol=symbol.key,
) as trades_stream:
with trio.fail_after(10):
await book._ready_to_receive.wait()
) as (ctx, positions),
try:
yield book, trades_stream
# open 2-way trade command stream
ctx.open_stream() as trades_stream,
):
async with trio.open_nursery() as n:
n.start_soon(
relay_order_cmds_from_sync_code,
symbol.key,
trades_stream
)
finally:
# TODO: we want to eventually keep this up (by having
# the exec loop keep running in the pikerd tree) but for
# now we have to kill the context to avoid backpressure
# build-up on the shm write loop.
with trio.CancelScope(shield=True):
await trades_stream.aclose()
yield book, trades_stream, positions

669
piker/clearing/_ems.py
View File

@ -21,19 +21,22 @@ In da suit parlances: "Execution management systems"
from pprint import pformat
import time
from dataclasses import dataclass, field
from typing import (
AsyncIterator, Dict, Callable, Tuple,
)
from typing import AsyncIterator, Callable, Any
from bidict import bidict
from pydantic import BaseModel
import trio
from trio_typing import TaskStatus
import tractor
from .. import data
from ..log import get_logger
from ..data._normalize import iterticks
from ._paper_engine import PaperBoi, simulate_fills
from . import _paper_engine as paper
from ._messages import (
Status, Order,
BrokerdCancel, BrokerdOrder, BrokerdOrderAck, BrokerdStatus,
BrokerdFill, BrokerdError, BrokerdPosition,
)
log = get_logger(__name__)
@ -89,11 +92,11 @@ class _DarkBook:
broker: str
# levels which have an executable action (eg. alert, order, signal)
orders: Dict[
orders: dict[
str, # symbol
Dict[
dict[
str, # uuid
Tuple[
tuple[
Callable[[float], bool], # predicate
str, # name
dict, # cmd / msg type
@ -102,22 +105,14 @@ class _DarkBook:
] = field(default_factory=dict)
# tracks most recent values per symbol each from data feed
lasts: Dict[
Tuple[str, str],
lasts: dict[
tuple[str, str],
float
] = field(default_factory=dict)
# mapping of broker order ids to piker ems ids
_broker2ems_ids: Dict[str, str] = field(default_factory=bidict)
_books: Dict[str, _DarkBook] = {}
def get_dark_book(broker: str) -> _DarkBook:
global _books
return _books.setdefault(broker, _DarkBook(broker))
# mapping of piker ems order ids to current brokerd order flow message
_ems_entries: dict[str, str] = field(default_factory=dict)
_ems2brokerd_ids: dict[str, str] = field(default_factory=bidict)
# XXX: this is in place to prevent accidental positions that are too
@ -127,13 +122,17 @@ def get_dark_book(broker: str) -> _DarkBook:
_DEFAULT_SIZE: float = 1.0
async def execute_triggers(
async def clear_dark_triggers(
brokerd_orders_stream: tractor.MsgStream,
ems_client_order_stream: tractor.MsgStream,
quote_stream: tractor.ReceiveMsgStream, # noqa
broker: str,
symbol: str,
stream: 'tractor.ReceiveStream', # noqa
ctx: tractor.Context,
client: 'Client', # noqa
book: _DarkBook,
) -> None:
"""Core dark order trigger loop.
@ -143,7 +142,7 @@ async def execute_triggers(
"""
# this stream may eventually contain multiple symbols
# XXX: optimize this for speed!
async for quotes in stream:
async for quotes in quote_stream:
# TODO: numba all this!
@ -179,10 +178,15 @@ async def execute_triggers(
# majority of iterations will be non-matches
continue
action = cmd['action']
action: str = cmd['action']
symbol: str = cmd['symbol']
if action != 'alert':
# executable order submission
if action == 'alert':
# nothing to do but relay a status
# message back to the requesting ems client
resp = 'alert_triggered'
else: # executable order submission
# submit_price = price + price*percent_away
submit_price = price + abs_diff_away
@ -191,47 +195,56 @@ async def execute_triggers(
f'Dark order triggered for price {price}\n'
f'Submitting order @ price {submit_price}')
reqid = await client.submit_limit(
msg = BrokerdOrder(
action=cmd['action'],
oid=oid,
time_ns=time.time_ns(),
# this is a brand new order request for the
# underlying broker so we set out "broker request
# id" (brid) as nothing so that the broker
# client knows that we aren't trying to modify
# an existing order.
brid=None,
# this **creates** new order request for the
# underlying broker so we set a "broker
# request id" (``reqid`` kwarg) to ``None``
# so that the broker client knows that we
# aren't trying to modify an existing
# order-request and instead create a new one.
reqid=None,
symbol=sym,
action=cmd['action'],
price=submit_price,
size=cmd['size'],
)
await brokerd_orders_stream.send(msg.dict())
# register broker request id to ems id
book._broker2ems_ids[reqid] = oid
# mark this entry as having sent an order
# request. the entry will be replaced once the
# target broker replies back with
# a ``BrokerdOrderAck`` msg including the
# allocated unique ``BrokerdOrderAck.reqid`` key
# generated by the broker's own systems.
book._ems_entries[oid] = msg
else:
# alerts have no broker request id
reqid = ''
# our internal status value for client-side
# triggered "dark orders"
resp = 'dark_triggered'
resp = {
'resp': 'dark_executed',
'time_ns': time.time_ns(),
'trigger_price': price,
msg = Status(
oid=oid, # ems order id
resp=resp,
time_ns=time.time_ns(),
'cmd': cmd, # original request message
symbol=symbol,
trigger_price=price,
'broker_reqid': reqid,
'broker': broker,
'oid': oid, # piker order id
broker_details={'name': broker},
}
cmd=cmd, # original request message
).dict()
# remove exec-condition from set
log.info(f'removing pred for {oid}')
execs.pop(oid)
await ctx.send_yield(resp)
await ems_client_order_stream.send(msg)
else: # condition scan loop complete
log.debug(f'execs are {execs}')
@ -241,78 +254,6 @@ async def execute_triggers(
# print(f'execs scan took: {time.time() - start}')
async def exec_loop(
ctx: tractor.Context,
feed: 'Feed', # noqa
broker: str,
symbol: str,
_exec_mode: str,
task_status: TaskStatus[dict] = trio.TASK_STATUS_IGNORED,
) -> AsyncIterator[dict]:
"""Main scan loop for order execution conditions and submission
to brokers.
"""
# XXX: this should be initial price quote from target provider
first_quote = await feed.receive()
book = get_dark_book(broker)
book.lasts[(broker, symbol)] = first_quote[symbol]['last']
# TODO: wrap this in a more re-usable general api
client_factory = getattr(feed.mod, 'get_client_proxy', None)
if client_factory is not None and _exec_mode != 'paper':
# we have an order API for this broker
client = client_factory(feed._brokerd_portal)
else:
# force paper mode
log.warning(f'Entering paper trading mode for {broker}')
client = PaperBoi(
broker,
*trio.open_memory_channel(100),
_buys={},
_sells={},
_reqids={},
)
# for paper mode we need to mock this trades response feed
# so we pass a duck-typed feed-looking mem chan which is fed
# fill and submission events from the exec loop
feed._trade_stream = client.trade_stream
# init the trades stream
client._to_trade_stream.send_nowait({'local_trades': 'start'})
_exec_mode = 'paper'
# return control to parent task
task_status.started((first_quote, feed, client))
stream = feed.stream
async with trio.open_nursery() as n:
n.start_soon(
execute_triggers,
broker,
symbol,
stream,
ctx,
client,
book
)
if _exec_mode == 'paper':
# TODO: make this an actual broadcast channels as in:
# https://github.com/python-trio/trio/issues/987
n.start_soon(simulate_fills, stream, client)
# TODO: lots of cases still to handle
# XXX: right now this is very very ad-hoc to IB
# - short-sale but securities haven't been located, in this case we
@ -323,11 +264,13 @@ async def exec_loop(
# reqId 1550: Order held while securities are located.'),
# status='PreSubmitted', message='')],
async def process_broker_trades(
ctx: tractor.Context,
feed: 'Feed', # noqa
async def translate_and_relay_brokerd_events(
broker: str,
ems_client_order_stream: tractor.MsgStream,
brokerd_trades_stream: tractor.MsgStream,
book: _DarkBook,
task_status: TaskStatus[dict] = trio.TASK_STATUS_IGNORED,
) -> AsyncIterator[dict]:
"""Trades update loop - receive updates from broker, convert
to EMS responses, transmit to ordering client(s).
@ -347,59 +290,93 @@ async def process_broker_trades(
{'presubmitted', 'submitted', 'cancelled', 'inactive'}
"""
broker = feed.mod.name
async for brokerd_msg in brokerd_trades_stream:
# TODO: make this a context
# in the paper engine case this is just a mem receive channel
async with feed.receive_trades_data() as trades_stream:
name = brokerd_msg['name']
first = await trades_stream.__anext__()
# startup msg expected as first from broker backend
assert first['local_trades'] == 'start'
task_status.started()
async for event in trades_stream:
name, msg = event['local_trades']
log.info(f'Received broker trade event:\n{pformat(msg)}')
log.info(f'Received broker trade event:\n{pformat(brokerd_msg)}')
if name == 'position':
msg['resp'] = 'position'
# relay through
await ctx.send_yield(msg)
# relay through position msgs immediately
await ems_client_order_stream.send(
BrokerdPosition(**brokerd_msg).dict()
)
continue
# Get the broker (order) request id, this **must** be normalized
# into messaging provided by the broker backend
reqid = msg['reqid']
reqid = brokerd_msg['reqid']
# make response packet to EMS client(s)
oid = book._broker2ems_ids.get(reqid)
# all piker originated requests will have an ems generated oid field
oid = brokerd_msg.get(
'oid',
book._ems2brokerd_ids.inverse.get(reqid)
)
if oid is None:
# XXX: paper clearing special cases
# paper engine race case: ``Client.submit_limit()`` hasn't
# returned yet and provided an output reqid to register
# locally, so we need to retreive the oid that was already
# packed at submission since we already know it ahead of
# time
paper = msg.get('paper_info')
paper = brokerd_msg['broker_details'].get('paper_info')
if paper:
# paperboi keeps the ems id up front
oid = paper['oid']
else:
msg.get('external')
if not msg:
log.error(f"Unknown trade event {event}")
# may be an order msg specified as "external" to the
# piker ems flow (i.e. generated by some other
# external broker backend client (like tws for ib)
ext = brokerd_msg.get('external')
if ext:
log.error(f"External trade event {ext}")
continue
else:
# check for existing live flow entry
entry = book._ems_entries.get(oid)
# initial response to brokerd order request
if name == 'ack':
# register the brokerd request id (that was generated
# / created internally by the broker backend) with our
# local ems order id for reverse lookup later.
# a ``BrokerdOrderAck`` **must** be sent after an order
# request in order to establish this id mapping.
book._ems2brokerd_ids[oid] = reqid
# new order which has not yet be registered into the
# local ems book, insert it now and handle 2 cases:
# - the order has previously been requested to be
# cancelled by the ems controlling client before we
# received this ack, in which case we relay that cancel
# signal **asap** to the backend broker
if entry.action == 'cancel':
# assign newly providerd broker backend request id
entry.reqid = reqid
# tell broker to cancel immediately
await brokerd_trades_stream.send(entry.dict())
# - the order is now active and will be mirrored in
# our book -> registered as live flow
else:
# update the flow with the ack msg
book._ems_entries[oid] = BrokerdOrderAck(**brokerd_msg)
continue
resp = {
'resp': None, # placeholder
'oid': oid
}
# a live flow now exists
oid = entry.oid
resp = None
broker_details = {}
if name in (
'error',
@ -413,10 +390,10 @@ async def process_broker_trades(
# about. In most default situations, with composed orders
# (ex. brackets), most brokers seem to use a oca policy.
message = msg['message']
msg = BrokerdError(**brokerd_msg)
# XXX should we make one when it's blank?
log.error(pformat(message))
log.error(pformat(msg))
# TODO: getting this bs, prolly need to handle status messages
# 'Market data farm connection is OK:usfarm.nj'
@ -444,15 +421,15 @@ async def process_broker_trades(
# - Inactive (reject or cancelled but not by trader)
# everyone doin camel case
status = msg['status'].lower()
msg = BrokerdStatus(**brokerd_msg)
if status == 'filled':
if msg.status == 'filled':
# conditional execution is fully complete, no more
# fills for the noted order
if not msg['remaining']:
if not msg.remaining:
resp['resp'] = 'broker_executed'
resp = 'broker_executed'
log.info(f'Execution for {oid} is complete!')
@ -461,81 +438,134 @@ async def process_broker_trades(
log.info(f'{broker} filled {msg}')
else:
# one of (submitted, cancelled)
resp['resp'] = 'broker_' + status
# one of {submitted, cancelled}
resp = 'broker_' + msg.status
# pass the BrokerdStatus msg inside the broker details field
broker_details = msg.dict()
elif name in (
'fill',
):
msg = BrokerdFill(**brokerd_msg)
# proxy through the "fill" result(s)
resp['resp'] = 'broker_filled'
resp.update(msg)
resp = 'broker_filled'
broker_details = msg.dict()
log.info(f'\nFill for {oid} cleared with:\n{pformat(resp)}')
# respond to requesting client
await ctx.send_yield(resp)
else:
raise ValueError(f'Brokerd message {brokerd_msg} is invalid')
# Create and relay response status message
# to requesting EMS client
await ems_client_order_stream.send(
Status(
oid=oid,
resp=resp,
time_ns=time.time_ns(),
broker_reqid=reqid,
brokerd_msg=broker_details,
).dict()
)
async def process_order_cmds(
ctx: tractor.Context,
cmd_stream: 'tractor.ReceiveStream', # noqa
async def process_client_order_cmds(
client_order_stream: tractor.MsgStream, # noqa
brokerd_order_stream: tractor.MsgStream,
symbol: str,
feed: 'Feed', # noqa
client: 'Client', # noqa
dark_book: _DarkBook,
) -> None:
async for cmd in cmd_stream:
# cmd: dict
async for cmd in client_order_stream:
log.info(f'Received order cmd:\n{pformat(cmd)}')
action = cmd['action']
oid = cmd['oid']
brid = dark_book._broker2ems_ids.inverse.get(oid)
reqid = dark_book._ems2brokerd_ids.inverse.get(oid)
live_entry = dark_book._ems_entries.get(oid)
# TODO: can't wait for this stuff to land in 3.10
# https://www.python.org/dev/peps/pep-0636/#going-to-the-cloud-mappings
if action in ('cancel',):
# check for live-broker order
if brid:
if live_entry:
msg = BrokerdCancel(
oid=oid,
reqid=reqid or live_entry.reqid,
time_ns=time.time_ns(),
)
# send cancel to brokerd immediately!
log.info("Submitting cancel for live order")
await client.submit_cancel(reqid=brid)
# NOTE: cancel response will be relayed back in messages
# from corresponding broker
await brokerd_order_stream.send(msg.dict())
else:
# this might be a cancel for an order that hasn't been
# acked yet by a brokerd, so register a cancel for when
# the order ack does show up later
dark_book._ems_entries[oid] = msg
# check for EMS active exec
else:
try:
# remove from dark book clearing
dark_book.orders[symbol].pop(oid, None)
await ctx.send_yield({
'resp': 'dark_cancelled',
'oid': oid
})
# tell client side that we've cancelled the
# dark-trigger order
await client_order_stream.send(
Status(
resp='dark_cancelled',
oid=oid,
time_ns=time.time_ns(),
).dict()
)
except KeyError:
log.exception(f'No dark order for {symbol}?')
# TODO: 3.10 struct-pattern matching and unpacking here
elif action in ('alert', 'buy', 'sell',):
sym = cmd['symbol']
trigger_price = cmd['price']
size = cmd['size']
brokers = cmd['brokers']
exec_mode = cmd['exec_mode']
msg = Order(**cmd)
broker = brokers[0]
sym = msg.symbol
trigger_price = msg.price
size = msg.size
exec_mode = msg.exec_mode
broker = msg.brokers[0]
if exec_mode == 'live' and action in ('buy', 'sell',):
# register broker id for ems id
order_id = await client.submit_limit(
if live_entry is not None:
# sanity check on emsd id
assert live_entry.oid == oid
# if we already had a broker order id then
# this is likely an order update commmand.
log.info(
f"Modifying live {broker} order: {live_entry.reqid}")
msg = BrokerdOrder(
oid=oid, # no ib support for oids...
time_ns=time.time_ns(),
# if this is None, creates a new order
# otherwise will modify any existing one
brid=brid,
reqid=reqid,
symbol=sym,
action=action,
@ -543,25 +573,24 @@ async def process_order_cmds(
size=size,
)
if brid:
assert dark_book._broker2ems_ids[brid] == oid
# if we already had a broker order id then
# this is likely an order update commmand.
log.info(f"Modifying order: {brid}")
else:
dark_book._broker2ems_ids[order_id] = oid
# send request to backend
# XXX: the trades data broker response loop
# (``process_broker_trades()`` above) will
# handle sending the ems side acks back to
# the cmd sender from here
# (``translate_and_relay_brokerd_events()`` above) will
# handle relaying the ems side responses back to
# the client/cmd sender from this request
log.info(f'Sending live order to {broker}:\n{pformat(msg)}')
await brokerd_order_stream.send(msg.dict())
# an immediate response should be brokerd ack with order
# id but we register our request as part of the flow
dark_book._ems_entries[oid] = msg
elif exec_mode in ('dark', 'paper') or (
action in ('alert')
):
# submit order to local EMS
# submit order to local EMS book and scan loop,
# effectively a local clearing engine, which
# scans for conditions and triggers matching executions
# Auto-gen scanner predicate:
# we automatically figure out what the alert check
@ -595,8 +624,10 @@ async def process_order_cmds(
abs_diff_away = 0
# submit execution/order to EMS scan loop
# FYI: this may result in an override of an existing
# NOTE: this may result in an override of an existing
# dark book entry if the order id already exists
dark_book.orders.setdefault(
sym, {}
)[oid] = (
@ -606,25 +637,32 @@ async def process_order_cmds(
percent_away,
abs_diff_away
)
# TODO: if the predicate resolves immediately send the
# execution to the broker asap? Or no?
# ack-response that order is live in EMS
await ctx.send_yield({
'resp': 'dark_submitted',
'oid': oid
})
if action == 'alert':
resp = 'alert_submitted'
else:
resp = 'dark_submitted'
await client_order_stream.send(
Status(
resp=resp,
oid=oid,
time_ns=time.time_ns(),
).dict()
)
@tractor.stream
@tractor.context
async def _emsd_main(
ctx: tractor.Context,
client_actor_name: str,
broker: str,
symbol: str,
_mode: str = 'dark', # ('paper', 'dark', 'live')
_exec_mode: str = 'dark', # ('paper', 'dark', 'live')
loglevel: str = 'info',
) -> None:
"""EMS (sub)actor entrypoint providing the
'''EMS (sub)actor entrypoint providing the
execution management (micro)service which conducts broker
order control on behalf of clients.
@ -638,66 +676,183 @@ async def _emsd_main(
received in a stream from that client actor and then responses are
streamed back up to the original calling task in the same client.
The task tree is:
The primary ``emsd`` task tree is:
- ``_emsd_main()``:
accepts order cmds, registers execs with exec loop
sets up brokerd feed, order feed with ems client, trades dialogue with
brokderd trading api.
|
- ``clear_dark_triggers()``:
run (dark order) conditions on inputs and trigger brokerd "live"
order submissions.
|
- ``translate_and_relay_brokerd_events()``:
accept normalized trades responses from brokerd, process and
relay to ems client(s); this is a effectively a "trade event
reponse" proxy-broker.
|
- ``process_client_order_cmds()``:
accepts order cmds from requesting piker clients, registers
execs with exec loop
- ``exec_loop()``:
run (dark) conditions on inputs and trigger broker submissions
'''
global _router
dark_book = _router.get_dark_book(broker)
- ``process_broker_trades()``:
accept normalized trades responses, process and relay to ems client(s)
# TODO: would be nice if in tractor we can require either a ctx arg,
# or a named arg with ctx in it and a type annotation of
# tractor.Context instead of strictly requiring a ctx arg.
ems_ctx = ctx
"""
from ._client import send_order_cmds
dark_book = get_dark_book(broker)
cached_feed = _router.feeds.get((broker, symbol))
if cached_feed:
# TODO: use cached feeds per calling-actor
log.warning(f'Opening duplicate feed for {(broker, symbol)}')
# spawn one task per broker feed
async with trio.open_nursery() as n:
async with (
trio.open_nursery() as n,
# TODO: eventually support N-brokers
async with data.open_feed(
data.open_feed(
broker,
[symbol],
loglevel='info',
) as feed:
loglevel=loglevel,
) as feed,
):
if not cached_feed:
_router.feeds[(broker, symbol)] = feed
# get a portal back to the client
async with tractor.wait_for_actor(client_actor_name) as portal:
# XXX: this should be initial price quote from target provider
first_quote = await feed.receive()
# connect back to the calling actor (the one that is
# acting as an EMS client and will submit orders) to
# receive requests pushed over a tractor stream
# using (for now) an async generator.
async with portal.open_stream_from(
send_order_cmds,
symbol_key=symbol,
) as order_stream:
# open a stream with the brokerd backend for order
# flow dialogue
book = _router.get_dark_book(broker)
book.lasts[(broker, symbol)] = first_quote[symbol]['last']
trades_endpoint = getattr(feed.mod, 'trades_dialogue', None)
portal = feed._brokerd_portal
if trades_endpoint is None or _exec_mode == 'paper':
# for paper mode we need to mock this trades response feed
# so we load bidir stream to a new sub-actor running a
# paper-simulator clearing engine.
# load the paper trading engine
_exec_mode = 'paper'
log.warning(f'Entering paper trading mode for {broker}')
# load the paper trading engine as a subactor of this emsd
# actor to simulate the real IPC load it'll have when also
# pulling data from feeds
open_trades_endpoint = paper.open_paperboi(
broker=broker,
symbol=symbol,
loglevel=loglevel,
)
else:
# open live brokerd trades endpoint
open_trades_endpoint = portal.open_context(
trades_endpoint,
loglevel=loglevel,
)
async with (
open_trades_endpoint as (brokerd_ctx, positions),
brokerd_ctx.open_stream() as brokerd_trades_stream,
):
# signal to client that we're started
# TODO: we could eventually send back **all** brokerd
# positions here?
await ems_ctx.started(positions)
# establish 2-way stream with requesting order-client and
# begin handling inbound order requests and updates
async with ems_ctx.open_stream() as ems_client_order_stream:
# trigger scan and exec loop
n.start_soon(
clear_dark_triggers,
brokerd_trades_stream,
ems_client_order_stream,
feed.stream,
# start the condition scan loop
quote, feed, client = await n.start(
exec_loop,
ctx,
feed,
broker,
symbol,
_mode,
book
)
await n.start(
process_broker_trades,
ctx,
feed,
# begin processing order events from the target brokerd backend
# by receiving order submission response messages,
# normalizing them to EMS messages and relaying back to
# the piker order client.
n.start_soon(
translate_and_relay_brokerd_events,
broker,
ems_client_order_stream,
brokerd_trades_stream,
dark_book,
)
# start inbound order request processing
await process_order_cmds(
ctx,
order_stream,
# start inbound (from attached client) order request processing
await process_client_order_cmds(
ems_client_order_stream,
brokerd_trades_stream,
symbol,
feed,
client,
dark_book,
)
class _Router(BaseModel):
'''Order router which manages per-broker dark books, alerts,
and clearing related data feed management.
'''
nursery: trio.Nursery
feeds: dict[tuple[str, str], Any] = {}
books: dict[str, _DarkBook] = {}
class Config:
arbitrary_types_allowed = True
underscore_attrs_are_private = False
def get_dark_book(
self,
brokername: str,
) -> _DarkBook:
return self.books.setdefault(brokername, _DarkBook(brokername))
_router: _Router = None
@tractor.context
async def _setup_persistent_emsd(
ctx: tractor.Context,
) -> None:
global _router
# open a root "service nursery" for the ``emsd`` actor
async with trio.open_nursery() as service_nursery:
_router = _Router(nursery=service_nursery)
# TODO: send back the full set of persistent orders/execs persistent
await ctx.started()
# we pin this task to keep the feeds manager active until the
# parent actor decides to tear it down
await trio.sleep_forever()

238
piker/clearing/_messages.py 100644
View File

@ -0,0 +1,238 @@
# piker: trading gear for hackers
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
Clearing system messagingn types and protocols.
"""
from typing import Optional, Union
# TODO: try out just encoding/send direction for now?
# import msgspec
from pydantic import BaseModel
# Client -> emsd
class Cancel(BaseModel):
'''Cancel msg for removing a dark (ems triggered) or
broker-submitted (live) trigger/order.
'''
action: str = 'cancel'
oid: str # uuid4
symbol: str
class Order(BaseModel):
action: str # {'buy', 'sell', 'alert'}
# internal ``emdsd`` unique "order id"
oid: str # uuid4
symbol: str
price: float
size: float
brokers: list[str]
# Assigned once initial ack is received
# ack_time_ns: Optional[int] = None
# determines whether the create execution
# will be submitted to the ems or directly to
# the backend broker
exec_mode: str # {'dark', 'live', 'paper'}
# Client <- emsd
# update msgs from ems which relay state change info
# from the active clearing engine.
class Status(BaseModel):
name: str = 'status'
oid: str # uuid4
time_ns: int
# {
# 'dark_submitted',
# 'dark_cancelled',
# 'dark_triggered',
# 'broker_submitted',
# 'broker_cancelled',
# 'broker_executed',
# 'broker_filled',
# 'alert_submitted',
# 'alert_triggered',
# 'position',
# }
resp: str # "response", see above
# symbol: str
# trigger info
trigger_price: Optional[float] = None
# price: float
# broker: Optional[str] = None
# this maps normally to the ``BrokerdOrder.reqid`` below, an id
# normally allocated internally by the backend broker routing system
broker_reqid: Optional[Union[int, str]] = None
# for relaying backend msg data "through" the ems layer
brokerd_msg: dict = {}
# emsd -> brokerd
# requests *sent* from ems to respective backend broker daemon
class BrokerdCancel(BaseModel):
action: str = 'cancel'
oid: str # piker emsd order id
time_ns: int
# "broker request id": broker specific/internal order id if this is
# None, creates a new order otherwise if the id is valid the backend
# api must modify the existing matching order. If the broker allows
# for setting a unique order id then this value will be relayed back
# on the emsd order request stream as the ``BrokerdOrderAck.reqid``
# field
reqid: Optional[Union[int, str]] = None
class BrokerdOrder(BaseModel):
action: str # {buy, sell}
oid: str
time_ns: int
# "broker request id": broker specific/internal order id if this is
# None, creates a new order otherwise if the id is valid the backend
# api must modify the existing matching order. If the broker allows
# for setting a unique order id then this value will be relayed back
# on the emsd order request stream as the ``BrokerdOrderAck.reqid``
# field
reqid: Optional[Union[int, str]] = None
symbol: str # symbol.<providername> ?
price: float
size: float
# emsd <- brokerd
# requests *received* to ems from broker backend
class BrokerdOrderAck(BaseModel):
'''Immediate reponse to a brokerd order request providing
the broker specifci unique order id.
'''
name: str = 'ack'
# defined and provided by backend
reqid: Union[int, str]
# emsd id originally sent in matching request msg
oid: str
class BrokerdStatus(BaseModel):
name: str = 'status'
reqid: Union[int, str]
time_ns: int
# {
# 'submitted',
# 'cancelled',
# 'executed',
# }
status: str
filled: float = 0.0
reason: str = ''
remaining: float = 0.0
# XXX: better design/name here?
# flag that can be set to indicate a message for an order
# event that wasn't originated by piker's emsd (eg. some external
# trading system which does it's own order control but that you
# might want to "track" using piker UIs/systems).
external: bool = False
# XXX: not required schema as of yet
broker_details: dict = {
'name': '',
}
class BrokerdFill(BaseModel):
'''A single message indicating a "fill-details" event from the broker
if avaiable.
'''
name: str = 'fill'
reqid: Union[int, str]
time_ns: int
# order exeuction related
action: str
size: float
price: float
broker_details: dict = {} # meta-data (eg. commisions etc.)
# brokerd timestamp required for order mode arrow placement on x-axis
# TODO: maybe int if we force ns?
# we need to normalize this somehow since backends will use their
# own format and likely across many disparate epoch clocks...
broker_time: float
class BrokerdError(BaseModel):
'''Optional error type that can be relayed to emsd for error handling.
This is still a TODO thing since we're not sure how to employ it yet.
'''
name: str = 'error'
reqid: Union[int, str]
symbol: str
reason: str
broker_details: dict = {}
class BrokerdPosition(BaseModel):
'''Position update event from brokerd.
'''
name: str = 'position'
broker: str
account: str
symbol: str
currency: str
size: float
avg_price: float

288
piker/clearing/_paper_engine.py
View File

@ -18,17 +18,28 @@
Fake trading for forward testing.
"""
from contextlib import asynccontextmanager
from datetime import datetime
from operator import itemgetter
import time
from typing import Tuple, Optional
from typing import Tuple, Optional, Callable
import uuid
from bidict import bidict
import trio
import tractor
from dataclasses import dataclass
from .. import data
from ..data._normalize import iterticks
from ..log import get_logger
from ._messages import (
BrokerdCancel, BrokerdOrder, BrokerdOrderAck, BrokerdStatus,
BrokerdFill,
)
log = get_logger(__name__)
@dataclass
@ -41,8 +52,8 @@ class PaperBoi:
"""
broker: str
_to_trade_stream: trio.abc.SendChannel
trade_stream: trio.abc.ReceiveChannel
ems_trades_stream: tractor.MsgStream
# map of paper "live" orders which be used
# to simulate fills based on paper engine settings
@ -61,20 +72,20 @@ class PaperBoi:
price: float,
action: str,
size: float,
brid: Optional[str],
reqid: Optional[str],
) -> int:
"""Place an order and return integer request id provided by client.
"""
if brid is None:
is_modify: bool = False
if reqid is None:
reqid = str(uuid.uuid4())
else:
# order is already existing, this is a modify
(oid, symbol, action, old_price) = self._reqids[brid]
(oid, symbol, action, old_price) = self._reqids[reqid]
assert old_price != price
reqid = brid
is_modify = True
# register order internally
self._reqids[reqid] = (oid, symbol, action, price)
@ -90,22 +101,16 @@ class PaperBoi:
# in the broker trades event processing loop
await trio.sleep(0.05)
await self._to_trade_stream.send({
'local_trades': ('status', {
'time_ns': time.time_ns(),
'reqid': reqid,
'status': 'submitted',
'broker': self.broker,
# 'cmd': cmd, # original request message
'paper_info': {
'oid': oid,
},
}),
})
msg = BrokerdStatus(
status='submitted',
reqid=reqid,
broker=self.broker,
time_ns=time.time_ns(),
filled=0.0,
reason='paper_trigger',
remaining=size,
)
await self.ems_trades_stream.send(msg.dict())
# if we're already a clearing price simulate an immediate fill
if (
@ -129,7 +134,7 @@ class PaperBoi:
# and trigger by the simulated clearing task normally
# running ``simulate_fills()``.
if brid is not None:
if is_modify:
# remove any existing order for the old price
orders[symbol].pop((oid, old_price))
@ -144,7 +149,6 @@ class PaperBoi:
) -> None:
# TODO: fake market simulation effects
# await self._to_trade_stream.send(
oid, symbol, action, price = self._reqids[reqid]
if action == 'buy':
@ -155,21 +159,14 @@ class PaperBoi:
# TODO: net latency model
await trio.sleep(0.05)
await self._to_trade_stream.send({
'local_trades': ('status', {
'time_ns': time.time_ns(),
'oid': oid,
'reqid': reqid,
'status': 'cancelled',
'broker': self.broker,
# 'cmd': cmd, # original request message
'paper': True,
}),
})
msg = BrokerdStatus(
status='cancelled',
oid=oid,
reqid=reqid,
broker=self.broker,
time_ns=time.time_ns(),
)
await self.ems_trades_stream.send(msg.dict())
async def fake_fill(
self,
@ -191,56 +188,49 @@ class PaperBoi:
# TODO: net latency model
await trio.sleep(0.05)
# the trades stream expects events in the form
# {'local_trades': (event_name, msg)}
await self._to_trade_stream.send({
msg = BrokerdFill(
'local_trades': ('fill', {
reqid=reqid,
time_ns=time.time_ns(),
'status': 'filled',
'broker': self.broker,
# converted to float by us in ib backend
'broker_time': datetime.now().timestamp(),
'action': action,
'size': size,
'price': price,
'remaining': 0 if order_complete else remaining,
# normally filled by real `brokerd` daemon
'time': time.time_ns(),
'time_ns': time.time_ns(), # cuz why not
# fake ids
'reqid': reqid,
action=action,
size=size,
price=price,
broker_time=datetime.now().timestamp(),
broker_details={
'paper_info': {
'oid': oid,
},
# mocking ib
'name': self.broker + '_paper',
},
)
await self.ems_trades_stream.send(msg.dict())
# XXX: fields we might not need to emulate?
# execution id from broker
# 'execid': execu.execId,
# 'cmd': cmd, # original request message?
}),
})
if order_complete:
await self._to_trade_stream.send({
'local_trades': ('status', {
'reqid': reqid,
'status': 'filled',
'broker': self.broker,
'filled': size,
'remaining': 0 if order_complete else remaining,
msg = BrokerdStatus(
# converted to float by us in ib backend
'broker_time': datetime.now().timestamp(),
reqid=reqid,
time_ns=time.time_ns(),
status='filled',
filled=size,
remaining=0 if order_complete else remaining,
action=action,
size=size,
price=price,
broker_details={
'paper_info': {
'oid': oid,
},
}),
})
'name': self.broker,
},
)
await self.ems_trades_stream.send(msg.dict())
async def simulate_fills(
@ -327,3 +317,145 @@ async def simulate_fills(
else:
# prices are iterated in sorted order so we're done
break
async def handle_order_requests(
client: PaperBoi,
ems_order_stream: tractor.MsgStream,
) -> None:
# order_request: dict
async for request_msg in ems_order_stream:
action = request_msg['action']
if action in {'buy', 'sell'}:
# validate
order = BrokerdOrder(**request_msg)
# call our client api to submit the order
reqid = await client.submit_limit(
oid=order.oid,
symbol=order.symbol,
price=order.price,
action=order.action,
size=order.size,
# XXX: by default 0 tells ``ib_insync`` methods that
# there is no existing order so ask the client to create
# a new one (which it seems to do by allocating an int
# counter - collision prone..)
reqid=order.reqid,
)
# deliver ack that order has been submitted to broker routing
await ems_order_stream.send(
BrokerdOrderAck(
# ems order request id
oid=order.oid,
# broker specific request id
reqid=reqid,
).dict()
)
elif action == 'cancel':
msg = BrokerdCancel(**request_msg)
await client.submit_cancel(
reqid=msg.reqid
)
else:
log.error(f'Unknown order command: {request_msg}')
@tractor.context
async def trades_dialogue(
ctx: tractor.Context,
broker: str,
symbol: str,
loglevel: str = None,
) -> None:
async with (
data.open_feed(
broker,
[symbol],
loglevel=loglevel,
) as feed,
):
# TODO: load paper positions per broker from .toml config file
# and pass as symbol to position data mapping: ``dict[str, dict]``
# await ctx.started(all_positions)
await ctx.started({})
async with (
ctx.open_stream() as ems_stream,
trio.open_nursery() as n,
):
client = PaperBoi(
broker,
ems_stream,
_buys={},
_sells={},
_reqids={},
)
n.start_soon(handle_order_requests, client, ems_stream)
# paper engine simulator clearing task
await simulate_fills(feed.stream, client)
@asynccontextmanager
async def open_paperboi(
broker: str,
symbol: str,
loglevel: str,
) -> Callable:
'''Spawn a paper engine actor and yield through access to
its context.
'''
service_name = f'paperboi.{broker}'
async with (
tractor.find_actor(service_name) as portal,
tractor.open_nursery() as tn,
):
# only spawn if no paperboi already is up
# (we likely don't need more then one proc for basic
# simulated order clearing)
if portal is None:
portal = await tn.start_actor(
service_name,
enable_modules=[__name__]
)
async with portal.open_context(
trades_dialogue,
broker=broker,
symbol=symbol,
loglevel=loglevel,
) as (ctx, first):
try:
yield ctx, first
finally:
# be sure to tear down the paper service on exit
with trio.CancelScope(shield=True):
await portal.cancel_actor()

5
piker/cli/__init__.py
View File

@ -9,6 +9,7 @@ import tractor
from ..log import get_console_log, get_logger, colorize_json
from ..brokers import get_brokermod, config
from .._daemon import _tractor_kwargs
log = get_logger('cli')
@ -101,8 +102,9 @@ def cli(ctx, brokers, loglevel, tl, configdir):
def services(config, tl, names):
async def list_services():
async with tractor.get_arbiter(
*tractor.current_actor()._arb_addr
*_tractor_kwargs['arbiter_addr']
) as portal:
registry = await portal.run('self', 'get_registry')
json_d = {}
@ -118,6 +120,7 @@ def services(config, tl, names):
list_services,
name='service_query',
loglevel=config['loglevel'] if tl else None,
arbiter_addr=_tractor_kwargs['arbiter_addr'],
)

6
piker/data/_web_bs.py
View File

@ -64,13 +64,13 @@ class NoBsWs:
async def _connect(
self,
tries: int = 10000,
tries: int = 1000,
) -> None:
while True:
try:
await self._stack.aclose()
except (DisconnectionTimeout, RuntimeError):
await trio.sleep(1)
await trio.sleep(0.5)
else:
break
@ -95,7 +95,7 @@ class NoBsWs:
f'{self} connection bail with '
f'{type(err)}...retry attempt {i}'
)
await trio.sleep(1)
await trio.sleep(0.5)
continue
else:
log.exception('ws connection fail...')

4
piker/data/feed.py
View File

@ -217,8 +217,8 @@ async def allocate_persistent_feed(
times = shm.array['time']
delay_s = times[-1] - times[times != times[-1]][-1]
# pass OHLC sample rate in seconds
init_msg[symbol]['sample_rate'] = delay_s
# pass OHLC sample rate in seconds (be sure to use python int type)
init_msg[symbol]['sample_rate'] = int(delay_s)
# yield back control to starting nursery
task_status.started((init_msg, first_quote))

51
piker/ui/order_mode.py
View File

@ -127,10 +127,6 @@ class OrderMode:
"""
line = self.lines.commit_line(uuid)
req_msg = self.book._sent_orders.get(uuid)
if req_msg:
req_msg['ack_time_ns'] = time.time_ns()
return line
def on_fill(
@ -196,8 +192,10 @@ class OrderMode:
def submit_exec(
self,
size: Optional[float] = None,
) -> LevelLine:
"""Send execution order to EMS.
"""Send execution order to EMS return a level line to
represent the order on a chart.
"""
# register the "staged" line under the cursor
@ -226,6 +224,9 @@ class OrderMode:
exec_mode=self._exec_mode,
)
# TODO: update the line once an ack event comes back
# from the EMS!
# make line graphic if order push was
# sucessful
line = self.lines.create_order_line(
@ -266,14 +267,6 @@ class OrderMode:
price=line.value(),
)
# def on_key_press(
# self,
# key:
# mods:
# text: str,
# ) -> None:
# pass
@asynccontextmanager
async def open_order_mode(
@ -320,10 +313,14 @@ async def start_order_mode(
# spawn EMS actor-service
async with (
open_ems(brokername, symbol) as (book, trades_stream),
open_ems(brokername, symbol) as (book, trades_stream, positions),
open_order_mode(symbol, chart, book) as order_mode
):
# update any exising positions
for sym, msg in positions.items():
order_mode.on_position_update(msg)
def get_index(time: float):
# XXX: not sure why the time is so off here
@ -346,16 +343,15 @@ async def start_order_mode(
fmsg = pformat(msg)
log.info(f'Received order msg:\n{fmsg}')
resp = msg['resp']
if resp in (
name = msg['name']
if name in (
'position',
):
# show line label once order is live
order_mode.on_position_update(msg)
continue
# delete the line from view
resp = msg['resp']
oid = msg['oid']
# response to 'action' request (buy/sell)
@ -378,15 +374,15 @@ async def start_order_mode(
order_mode.on_cancel(oid)
elif resp in (
'dark_executed'
'dark_triggered'
):
log.info(f'Dark order triggered for {fmsg}')
# for alerts add a triangle and remove the
# level line
if msg['cmd']['action'] == 'alert':
elif resp in (
'alert_triggered'
):
# should only be one "fill" for an alert
# add a triangle and remove the level line
order_mode.on_fill(
oid,
price=msg['trigger_price'],
@ -403,12 +399,15 @@ async def start_order_mode(
# each clearing tick is responded individually
elif resp in ('broker_filled',):
action = msg['action']
action = book._sent_orders[oid].action
details = msg['brokerd_msg']
# TODO: some kinda progress system
order_mode.on_fill(
oid,
price=msg['price'],
arrow_index=get_index(msg['broker_time']),
price=details['price'],
pointing='up' if action == 'buy' else 'down',
# TODO: put the actual exchange timestamp
arrow_index=get_index(details['broker_time']),
)