Variety of IB backend improvements
- Move to new shared mem system only writing on the first (by process) entry to `stream_quotes()`. - Deliver bars before first quote arrives so that chart can populate and then wait for initial arrival. - Allow caching clients per actor. - Load bars using the same (cached) client that starts the quote stream thus speeding up initialization.bar_select
parent
b1093dc71d
commit
d93ce84a99
|
@ -26,8 +26,10 @@ import tractor
|
||||||
|
|
||||||
from ..log import get_logger, get_console_log
|
from ..log import get_logger, get_console_log
|
||||||
from ..data import (
|
from ..data import (
|
||||||
maybe_spawn_brokerd, iterticks, attach_shared_array,
|
maybe_spawn_brokerd,
|
||||||
incr_buffer,
|
iterticks,
|
||||||
|
attach_shm_array,
|
||||||
|
get_shm_token
|
||||||
)
|
)
|
||||||
from ..ui._source import from_df
|
from ..ui._source import from_df
|
||||||
|
|
||||||
|
@ -145,7 +147,7 @@ class Client:
|
||||||
# durationStr='1 D',
|
# durationStr='1 D',
|
||||||
|
|
||||||
# time length calcs
|
# time length calcs
|
||||||
durationStr='{count} S'.format(count=5000 * 5),
|
durationStr='{count} S'.format(count=1000 * 5),
|
||||||
barSizeSetting='5 secs',
|
barSizeSetting='5 secs',
|
||||||
|
|
||||||
# always use extended hours
|
# always use extended hours
|
||||||
|
@ -311,6 +313,8 @@ class Client:
|
||||||
_tws_port: int = 7497
|
_tws_port: int = 7497
|
||||||
_gw_port: int = 4002
|
_gw_port: int = 4002
|
||||||
_try_ports = [_tws_port, _gw_port]
|
_try_ports = [_tws_port, _gw_port]
|
||||||
|
_client_ids = itertools.count()
|
||||||
|
_client_cache = {}
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
|
@ -321,17 +325,18 @@ async def _aio_get_client(
|
||||||
) -> Client:
|
) -> Client:
|
||||||
"""Return an ``ib_insync.IB`` instance wrapped in our client API.
|
"""Return an ``ib_insync.IB`` instance wrapped in our client API.
|
||||||
"""
|
"""
|
||||||
|
# first check cache for existing client
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield _client_cache[(host, port)]
|
||||||
|
except KeyError:
|
||||||
|
# TODO: in case the arbiter has no record
|
||||||
|
# of existing brokerd we need to broadcast for one.
|
||||||
|
|
||||||
if client_id is None:
|
if client_id is None:
|
||||||
# if this is a persistent brokerd, try to allocate a new id for
|
# if this is a persistent brokerd, try to allocate a new id for
|
||||||
# each client
|
# each client
|
||||||
try:
|
client_id = next(_client_ids)
|
||||||
ss = tractor.current_actor().statespace
|
|
||||||
client_id = next(ss.setdefault('client_ids', itertools.count()))
|
|
||||||
# TODO: in case the arbiter has no record
|
|
||||||
# of existing brokerd we need to broadcase for one.
|
|
||||||
except RuntimeError:
|
|
||||||
# tractor likely isn't running
|
|
||||||
client_id = 1
|
|
||||||
|
|
||||||
ib = NonShittyIB()
|
ib = NonShittyIB()
|
||||||
ports = _try_ports if port is None else [port]
|
ports = _try_ports if port is None else [port]
|
||||||
|
@ -347,7 +352,9 @@ async def _aio_get_client(
|
||||||
raise ConnectionRefusedError(_err)
|
raise ConnectionRefusedError(_err)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
yield Client(ib)
|
client = Client(ib)
|
||||||
|
_client_cache[(host, port)] = client
|
||||||
|
yield client
|
||||||
except BaseException:
|
except BaseException:
|
||||||
ib.disconnect()
|
ib.disconnect()
|
||||||
raise
|
raise
|
||||||
|
@ -489,7 +496,7 @@ def normalize(
|
||||||
# @tractor.msg.pub
|
# @tractor.msg.pub
|
||||||
async def stream_quotes(
|
async def stream_quotes(
|
||||||
symbols: List[str],
|
symbols: List[str],
|
||||||
shared_array_token: Tuple[str, str],
|
shm_token: Tuple[str, str, List[tuple]],
|
||||||
loglevel: str = None,
|
loglevel: str = None,
|
||||||
# compat for @tractor.msg.pub
|
# compat for @tractor.msg.pub
|
||||||
topics: Any = None,
|
topics: Any = None,
|
||||||
|
@ -506,16 +513,35 @@ async def stream_quotes(
|
||||||
# TODO: support multiple subscriptions
|
# TODO: support multiple subscriptions
|
||||||
sym = symbols[0]
|
sym = symbols[0]
|
||||||
|
|
||||||
stream = await tractor.to_asyncio.run_task(
|
stream = await _trio_run_client_method(
|
||||||
_trio_run_client_method,
|
|
||||||
method='stream_ticker',
|
method='stream_ticker',
|
||||||
symbol=sym,
|
symbol=sym,
|
||||||
)
|
)
|
||||||
|
|
||||||
async with get_client() as client:
|
|
||||||
bars = await client.bars(symbol=sym)
|
|
||||||
|
|
||||||
async with aclosing(stream):
|
async with aclosing(stream):
|
||||||
|
|
||||||
|
# maybe load historical ohlcv in to shared mem
|
||||||
|
# check if shm has already been created by previous
|
||||||
|
# feed initialization
|
||||||
|
writer_exists = get_shm_token(shm_token['shm_name'])
|
||||||
|
|
||||||
|
if not writer_exists:
|
||||||
|
shm = attach_shm_array(
|
||||||
|
token=shm_token,
|
||||||
|
# we are writer
|
||||||
|
readonly=False,
|
||||||
|
)
|
||||||
|
bars = await _trio_run_client_method(
|
||||||
|
method='bars',
|
||||||
|
symbol=sym,
|
||||||
|
)
|
||||||
|
|
||||||
|
shm.push(bars)
|
||||||
|
shm_token = shm.token
|
||||||
|
|
||||||
|
# pass back token, and bool, signalling if we're the writer
|
||||||
|
yield shm_token, not writer_exists
|
||||||
|
|
||||||
# first quote can be ignored as a 2nd with newer data is sent?
|
# first quote can be ignored as a 2nd with newer data is sent?
|
||||||
first_ticker = await stream.__anext__()
|
first_ticker = await stream.__anext__()
|
||||||
quote = normalize(first_ticker)
|
quote = normalize(first_ticker)
|
||||||
|
@ -538,7 +564,7 @@ async def stream_quotes(
|
||||||
else:
|
else:
|
||||||
log.debug("Received first real volume tick")
|
log.debug("Received first real volume tick")
|
||||||
# ugh, clear ticks since we've consumed them
|
# ugh, clear ticks since we've consumed them
|
||||||
# (ahem, ib_insync is stateful trash)
|
# (ahem, ib_insync is truly stateful trash)
|
||||||
ticker.ticks = []
|
ticker.ticks = []
|
||||||
|
|
||||||
# XXX: this works because we don't use
|
# XXX: this works because we don't use
|
||||||
|
@ -555,27 +581,8 @@ async def stream_quotes(
|
||||||
topic = '.'.join((con['symbol'], con[suffix])).lower()
|
topic = '.'.join((con['symbol'], con[suffix])).lower()
|
||||||
first_quote = {topic: quote}
|
first_quote = {topic: quote}
|
||||||
ticker.ticks = []
|
ticker.ticks = []
|
||||||
|
# yield first quote asap
|
||||||
# load historical ohlcv in to shared mem
|
yield first_quote
|
||||||
ss = tractor.current_actor().statespace
|
|
||||||
existing_shm = ss.get(f'ib_shm.{sym}')
|
|
||||||
if not existing_shm:
|
|
||||||
readonly = False
|
|
||||||
else:
|
|
||||||
readonly = True
|
|
||||||
shm = existing_shm
|
|
||||||
|
|
||||||
with attach_shared_array(
|
|
||||||
token=shared_array_token,
|
|
||||||
readonly=readonly
|
|
||||||
) as shm:
|
|
||||||
if not existing_shm:
|
|
||||||
shm.push(bars)
|
|
||||||
ss[f'ib_shm.{sym}'] = shm
|
|
||||||
|
|
||||||
yield (first_quote, shm.token)
|
|
||||||
else:
|
|
||||||
yield (first_quote, None)
|
|
||||||
|
|
||||||
async for ticker in stream:
|
async for ticker in stream:
|
||||||
quote = normalize(
|
quote = normalize(
|
||||||
|
@ -589,8 +596,9 @@ async def stream_quotes(
|
||||||
# at the yield such that the array write isn't delayed
|
# at the yield such that the array write isn't delayed
|
||||||
# while another consumer is serviced..
|
# while another consumer is serviced..
|
||||||
|
|
||||||
# if we are the lone tick writer
|
# if we are the lone tick writer start writing
|
||||||
if not existing_shm:
|
# the buffer with appropriate trade data
|
||||||
|
if not writer_exists:
|
||||||
for tick in iterticks(quote, type='trade'):
|
for tick in iterticks(quote, type='trade'):
|
||||||
last = tick['price']
|
last = tick['price']
|
||||||
# print(f'broker last: {tick}')
|
# print(f'broker last: {tick}')
|
||||||
|
|
Loading…
Reference in New Issue