Compare commits
14 Commits
30483c3218
...
bdb613139d
| Author | SHA1 | Date |
|---|---|---|
|
|
bdb613139d | |
|
|
96d0711b15 | |
|
|
33af9a0b6c | |
|
|
e8575d6f69 | |
|
|
25d4f3b9fc | |
|
|
f07a2dd523 | |
|
|
a00e9c0e64 | |
|
|
cb694700c2 | |
|
|
11c931f65d | |
|
|
60390ae596 | |
|
|
9592735aaa | |
|
|
49841f5b91 | |
|
|
b2827ef3c3 | |
|
|
2fc4ccf011 |
|
|
@ -98,13 +98,14 @@ async def open_cached_client(
|
|||
If one has not been setup do it and cache it.
|
||||
|
||||
'''
|
||||
brokermod = get_brokermod(brokername)
|
||||
brokermod: ModuleType = get_brokermod(brokername)
|
||||
|
||||
# TODO: make abstract or `typing.Protocol`
|
||||
# client: Client
|
||||
async with maybe_open_context(
|
||||
acm_func=brokermod.get_client,
|
||||
kwargs=kwargs,
|
||||
|
||||
) as (cache_hit, client):
|
||||
|
||||
if cache_hit:
|
||||
log.runtime(f'Reusing existing {client}')
|
||||
|
||||
|
|
|
|||
|
|
@ -471,11 +471,15 @@ def search(
|
|||
|
||||
'''
|
||||
# global opts
|
||||
brokermods = list(config['brokermods'].values())
|
||||
brokermods: list[ModuleType] = list(config['brokermods'].values())
|
||||
|
||||
# TODO: this is coming from the `search --pdb` NOT from
|
||||
# the `piker --pdb` XD ..
|
||||
# -[ ] pull from the parent click ctx's values..dumdum
|
||||
# assert pdb
|
||||
|
||||
# define tractor entrypoint
|
||||
async def main(func):
|
||||
|
||||
async with maybe_open_pikerd(
|
||||
loglevel=config['loglevel'],
|
||||
debug_mode=pdb,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,9 @@ routines should be primitive data types where possible.
|
|||
"""
|
||||
import inspect
|
||||
from types import ModuleType
|
||||
from typing import List, Dict, Any, Optional
|
||||
from typing import (
|
||||
Any,
|
||||
)
|
||||
|
||||
import trio
|
||||
|
||||
|
|
@ -34,8 +36,10 @@ from ..accounting import MktPair
|
|||
|
||||
|
||||
async def api(brokername: str, methname: str, **kwargs) -> dict:
|
||||
"""Make (proxy through) a broker API call by name and return its result.
|
||||
"""
|
||||
'''
|
||||
Make (proxy through) a broker API call by name and return its result.
|
||||
|
||||
'''
|
||||
brokermod = get_brokermod(brokername)
|
||||
async with brokermod.get_client() as client:
|
||||
meth = getattr(client, methname, None)
|
||||
|
|
@ -62,10 +66,14 @@ async def api(brokername: str, methname: str, **kwargs) -> dict:
|
|||
|
||||
async def stocks_quote(
|
||||
brokermod: ModuleType,
|
||||
tickers: List[str]
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
"""Return quotes dict for ``tickers``.
|
||||
"""
|
||||
tickers: list[str]
|
||||
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
'''
|
||||
Return a `dict` of snapshot quotes for the provided input
|
||||
`tickers`: a `list` of fqmes.
|
||||
|
||||
'''
|
||||
async with brokermod.get_client() as client:
|
||||
return await client.quote(tickers)
|
||||
|
||||
|
|
@ -74,13 +82,15 @@ async def stocks_quote(
|
|||
async def option_chain(
|
||||
brokermod: ModuleType,
|
||||
symbol: str,
|
||||
date: Optional[str] = None,
|
||||
) -> Dict[str, Dict[str, Dict[str, Any]]]:
|
||||
"""Return option chain for ``symbol`` for ``date``.
|
||||
date: str|None = None,
|
||||
) -> dict[str, dict[str, dict[str, Any]]]:
|
||||
'''
|
||||
Return option chain for ``symbol`` for ``date``.
|
||||
|
||||
By default all expiries are returned. If ``date`` is provided
|
||||
then contract quotes for that single expiry are returned.
|
||||
"""
|
||||
|
||||
'''
|
||||
async with brokermod.get_client() as client:
|
||||
if date:
|
||||
id = int((await client.tickers2ids([symbol]))[symbol])
|
||||
|
|
@ -98,7 +108,7 @@ async def option_chain(
|
|||
# async def contracts(
|
||||
# brokermod: ModuleType,
|
||||
# symbol: str,
|
||||
# ) -> Dict[str, Dict[str, Dict[str, Any]]]:
|
||||
# ) -> dict[str, dict[str, dict[str, Any]]]:
|
||||
# """Return option contracts (all expiries) for ``symbol``.
|
||||
# """
|
||||
# async with brokermod.get_client() as client:
|
||||
|
|
@ -110,15 +120,24 @@ async def bars(
|
|||
brokermod: ModuleType,
|
||||
symbol: str,
|
||||
**kwargs,
|
||||
) -> Dict[str, Dict[str, Dict[str, Any]]]:
|
||||
"""Return option contracts (all expiries) for ``symbol``.
|
||||
"""
|
||||
) -> dict[str, dict[str, dict[str, Any]]]:
|
||||
'''
|
||||
Return option contracts (all expiries) for ``symbol``.
|
||||
|
||||
'''
|
||||
async with brokermod.get_client() as client:
|
||||
return await client.bars(symbol, **kwargs)
|
||||
|
||||
|
||||
async def search_w_brokerd(name: str, pattern: str) -> dict:
|
||||
async def search_w_brokerd(
|
||||
name: str,
|
||||
pattern: str,
|
||||
) -> dict:
|
||||
|
||||
# TODO: WHY NOT WORK!?!
|
||||
# when we `step` through the next block?
|
||||
# import tractor
|
||||
# await tractor.pause()
|
||||
async with open_cached_client(name) as client:
|
||||
|
||||
# TODO: support multiple asset type concurrent searches.
|
||||
|
|
@ -130,12 +149,12 @@ async def symbol_search(
|
|||
pattern: str,
|
||||
**kwargs,
|
||||
|
||||
) -> Dict[str, Dict[str, Dict[str, Any]]]:
|
||||
) -> dict[str, dict[str, dict[str, Any]]]:
|
||||
'''
|
||||
Return symbol info from broker.
|
||||
|
||||
'''
|
||||
results = []
|
||||
results: list[str] = []
|
||||
|
||||
async def search_backend(
|
||||
brokermod: ModuleType
|
||||
|
|
@ -143,6 +162,13 @@ async def symbol_search(
|
|||
|
||||
brokername: str = mod.name
|
||||
|
||||
# TODO: figure this the FUCK OUT
|
||||
# -> ok so obvi in the root actor any async task that's
|
||||
# spawned outside the main tractor-root-actor task needs to
|
||||
# call this..
|
||||
# await tractor.devx._debug.maybe_init_greenback()
|
||||
# tractor.pause_from_sync()
|
||||
|
||||
async with maybe_spawn_brokerd(
|
||||
mod.name,
|
||||
infect_asyncio=getattr(
|
||||
|
|
@ -162,7 +188,6 @@ async def symbol_search(
|
|||
))
|
||||
|
||||
async with trio.open_nursery() as n:
|
||||
|
||||
for mod in brokermods:
|
||||
n.start_soon(search_backend, mod.name)
|
||||
|
||||
|
|
@ -172,11 +197,13 @@ async def symbol_search(
|
|||
async def mkt_info(
|
||||
brokermod: ModuleType,
|
||||
fqme: str,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> MktPair:
|
||||
'''
|
||||
Return MktPair info from broker including src and dst assets.
|
||||
Return the `piker.accounting.MktPair` info struct from a given
|
||||
backend broker tradable src/dst asset pair.
|
||||
|
||||
'''
|
||||
async with open_cached_client(brokermod.name) as client:
|
||||
|
|
|
|||
|
|
@ -587,7 +587,7 @@ async def get_bars(
|
|||
data_cs.cancel()
|
||||
|
||||
# spawn new data reset task
|
||||
data_cs, reset_done = await nurse.start(
|
||||
data_cs, reset_done = await tn.start(
|
||||
partial(
|
||||
wait_on_data_reset,
|
||||
proxy,
|
||||
|
|
@ -607,11 +607,11 @@ async def get_bars(
|
|||
# such that simultaneous symbol queries don't try data resettingn
|
||||
# too fast..
|
||||
unset_resetter: bool = False
|
||||
async with trio.open_nursery() as nurse:
|
||||
async with trio.open_nursery() as tn:
|
||||
|
||||
# start history request that we allow
|
||||
# to run indefinitely until a result is acquired
|
||||
nurse.start_soon(query)
|
||||
tn.start_soon(query)
|
||||
|
||||
# start history reset loop which waits up to the timeout
|
||||
# for a result before triggering a data feed reset.
|
||||
|
|
@ -631,7 +631,7 @@ async def get_bars(
|
|||
unset_resetter: bool = True
|
||||
|
||||
# spawn new data reset task
|
||||
data_cs, reset_done = await nurse.start(
|
||||
data_cs, reset_done = await tn.start(
|
||||
partial(
|
||||
wait_on_data_reset,
|
||||
proxy,
|
||||
|
|
@ -705,7 +705,9 @@ async def _setup_quote_stream(
|
|||
# to_trio, from_aio = trio.open_memory_channel(2**8) # type: ignore
|
||||
def teardown():
|
||||
ticker.updateEvent.disconnect(push)
|
||||
log.error(f"Disconnected stream for `{symbol}`")
|
||||
log.error(
|
||||
f'Disconnected stream for `{symbol}`'
|
||||
)
|
||||
client.ib.cancelMktData(contract)
|
||||
|
||||
# decouple broadcast mem chan
|
||||
|
|
@ -761,7 +763,10 @@ async def open_aio_quote_stream(
|
|||
symbol: str,
|
||||
contract: Contract | None = None,
|
||||
|
||||
) -> trio.abc.ReceiveStream:
|
||||
) -> (
|
||||
trio.abc.Channel| # iface
|
||||
tractor.to_asyncio.LinkedTaskChannel # actually
|
||||
):
|
||||
|
||||
from tractor.trionics import broadcast_receiver
|
||||
global _quote_streams
|
||||
|
|
@ -778,6 +783,7 @@ async def open_aio_quote_stream(
|
|||
yield from_aio
|
||||
return
|
||||
|
||||
from_aio: tractor.to_asyncio.LinkedTaskChannel
|
||||
async with tractor.to_asyncio.open_channel_from(
|
||||
_setup_quote_stream,
|
||||
symbol=symbol,
|
||||
|
|
@ -983,17 +989,18 @@ async def stream_quotes(
|
|||
)
|
||||
cs: trio.CancelScope | None = None
|
||||
startup: bool = True
|
||||
iter_quotes: trio.abc.Channel
|
||||
while (
|
||||
startup
|
||||
or cs.cancel_called
|
||||
):
|
||||
with trio.CancelScope() as cs:
|
||||
async with (
|
||||
trio.open_nursery() as nurse,
|
||||
trio.open_nursery() as tn,
|
||||
open_aio_quote_stream(
|
||||
symbol=sym,
|
||||
contract=con,
|
||||
) as stream,
|
||||
) as iter_quotes,
|
||||
):
|
||||
# ugh, clear ticks since we've consumed them
|
||||
# (ahem, ib_insync is stateful trash)
|
||||
|
|
@ -1021,9 +1028,9 @@ async def stream_quotes(
|
|||
await rt_ev.wait()
|
||||
cs.cancel() # cancel called should now be set
|
||||
|
||||
nurse.start_soon(reset_on_feed)
|
||||
tn.start_soon(reset_on_feed)
|
||||
|
||||
async with aclosing(stream):
|
||||
async with aclosing(iter_quotes):
|
||||
# if syminfo.get('no_vlm', False):
|
||||
if not init_msg.shm_write_opts['has_vlm']:
|
||||
|
||||
|
|
@ -1038,19 +1045,21 @@ async def stream_quotes(
|
|||
# wait for real volume on feed (trading might be
|
||||
# closed)
|
||||
while True:
|
||||
ticker = await stream.receive()
|
||||
ticker = await iter_quotes.receive()
|
||||
|
||||
# for a real volume contract we rait for
|
||||
# the first "real" trade to take place
|
||||
if (
|
||||
# not calc_price
|
||||
# and not ticker.rtTime
|
||||
not ticker.rtTime
|
||||
False
|
||||
# not ticker.rtTime
|
||||
):
|
||||
# spin consuming tickers until we
|
||||
# get a real market datum
|
||||
log.debug(f"New unsent ticker: {ticker}")
|
||||
continue
|
||||
|
||||
else:
|
||||
log.debug("Received first volume tick")
|
||||
# ugh, clear ticks since we've
|
||||
|
|
@ -1066,13 +1075,18 @@ async def stream_quotes(
|
|||
log.debug(f"First ticker received {quote}")
|
||||
|
||||
# tell data-layer spawner-caller that live
|
||||
# quotes are now streaming.
|
||||
# quotes are now active desptie not having
|
||||
# necessarily received a first vlm/clearing
|
||||
# tick.
|
||||
ticker = await iter_quotes.receive()
|
||||
feed_is_live.set()
|
||||
fqme: str = quote['fqme']
|
||||
await send_chan.send({fqme: quote})
|
||||
|
||||
# last = time.time()
|
||||
async for ticker in stream:
|
||||
async for ticker in iter_quotes:
|
||||
quote = normalize(ticker)
|
||||
fqme = quote['fqme']
|
||||
fqme: str = quote['fqme']
|
||||
await send_chan.send({fqme: quote})
|
||||
|
||||
# ugh, clear ticks since we've consumed them
|
||||
|
|
|
|||
|
|
@ -544,7 +544,7 @@ async def open_trade_dialog(
|
|||
# to be reloaded.
|
||||
balances: dict[str, float] = await client.get_balances()
|
||||
|
||||
verify_balances(
|
||||
await verify_balances(
|
||||
acnt,
|
||||
src_fiat,
|
||||
balances,
|
||||
|
|
|
|||
|
|
@ -37,6 +37,12 @@ import tractor
|
|||
from async_generator import asynccontextmanager
|
||||
import numpy as np
|
||||
import wrapt
|
||||
|
||||
# TODO, port to `httpx`/`trio-websocket` whenver i get back to
|
||||
# writing a proper ws-api streamer for this backend (since the data
|
||||
# feeds are free now) as per GH feat-req:
|
||||
# https://github.com/pikers/piker/issues/509
|
||||
#
|
||||
import asks
|
||||
|
||||
from ..calc import humanize, percent_change
|
||||
|
|
|
|||
|
|
@ -168,7 +168,6 @@ class OrderClient(Struct):
|
|||
|
||||
|
||||
async def relay_orders_from_sync_code(
|
||||
|
||||
client: OrderClient,
|
||||
symbol_key: str,
|
||||
to_ems_stream: tractor.MsgStream,
|
||||
|
|
@ -242,6 +241,11 @@ async def open_ems(
|
|||
|
||||
async with maybe_open_emsd(
|
||||
broker,
|
||||
# XXX NOTE, LOL so this determines the daemon `emsd` loglevel
|
||||
# then FYI.. that's kinda wrong no?
|
||||
# -[ ] shouldn't it be set by `pikerd -l` or no?
|
||||
# -[ ] would make a lot more sense to have a subsys ctl for
|
||||
# levels.. like `-l emsd.info` or something?
|
||||
loglevel=loglevel,
|
||||
) as portal:
|
||||
|
||||
|
|
|
|||
|
|
@ -653,7 +653,11 @@ class Router(Struct):
|
|||
flume = feed.flumes[fqme]
|
||||
first_quote: dict = flume.first_quote
|
||||
book: DarkBook = self.get_dark_book(broker)
|
||||
book.lasts[fqme]: float = float(first_quote['last'])
|
||||
|
||||
if not (last := first_quote.get('last')):
|
||||
last: float = flume.rt_shm.array[-1]['close']
|
||||
|
||||
book.lasts[fqme]: float = float(last)
|
||||
|
||||
async with self.maybe_open_brokerd_dialog(
|
||||
brokermod=brokermod,
|
||||
|
|
@ -716,7 +720,7 @@ class Router(Struct):
|
|||
subs = self.subscribers[sub_key]
|
||||
|
||||
sent_some: bool = False
|
||||
for client_stream in subs:
|
||||
for client_stream in subs.copy():
|
||||
try:
|
||||
await client_stream.send(msg)
|
||||
sent_some = True
|
||||
|
|
@ -1010,10 +1014,14 @@ async def translate_and_relay_brokerd_events(
|
|||
status_msg.brokerd_msg = msg
|
||||
status_msg.src = msg.broker_details['name']
|
||||
|
||||
await router.client_broadcast(
|
||||
status_msg.req.symbol,
|
||||
status_msg,
|
||||
)
|
||||
if not status_msg.req:
|
||||
# likely some order change state?
|
||||
await tractor.pause()
|
||||
else:
|
||||
await router.client_broadcast(
|
||||
status_msg.req.symbol,
|
||||
status_msg,
|
||||
)
|
||||
|
||||
if status == 'closed':
|
||||
log.info(f'Execution for {oid} is complete!')
|
||||
|
|
|
|||
|
|
@ -297,6 +297,8 @@ class PaperBoi(Struct):
|
|||
|
||||
# transmit pp msg to ems
|
||||
pp: Position = self.acnt.pps[bs_mktid]
|
||||
# TODO, this will break if `require_only=True` was passed to
|
||||
# `.update_from_ledger()`
|
||||
|
||||
pp_msg = BrokerdPosition(
|
||||
broker=self.broker,
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ subsys: str = 'piker.clearing'
|
|||
|
||||
log = get_logger(subsys)
|
||||
|
||||
# TODO, oof doesn't this ignore the `loglevel` then???
|
||||
get_console_log = partial(
|
||||
get_console_log,
|
||||
name=subsys,
|
||||
|
|
|
|||
|
|
@ -95,6 +95,12 @@ class Sampler:
|
|||
# history loading.
|
||||
incr_task_cs: trio.CancelScope | None = None
|
||||
|
||||
bcast_errors: tuple[Exception] = (
|
||||
trio.BrokenResourceError,
|
||||
trio.ClosedResourceError,
|
||||
trio.EndOfChannel,
|
||||
)
|
||||
|
||||
# holds all the ``tractor.Context`` remote subscriptions for
|
||||
# a particular sample period increment event: all subscribers are
|
||||
# notified on a step.
|
||||
|
|
@ -258,14 +264,15 @@ class Sampler:
|
|||
subs: set
|
||||
last_ts, subs = pair
|
||||
|
||||
task = trio.lowlevel.current_task()
|
||||
log.debug(
|
||||
f'SUBS {self.subscribers}\n'
|
||||
f'PAIR {pair}\n'
|
||||
f'TASK: {task}: {id(task)}\n'
|
||||
f'broadcasting {period_s} -> {last_ts}\n'
|
||||
# f'consumers: {subs}'
|
||||
)
|
||||
# NOTE, for debugging pub-sub issues
|
||||
# task = trio.lowlevel.current_task()
|
||||
# log.debug(
|
||||
# f'AlL-SUBS@{period_s!r}: {self.subscribers}\n'
|
||||
# f'PAIR: {pair}\n'
|
||||
# f'TASK: {task}: {id(task)}\n'
|
||||
# f'broadcasting {period_s} -> {last_ts}\n'
|
||||
# f'consumers: {subs}'
|
||||
# )
|
||||
borked: set[MsgStream] = set()
|
||||
sent: set[MsgStream] = set()
|
||||
while True:
|
||||
|
|
@ -282,12 +289,11 @@ class Sampler:
|
|||
await stream.send(msg)
|
||||
sent.add(stream)
|
||||
|
||||
except (
|
||||
trio.BrokenResourceError,
|
||||
trio.ClosedResourceError
|
||||
):
|
||||
except self.bcast_errors as err:
|
||||
log.error(
|
||||
f'{stream._ctx.chan.uid} dropped connection'
|
||||
f'Connection dropped for IPC ctx\n'
|
||||
f'{stream._ctx}\n\n'
|
||||
f'Due to {type(err)}'
|
||||
)
|
||||
borked.add(stream)
|
||||
else:
|
||||
|
|
@ -394,7 +400,8 @@ async def register_with_sampler(
|
|||
finally:
|
||||
if (
|
||||
sub_for_broadcasts
|
||||
and subs
|
||||
and
|
||||
subs
|
||||
):
|
||||
try:
|
||||
subs.remove(stream)
|
||||
|
|
@ -561,8 +568,7 @@ async def open_sample_stream(
|
|||
|
||||
|
||||
async def sample_and_broadcast(
|
||||
|
||||
bus: _FeedsBus, # noqa
|
||||
bus: _FeedsBus,
|
||||
rt_shm: ShmArray,
|
||||
hist_shm: ShmArray,
|
||||
quote_stream: trio.abc.ReceiveChannel,
|
||||
|
|
@ -582,11 +588,33 @@ async def sample_and_broadcast(
|
|||
|
||||
overruns = Counter()
|
||||
|
||||
# NOTE, only used for debugging live-data-feed issues, though
|
||||
# this should be resolved more correctly in the future using the
|
||||
# new typed-msgspec feats of `tractor`!
|
||||
#
|
||||
# XXX, a multiline nested `dict` formatter (since rn quote-msgs
|
||||
# are just that).
|
||||
# pfmt: Callable[[str], str] = mk_repr()
|
||||
|
||||
# iterate stream delivered by broker
|
||||
async for quotes in quote_stream:
|
||||
# print(quotes)
|
||||
|
||||
# TODO: ``numba`` this!
|
||||
# XXX WARNING XXX only enable for debugging bc ow can cost
|
||||
# ALOT of perf with HF-feedz!!!
|
||||
#
|
||||
# log.info(
|
||||
# 'Rx live quotes:\n'
|
||||
# f'{pfmt(quotes)}'
|
||||
# )
|
||||
|
||||
# TODO,
|
||||
# -[ ] `numba` or `cython`-nize this loop possibly?
|
||||
# |_alternatively could we do it in rust somehow by upacking
|
||||
# arrow msgs instead of using `msgspec`?
|
||||
# -[ ] use `msgspec.Struct` support in new typed-msging from
|
||||
# `tractor` to ensure only allowed msgs are transmitted?
|
||||
#
|
||||
for broker_symbol, quote in quotes.items():
|
||||
# TODO: in theory you can send the IPC msg *before* writing
|
||||
# to the sharedmem array to decrease latency, however, that
|
||||
|
|
@ -659,6 +687,21 @@ async def sample_and_broadcast(
|
|||
sub_key: str = broker_symbol.lower()
|
||||
subs: set[Sub] = bus.get_subs(sub_key)
|
||||
|
||||
# TODO, figure out how to make this useful whilst
|
||||
# incoporating feed "pausing" ..
|
||||
#
|
||||
# if not subs:
|
||||
# all_bs_fqmes: list[str] = list(
|
||||
# bus._subscribers.keys()
|
||||
# )
|
||||
# log.warning(
|
||||
# f'No subscribers for {brokername!r} live-quote ??\n'
|
||||
# f'broker_symbol: {broker_symbol}\n\n'
|
||||
|
||||
# f'Maybe the backend-sys symbol does not match one of,\n'
|
||||
# f'{pfmt(all_bs_fqmes)}\n'
|
||||
# )
|
||||
|
||||
# NOTE: by default the broker backend doesn't append
|
||||
# it's own "name" into the fqme schema (but maybe it
|
||||
# should?) so we have to manually generate the correct
|
||||
|
|
@ -728,18 +771,14 @@ async def sample_and_broadcast(
|
|||
if lags > 10:
|
||||
await tractor.pause()
|
||||
|
||||
except (
|
||||
trio.BrokenResourceError,
|
||||
trio.ClosedResourceError,
|
||||
trio.EndOfChannel,
|
||||
):
|
||||
except Sampler.bcast_errors as ipc_err:
|
||||
ctx: Context = ipc._ctx
|
||||
chan: Channel = ctx.chan
|
||||
if ctx:
|
||||
log.warning(
|
||||
'Dropped `brokerd`-quotes-feed connection:\n'
|
||||
f'{broker_symbol}:'
|
||||
f'{ctx.cid}@{chan.uid}'
|
||||
f'Dropped `brokerd`-feed for {broker_symbol!r} due to,\n'
|
||||
f'x>) {ctx.cid}@{chan.uid}'
|
||||
f'|_{ipc_err!r}\n\n'
|
||||
)
|
||||
if sub.throttle_rate:
|
||||
assert ipc._closed
|
||||
|
|
@ -756,12 +795,11 @@ async def sample_and_broadcast(
|
|||
|
||||
|
||||
async def uniform_rate_send(
|
||||
|
||||
rate: float,
|
||||
quote_stream: trio.abc.ReceiveChannel,
|
||||
stream: MsgStream,
|
||||
|
||||
task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
|
||||
task_status: TaskStatus[None] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
|
|
@ -779,13 +817,16 @@ async def uniform_rate_send(
|
|||
https://gist.github.com/njsmith/7ea44ec07e901cb78ebe1dd8dd846cb9
|
||||
|
||||
'''
|
||||
# TODO: compute the approx overhead latency per cycle
|
||||
left_to_sleep = throttle_period = 1/rate - 0.000616
|
||||
# ?TODO? dynamically compute the **actual** approx overhead latency per cycle
|
||||
# instead of this magic # bidinezz?
|
||||
throttle_period: float = 1/rate - 0.000616
|
||||
left_to_sleep: float = throttle_period
|
||||
|
||||
# send cycle state
|
||||
first_quote: dict|None
|
||||
first_quote = last_quote = None
|
||||
last_send = time.time()
|
||||
diff = 0
|
||||
last_send: float = time.time()
|
||||
diff: float = 0
|
||||
|
||||
task_status.started()
|
||||
ticks_by_type: dict[
|
||||
|
|
@ -796,22 +837,28 @@ async def uniform_rate_send(
|
|||
clear_types = _tick_groups['clears']
|
||||
|
||||
while True:
|
||||
|
||||
# compute the remaining time to sleep for this throttled cycle
|
||||
left_to_sleep = throttle_period - diff
|
||||
left_to_sleep: float = throttle_period - diff
|
||||
|
||||
if left_to_sleep > 0:
|
||||
cs: trio.CancelScope
|
||||
with trio.move_on_after(left_to_sleep) as cs:
|
||||
sym: str
|
||||
last_quote: dict
|
||||
try:
|
||||
sym, last_quote = await quote_stream.receive()
|
||||
except trio.EndOfChannel:
|
||||
log.exception(f"feed for {stream} ended?")
|
||||
log.exception(
|
||||
f'Live stream for feed for ended?\n'
|
||||
f'<=c\n'
|
||||
f' |_[{stream!r}\n'
|
||||
)
|
||||
break
|
||||
|
||||
diff = time.time() - last_send
|
||||
diff: float = time.time() - last_send
|
||||
|
||||
if not first_quote:
|
||||
first_quote = last_quote
|
||||
first_quote: float = last_quote
|
||||
# first_quote['tbt'] = ticks_by_type
|
||||
|
||||
if (throttle_period - diff) > 0:
|
||||
|
|
@ -872,7 +919,9 @@ async def uniform_rate_send(
|
|||
# TODO: now if only we could sync this to the display
|
||||
# rate timing exactly lul
|
||||
try:
|
||||
await stream.send({sym: first_quote})
|
||||
await stream.send({
|
||||
sym: first_quote
|
||||
})
|
||||
except tractor.RemoteActorError as rme:
|
||||
if rme.type is not tractor._exceptions.StreamOverrun:
|
||||
raise
|
||||
|
|
@ -883,19 +932,28 @@ async def uniform_rate_send(
|
|||
f'{sym}:{ctx.cid}@{chan.uid}'
|
||||
)
|
||||
|
||||
# NOTE: any of these can be raised by `tractor`'s IPC
|
||||
# transport-layer and we want to be highly resilient
|
||||
# to consumers which crash or lose network connection.
|
||||
# I.e. we **DO NOT** want to crash and propagate up to
|
||||
# ``pikerd`` these kinds of errors!
|
||||
except (
|
||||
# NOTE: any of these can be raised by ``tractor``'s IPC
|
||||
# transport-layer and we want to be highly resilient
|
||||
# to consumers which crash or lose network connection.
|
||||
# I.e. we **DO NOT** want to crash and propagate up to
|
||||
# ``pikerd`` these kinds of errors!
|
||||
trio.ClosedResourceError,
|
||||
trio.BrokenResourceError,
|
||||
ConnectionResetError,
|
||||
):
|
||||
# if the feed consumer goes down then drop
|
||||
# out of this rate limiter
|
||||
log.warning(f'{stream} closed')
|
||||
) + Sampler.bcast_errors as ipc_err:
|
||||
match ipc_err:
|
||||
case trio.EndOfChannel():
|
||||
log.info(
|
||||
f'{stream} terminated by peer,\n'
|
||||
f'{ipc_err!r}'
|
||||
)
|
||||
case _:
|
||||
# if the feed consumer goes down then drop
|
||||
# out of this rate limiter
|
||||
log.warning(
|
||||
f'{stream} closed due to,\n'
|
||||
f'{ipc_err!r}'
|
||||
)
|
||||
|
||||
await stream.aclose()
|
||||
return
|
||||
|
||||
|
|
|
|||
30
piker/log.py
30
piker/log.py
|
|
@ -19,6 +19,10 @@ Log like a forester!
|
|||
"""
|
||||
import logging
|
||||
import json
|
||||
import reprlib
|
||||
from typing import (
|
||||
Callable,
|
||||
)
|
||||
|
||||
import tractor
|
||||
from pygments import (
|
||||
|
|
@ -84,3 +88,29 @@ def colorize_json(
|
|||
# likeable styles: algol_nu, tango, monokai
|
||||
formatters.TerminalTrueColorFormatter(style=style)
|
||||
)
|
||||
|
||||
|
||||
# TODO, eventually defer to the version in `modden` once
|
||||
# it becomes a dep!
|
||||
def mk_repr(
|
||||
**repr_kws,
|
||||
) -> Callable[[str], str]:
|
||||
'''
|
||||
Allocate and deliver a `repr.Repr` instance with provided input
|
||||
settings using the std-lib's `reprlib` mod,
|
||||
* https://docs.python.org/3/library/reprlib.html
|
||||
|
||||
------ Ex. ------
|
||||
An up to 6-layer-nested `dict` as multi-line:
|
||||
- https://stackoverflow.com/a/79102479
|
||||
- https://docs.python.org/3/library/reprlib.html#reprlib.Repr.maxlevel
|
||||
|
||||
'''
|
||||
def_kws: dict[str, int] = dict(
|
||||
indent=2,
|
||||
maxlevel=6, # recursion levels
|
||||
maxstring=66, # match editor line-len limit
|
||||
)
|
||||
def_kws |= repr_kws
|
||||
reprr = reprlib.Repr(**def_kws)
|
||||
return reprr.repr
|
||||
|
|
|
|||
|
|
@ -130,4 +130,5 @@ pyqtgraph = { git = "https://github.com/pikers/pyqtgraph.git" }
|
|||
asyncvnc = { git = "https://github.com/pikers/asyncvnc.git", branch = "main" }
|
||||
tomlkit = { git = "https://github.com/pikers/tomlkit.git", branch ="piker_pin" }
|
||||
msgspec = { git = "https://github.com/jcrist/msgspec.git" }
|
||||
tractor = { path = "../tractor", editable = true }
|
||||
tractor = { git = "https://pikers.dev/goodboy/tractor", branch = "piker_pin" }
|
||||
# tractor = { path = "../tractor", editable = true }
|
||||
|
|
|
|||
Loading…
Reference in New Issue