Compare commits
14 Commits
3ac112d402
...
23495a92d5
| Author | SHA1 | Date |
|---|---|---|
|
|
23495a92d5 | |
|
|
70d01a9bb4 | |
|
|
8b8747db27 | |
|
|
19becf8f58 | |
|
|
9c5dd7e4f1 | |
|
|
545efc3aef | |
|
|
a00e9c0e64 | |
|
|
cb694700c2 | |
|
|
11c931f65d | |
|
|
60390ae596 | |
|
|
9592735aaa | |
|
|
49841f5b91 | |
|
|
b2827ef3c3 | |
|
|
2fc4ccf011 |
|
|
@ -1,8 +1,40 @@
|
||||||
.accounting
|
piker.accounting
|
||||||
-----------
|
________________
|
||||||
A subsystem for transaction processing, storage and historical
|
A subsystem for transaction processing, storage and historical
|
||||||
measurement.
|
measurement.
|
||||||
|
|
||||||
|
synopsis
|
||||||
|
--------
|
||||||
|
The big question for any trader is this:
|
||||||
|
|
||||||
|
*what is the price that determines whether i take a loss or a gain on my
|
||||||
|
trade?*
|
||||||
|
|
||||||
|
In other words, at any given state of accounting your current assets,
|
||||||
|
what is the price between any 2 assets you've transacted that determines
|
||||||
|
at which price you can conduct **the next** transaction and know if you
|
||||||
|
are making or losing more (or less) of the *source* asset versus the
|
||||||
|
*destination* asset?
|
||||||
|
|
||||||
|
Let's do a very simple example:
|
||||||
|
|
||||||
|
> Joe wants to buy some tacos bc they're super hungo.
|
||||||
|
> Joe has a friend who also likes tacos but doesn't mind if they're fresh; he doesn't mind having day old tacos.
|
||||||
|
> Inflation is rampant and taco prices are trending up for no good reason besides everyone thinks prices are going up.
|
||||||
|
> Joe goes to the taco stand and buys 4 tacos at 25 mxn.
|
||||||
|
> This makes Joe's net cost `4 * 25 = 200` mxn.
|
||||||
|
> Joe eats 3 tacos and realizes that he can't finish the last, so he puts it in the fridge to save for the next day (since he owns a comal).
|
||||||
|
> The next day the price of tacos goes up to 30 mxn (for no good reason > besides the taco stand noticing Joe is a tourist and that > "inflation" is some thing that's used as an excuse for price changes).
|
||||||
|
> Joe's friend from before got lit up (like he does every morning) and msgs Joe to buy him 2 tacos for when he shows up in the late morning.
|
||||||
|
> Joe says "sure, but i also have a leftover if you want it, and I'm fasting today so you can have my sobras and i'll buy you a new one".
|
||||||
|
> The friend coughs a couple times, and says "yee no problem man, just make sure you get them"
|
||||||
|
>
|
||||||
|
|
||||||
|
|
||||||
|
Prior *suit* definitions:
|
||||||
|
|
||||||
|
- the canucks equiv of the IRS call this idea ["Adjusted cost base"](https://www.canada.ca/en/revenue-agency/services/tax/individuals/topics/about-your-tax-return/tax-return/completing-a-tax-return/personal-income/line-12700-capital-gains/definitions-capital-gains.html#Adjustedcostbase)
|
||||||
|
|
||||||
|
|
||||||
.pnl
|
.pnl
|
||||||
----
|
----
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,7 @@ import tomli_w # for fast ledger writing
|
||||||
|
|
||||||
from piker.types import Struct
|
from piker.types import Struct
|
||||||
from piker import config
|
from piker import config
|
||||||
from ..log import get_logger
|
from piker.log import get_logger
|
||||||
from .calc import (
|
from .calc import (
|
||||||
iter_by_dt,
|
iter_by_dt,
|
||||||
)
|
)
|
||||||
|
|
@ -239,7 +239,9 @@ class TransactionLedger(UserDict):
|
||||||
|
|
||||||
symcache: SymbologyCache = self._symcache
|
symcache: SymbologyCache = self._symcache
|
||||||
towrite: dict[str, Any] = {}
|
towrite: dict[str, Any] = {}
|
||||||
for tid, txdict in self.tx_sort(self.data.copy()):
|
for tid, txdict in self.tx_sort(
|
||||||
|
self.data.copy()
|
||||||
|
):
|
||||||
# write blank-str expiry for non-expiring assets
|
# write blank-str expiry for non-expiring assets
|
||||||
if (
|
if (
|
||||||
'expiry' in txdict
|
'expiry' in txdict
|
||||||
|
|
@ -377,7 +379,7 @@ def open_trade_ledger(
|
||||||
account,
|
account,
|
||||||
dirpath=_fp,
|
dirpath=_fp,
|
||||||
)
|
)
|
||||||
cpy = ledger_dict.copy()
|
cpy: dict = ledger_dict.copy()
|
||||||
|
|
||||||
# XXX NOTE: if not provided presume we are being called from
|
# XXX NOTE: if not provided presume we are being called from
|
||||||
# sync code and need to maybe run `trio` to generate..
|
# sync code and need to maybe run `trio` to generate..
|
||||||
|
|
@ -406,7 +408,13 @@ def open_trade_ledger(
|
||||||
account=account,
|
account=account,
|
||||||
mod=mod,
|
mod=mod,
|
||||||
symcache=symcache,
|
symcache=symcache,
|
||||||
tx_sort=getattr(mod, 'tx_sort', tx_sort),
|
|
||||||
|
# NOTE: allow backends to provide custom ledger sorting
|
||||||
|
tx_sort=getattr(
|
||||||
|
mod,
|
||||||
|
'tx_sort',
|
||||||
|
tx_sort,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
yield ledger
|
yield ledger
|
||||||
|
|
|
||||||
|
|
@ -305,8 +305,8 @@ class MktPair(Struct, frozen=True):
|
||||||
# config right?
|
# config right?
|
||||||
# src_type: AssetTypeName
|
# src_type: AssetTypeName
|
||||||
|
|
||||||
# for derivs, info describing contract, egs.
|
# for derivs, info describing contract, egs. strike price, call
|
||||||
# strike price, call or put, swap type, exercise model, etc.
|
# or put, swap type, exercise model, etc.
|
||||||
contract_info: list[str] | None = None
|
contract_info: list[str] | None = None
|
||||||
|
|
||||||
# TODO: rename to sectype since all of these can
|
# TODO: rename to sectype since all of these can
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,8 @@ from types import ModuleType
|
||||||
from typing import (
|
from typing import (
|
||||||
Any,
|
Any,
|
||||||
Iterator,
|
Iterator,
|
||||||
Generator
|
Generator,
|
||||||
|
TYPE_CHECKING,
|
||||||
)
|
)
|
||||||
|
|
||||||
import pendulum
|
import pendulum
|
||||||
|
|
@ -59,8 +60,10 @@ from ..clearing._messages import (
|
||||||
BrokerdPosition,
|
BrokerdPosition,
|
||||||
)
|
)
|
||||||
from piker.types import Struct
|
from piker.types import Struct
|
||||||
|
from piker.log import get_logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
from piker.data._symcache import SymbologyCache
|
from piker.data._symcache import SymbologyCache
|
||||||
from ..log import get_logger
|
|
||||||
|
|
||||||
log = get_logger(__name__)
|
log = get_logger(__name__)
|
||||||
|
|
||||||
|
|
@ -493,6 +496,17 @@ class Account(Struct):
|
||||||
|
|
||||||
_mktmap_table: dict[str, MktPair] | None = None,
|
_mktmap_table: dict[str, MktPair] | None = None,
|
||||||
|
|
||||||
|
only_require: list[str]|True = True,
|
||||||
|
# ^list of fqmes that are "required" to be processed from
|
||||||
|
# this ledger pass; we often don't care about others and
|
||||||
|
# definitely shouldn't always error in such cases.
|
||||||
|
# (eg. broker backend loaded that doesn't yet supsport the
|
||||||
|
# symcache but also, inside the paper engine we don't ad-hoc
|
||||||
|
# request `get_mkt_info()` for every symbol in the ledger,
|
||||||
|
# only the one for which we're simulating against).
|
||||||
|
# TODO, not sure if there's a better soln for this, ideally
|
||||||
|
# all backends get symcache support afap i guess..
|
||||||
|
|
||||||
) -> dict[str, Position]:
|
) -> dict[str, Position]:
|
||||||
'''
|
'''
|
||||||
Update the internal `.pps[str, Position]` table from input
|
Update the internal `.pps[str, Position]` table from input
|
||||||
|
|
@ -535,11 +549,32 @@ class Account(Struct):
|
||||||
if _mktmap_table is None:
|
if _mktmap_table is None:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
required: bool = (
|
||||||
|
only_require is True
|
||||||
|
or (
|
||||||
|
only_require is not True
|
||||||
|
and
|
||||||
|
fqme in only_require
|
||||||
|
)
|
||||||
|
)
|
||||||
# XXX: caller is allowed to provide a fallback
|
# XXX: caller is allowed to provide a fallback
|
||||||
# mktmap table for the case where a new position is
|
# mktmap table for the case where a new position is
|
||||||
# being added and the preloaded symcache didn't
|
# being added and the preloaded symcache didn't
|
||||||
# have this entry prior (eg. with frickin IB..)
|
# have this entry prior (eg. with frickin IB..)
|
||||||
mkt = _mktmap_table[fqme]
|
if (
|
||||||
|
not (mkt := _mktmap_table.get(fqme))
|
||||||
|
and
|
||||||
|
required
|
||||||
|
):
|
||||||
|
raise
|
||||||
|
|
||||||
|
elif not required:
|
||||||
|
continue
|
||||||
|
|
||||||
|
else:
|
||||||
|
# should be an entry retreived somewhere
|
||||||
|
assert mkt
|
||||||
|
|
||||||
|
|
||||||
if not (pos := pps.get(bs_mktid)):
|
if not (pos := pps.get(bs_mktid)):
|
||||||
|
|
||||||
|
|
@ -656,7 +691,7 @@ class Account(Struct):
|
||||||
def write_config(self) -> None:
|
def write_config(self) -> None:
|
||||||
'''
|
'''
|
||||||
Write the current account state to the user's account TOML file, normally
|
Write the current account state to the user's account TOML file, normally
|
||||||
something like ``pps.toml``.
|
something like `pps.toml`.
|
||||||
|
|
||||||
'''
|
'''
|
||||||
# TODO: show diff output?
|
# TODO: show diff output?
|
||||||
|
|
|
||||||
|
|
@ -251,10 +251,16 @@ def iter_by_dt(
|
||||||
for k in parsers:
|
for k in parsers:
|
||||||
if (
|
if (
|
||||||
isdict and k in tx
|
isdict and k in tx
|
||||||
or getattr(tx, k, None)
|
or
|
||||||
|
getattr(tx, k, None)
|
||||||
):
|
):
|
||||||
v = tx[k] if isdict else tx.dt
|
v = (
|
||||||
assert v is not None, f'No valid value for `{k}`!?'
|
tx[k] if isdict
|
||||||
|
else tx.dt
|
||||||
|
)
|
||||||
|
assert v is not None, (
|
||||||
|
f'No valid value for `{k}`!?'
|
||||||
|
)
|
||||||
|
|
||||||
# only call parser on the value if not None from
|
# only call parser on the value if not None from
|
||||||
# the `parsers` table above (when NOT using
|
# the `parsers` table above (when NOT using
|
||||||
|
|
@ -269,8 +275,21 @@ def iter_by_dt(
|
||||||
return v
|
return v
|
||||||
|
|
||||||
else:
|
else:
|
||||||
# XXX: should never get here..
|
# TODO: move to top?
|
||||||
breakpoint()
|
from piker.log import get_logger
|
||||||
|
log = get_logger(__name__)
|
||||||
|
|
||||||
|
# XXX: we should really never get here..
|
||||||
|
# only if a ledger record has no expected sort(able)
|
||||||
|
# field will we likely hit this.. like with ze IB.
|
||||||
|
# if no sortable field just deliver epoch?
|
||||||
|
log.warning(
|
||||||
|
'No (time) sortable field for TXN:\n'
|
||||||
|
f'{tx}\n'
|
||||||
|
)
|
||||||
|
return from_timestamp(0)
|
||||||
|
# breakpoint()
|
||||||
|
|
||||||
|
|
||||||
entry: tuple[str, dict] | Transaction
|
entry: tuple[str, dict] | Transaction
|
||||||
for entry in sorted(
|
for entry in sorted(
|
||||||
|
|
|
||||||
|
|
@ -300,7 +300,8 @@ def disect(
|
||||||
assert not df.is_empty()
|
assert not df.is_empty()
|
||||||
|
|
||||||
# muck around in pdbp REPL
|
# muck around in pdbp REPL
|
||||||
breakpoint()
|
# tractor.devx.mk_pdb().set_trace()
|
||||||
|
# breakpoint()
|
||||||
|
|
||||||
# TODO: we REALLY need a better console REPL for this
|
# TODO: we REALLY need a better console REPL for this
|
||||||
# kinda thing..
|
# kinda thing..
|
||||||
|
|
|
||||||
|
|
@ -168,7 +168,6 @@ class OrderClient(Struct):
|
||||||
|
|
||||||
|
|
||||||
async def relay_orders_from_sync_code(
|
async def relay_orders_from_sync_code(
|
||||||
|
|
||||||
client: OrderClient,
|
client: OrderClient,
|
||||||
symbol_key: str,
|
symbol_key: str,
|
||||||
to_ems_stream: tractor.MsgStream,
|
to_ems_stream: tractor.MsgStream,
|
||||||
|
|
@ -242,6 +241,11 @@ async def open_ems(
|
||||||
|
|
||||||
async with maybe_open_emsd(
|
async with maybe_open_emsd(
|
||||||
broker,
|
broker,
|
||||||
|
# XXX NOTE, LOL so this determines the daemon `emsd` loglevel
|
||||||
|
# then FYI.. that's kinda wrong no?
|
||||||
|
# -[ ] shouldn't it be set by `pikerd -l` or no?
|
||||||
|
# -[ ] would make a lot more sense to have a subsys ctl for
|
||||||
|
# levels.. like `-l emsd.info` or something?
|
||||||
loglevel=loglevel,
|
loglevel=loglevel,
|
||||||
) as portal:
|
) as portal:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -653,7 +653,11 @@ class Router(Struct):
|
||||||
flume = feed.flumes[fqme]
|
flume = feed.flumes[fqme]
|
||||||
first_quote: dict = flume.first_quote
|
first_quote: dict = flume.first_quote
|
||||||
book: DarkBook = self.get_dark_book(broker)
|
book: DarkBook = self.get_dark_book(broker)
|
||||||
book.lasts[fqme]: float = float(first_quote['last'])
|
|
||||||
|
if not (last := first_quote.get('last')):
|
||||||
|
last: float = flume.rt_shm.array[-1]['close']
|
||||||
|
|
||||||
|
book.lasts[fqme]: float = float(last)
|
||||||
|
|
||||||
async with self.maybe_open_brokerd_dialog(
|
async with self.maybe_open_brokerd_dialog(
|
||||||
brokermod=brokermod,
|
brokermod=brokermod,
|
||||||
|
|
@ -716,7 +720,7 @@ class Router(Struct):
|
||||||
subs = self.subscribers[sub_key]
|
subs = self.subscribers[sub_key]
|
||||||
|
|
||||||
sent_some: bool = False
|
sent_some: bool = False
|
||||||
for client_stream in subs:
|
for client_stream in subs.copy():
|
||||||
try:
|
try:
|
||||||
await client_stream.send(msg)
|
await client_stream.send(msg)
|
||||||
sent_some = True
|
sent_some = True
|
||||||
|
|
@ -1010,6 +1014,10 @@ async def translate_and_relay_brokerd_events(
|
||||||
status_msg.brokerd_msg = msg
|
status_msg.brokerd_msg = msg
|
||||||
status_msg.src = msg.broker_details['name']
|
status_msg.src = msg.broker_details['name']
|
||||||
|
|
||||||
|
if not status_msg.req:
|
||||||
|
# likely some order change state?
|
||||||
|
await tractor.pause()
|
||||||
|
else:
|
||||||
await router.client_broadcast(
|
await router.client_broadcast(
|
||||||
status_msg.req.symbol,
|
status_msg.req.symbol,
|
||||||
status_msg,
|
status_msg,
|
||||||
|
|
|
||||||
|
|
@ -297,6 +297,8 @@ class PaperBoi(Struct):
|
||||||
|
|
||||||
# transmit pp msg to ems
|
# transmit pp msg to ems
|
||||||
pp: Position = self.acnt.pps[bs_mktid]
|
pp: Position = self.acnt.pps[bs_mktid]
|
||||||
|
# TODO, this will break if `require_only=True` was passed to
|
||||||
|
# `.update_from_ledger()`
|
||||||
|
|
||||||
pp_msg = BrokerdPosition(
|
pp_msg = BrokerdPosition(
|
||||||
broker=self.broker,
|
broker=self.broker,
|
||||||
|
|
@ -653,6 +655,7 @@ async def open_trade_dialog(
|
||||||
# in) use manually constructed table from calling
|
# in) use manually constructed table from calling
|
||||||
# the `.get_mkt_info()` provider EP above.
|
# the `.get_mkt_info()` provider EP above.
|
||||||
_mktmap_table=mkt_by_fqme,
|
_mktmap_table=mkt_by_fqme,
|
||||||
|
only_require=list(mkt_by_fqme),
|
||||||
)
|
)
|
||||||
|
|
||||||
pp_msgs: list[BrokerdPosition] = []
|
pp_msgs: list[BrokerdPosition] = []
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,7 @@ subsys: str = 'piker.clearing'
|
||||||
|
|
||||||
log = get_logger(subsys)
|
log = get_logger(subsys)
|
||||||
|
|
||||||
|
# TODO, oof doesn't this ignore the `loglevel` then???
|
||||||
get_console_log = partial(
|
get_console_log = partial(
|
||||||
get_console_log,
|
get_console_log,
|
||||||
name=subsys,
|
name=subsys,
|
||||||
|
|
|
||||||
|
|
@ -95,6 +95,12 @@ class Sampler:
|
||||||
# history loading.
|
# history loading.
|
||||||
incr_task_cs: trio.CancelScope | None = None
|
incr_task_cs: trio.CancelScope | None = None
|
||||||
|
|
||||||
|
bcast_errors: tuple[Exception] = (
|
||||||
|
trio.BrokenResourceError,
|
||||||
|
trio.ClosedResourceError,
|
||||||
|
trio.EndOfChannel,
|
||||||
|
)
|
||||||
|
|
||||||
# holds all the ``tractor.Context`` remote subscriptions for
|
# holds all the ``tractor.Context`` remote subscriptions for
|
||||||
# a particular sample period increment event: all subscribers are
|
# a particular sample period increment event: all subscribers are
|
||||||
# notified on a step.
|
# notified on a step.
|
||||||
|
|
@ -258,14 +264,15 @@ class Sampler:
|
||||||
subs: set
|
subs: set
|
||||||
last_ts, subs = pair
|
last_ts, subs = pair
|
||||||
|
|
||||||
task = trio.lowlevel.current_task()
|
# NOTE, for debugging pub-sub issues
|
||||||
log.debug(
|
# task = trio.lowlevel.current_task()
|
||||||
f'SUBS {self.subscribers}\n'
|
# log.debug(
|
||||||
f'PAIR {pair}\n'
|
# f'AlL-SUBS@{period_s!r}: {self.subscribers}\n'
|
||||||
f'TASK: {task}: {id(task)}\n'
|
# f'PAIR: {pair}\n'
|
||||||
f'broadcasting {period_s} -> {last_ts}\n'
|
# f'TASK: {task}: {id(task)}\n'
|
||||||
|
# f'broadcasting {period_s} -> {last_ts}\n'
|
||||||
# f'consumers: {subs}'
|
# f'consumers: {subs}'
|
||||||
)
|
# )
|
||||||
borked: set[MsgStream] = set()
|
borked: set[MsgStream] = set()
|
||||||
sent: set[MsgStream] = set()
|
sent: set[MsgStream] = set()
|
||||||
while True:
|
while True:
|
||||||
|
|
@ -282,12 +289,11 @@ class Sampler:
|
||||||
await stream.send(msg)
|
await stream.send(msg)
|
||||||
sent.add(stream)
|
sent.add(stream)
|
||||||
|
|
||||||
except (
|
except self.bcast_errors as err:
|
||||||
trio.BrokenResourceError,
|
|
||||||
trio.ClosedResourceError
|
|
||||||
):
|
|
||||||
log.error(
|
log.error(
|
||||||
f'{stream._ctx.chan.uid} dropped connection'
|
f'Connection dropped for IPC ctx\n'
|
||||||
|
f'{stream._ctx}\n\n'
|
||||||
|
f'Due to {type(err)}'
|
||||||
)
|
)
|
||||||
borked.add(stream)
|
borked.add(stream)
|
||||||
else:
|
else:
|
||||||
|
|
@ -394,7 +400,8 @@ async def register_with_sampler(
|
||||||
finally:
|
finally:
|
||||||
if (
|
if (
|
||||||
sub_for_broadcasts
|
sub_for_broadcasts
|
||||||
and subs
|
and
|
||||||
|
subs
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
subs.remove(stream)
|
subs.remove(stream)
|
||||||
|
|
@ -561,8 +568,7 @@ async def open_sample_stream(
|
||||||
|
|
||||||
|
|
||||||
async def sample_and_broadcast(
|
async def sample_and_broadcast(
|
||||||
|
bus: _FeedsBus,
|
||||||
bus: _FeedsBus, # noqa
|
|
||||||
rt_shm: ShmArray,
|
rt_shm: ShmArray,
|
||||||
hist_shm: ShmArray,
|
hist_shm: ShmArray,
|
||||||
quote_stream: trio.abc.ReceiveChannel,
|
quote_stream: trio.abc.ReceiveChannel,
|
||||||
|
|
@ -582,11 +588,33 @@ async def sample_and_broadcast(
|
||||||
|
|
||||||
overruns = Counter()
|
overruns = Counter()
|
||||||
|
|
||||||
|
# NOTE, only used for debugging live-data-feed issues, though
|
||||||
|
# this should be resolved more correctly in the future using the
|
||||||
|
# new typed-msgspec feats of `tractor`!
|
||||||
|
#
|
||||||
|
# XXX, a multiline nested `dict` formatter (since rn quote-msgs
|
||||||
|
# are just that).
|
||||||
|
# pfmt: Callable[[str], str] = mk_repr()
|
||||||
|
|
||||||
# iterate stream delivered by broker
|
# iterate stream delivered by broker
|
||||||
async for quotes in quote_stream:
|
async for quotes in quote_stream:
|
||||||
# print(quotes)
|
# print(quotes)
|
||||||
|
|
||||||
# TODO: ``numba`` this!
|
# XXX WARNING XXX only enable for debugging bc ow can cost
|
||||||
|
# ALOT of perf with HF-feedz!!!
|
||||||
|
#
|
||||||
|
# log.info(
|
||||||
|
# 'Rx live quotes:\n'
|
||||||
|
# f'{pfmt(quotes)}'
|
||||||
|
# )
|
||||||
|
|
||||||
|
# TODO,
|
||||||
|
# -[ ] `numba` or `cython`-nize this loop possibly?
|
||||||
|
# |_alternatively could we do it in rust somehow by upacking
|
||||||
|
# arrow msgs instead of using `msgspec`?
|
||||||
|
# -[ ] use `msgspec.Struct` support in new typed-msging from
|
||||||
|
# `tractor` to ensure only allowed msgs are transmitted?
|
||||||
|
#
|
||||||
for broker_symbol, quote in quotes.items():
|
for broker_symbol, quote in quotes.items():
|
||||||
# TODO: in theory you can send the IPC msg *before* writing
|
# TODO: in theory you can send the IPC msg *before* writing
|
||||||
# to the sharedmem array to decrease latency, however, that
|
# to the sharedmem array to decrease latency, however, that
|
||||||
|
|
@ -659,6 +687,21 @@ async def sample_and_broadcast(
|
||||||
sub_key: str = broker_symbol.lower()
|
sub_key: str = broker_symbol.lower()
|
||||||
subs: set[Sub] = bus.get_subs(sub_key)
|
subs: set[Sub] = bus.get_subs(sub_key)
|
||||||
|
|
||||||
|
# TODO, figure out how to make this useful whilst
|
||||||
|
# incoporating feed "pausing" ..
|
||||||
|
#
|
||||||
|
# if not subs:
|
||||||
|
# all_bs_fqmes: list[str] = list(
|
||||||
|
# bus._subscribers.keys()
|
||||||
|
# )
|
||||||
|
# log.warning(
|
||||||
|
# f'No subscribers for {brokername!r} live-quote ??\n'
|
||||||
|
# f'broker_symbol: {broker_symbol}\n\n'
|
||||||
|
|
||||||
|
# f'Maybe the backend-sys symbol does not match one of,\n'
|
||||||
|
# f'{pfmt(all_bs_fqmes)}\n'
|
||||||
|
# )
|
||||||
|
|
||||||
# NOTE: by default the broker backend doesn't append
|
# NOTE: by default the broker backend doesn't append
|
||||||
# it's own "name" into the fqme schema (but maybe it
|
# it's own "name" into the fqme schema (but maybe it
|
||||||
# should?) so we have to manually generate the correct
|
# should?) so we have to manually generate the correct
|
||||||
|
|
@ -728,18 +771,14 @@ async def sample_and_broadcast(
|
||||||
if lags > 10:
|
if lags > 10:
|
||||||
await tractor.pause()
|
await tractor.pause()
|
||||||
|
|
||||||
except (
|
except Sampler.bcast_errors as ipc_err:
|
||||||
trio.BrokenResourceError,
|
|
||||||
trio.ClosedResourceError,
|
|
||||||
trio.EndOfChannel,
|
|
||||||
):
|
|
||||||
ctx: Context = ipc._ctx
|
ctx: Context = ipc._ctx
|
||||||
chan: Channel = ctx.chan
|
chan: Channel = ctx.chan
|
||||||
if ctx:
|
if ctx:
|
||||||
log.warning(
|
log.warning(
|
||||||
'Dropped `brokerd`-quotes-feed connection:\n'
|
f'Dropped `brokerd`-feed for {broker_symbol!r} due to,\n'
|
||||||
f'{broker_symbol}:'
|
f'x>) {ctx.cid}@{chan.uid}'
|
||||||
f'{ctx.cid}@{chan.uid}'
|
f'|_{ipc_err!r}\n\n'
|
||||||
)
|
)
|
||||||
if sub.throttle_rate:
|
if sub.throttle_rate:
|
||||||
assert ipc._closed
|
assert ipc._closed
|
||||||
|
|
@ -756,12 +795,11 @@ async def sample_and_broadcast(
|
||||||
|
|
||||||
|
|
||||||
async def uniform_rate_send(
|
async def uniform_rate_send(
|
||||||
|
|
||||||
rate: float,
|
rate: float,
|
||||||
quote_stream: trio.abc.ReceiveChannel,
|
quote_stream: trio.abc.ReceiveChannel,
|
||||||
stream: MsgStream,
|
stream: MsgStream,
|
||||||
|
|
||||||
task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
|
task_status: TaskStatus[None] = trio.TASK_STATUS_IGNORED,
|
||||||
|
|
||||||
) -> None:
|
) -> None:
|
||||||
'''
|
'''
|
||||||
|
|
@ -779,13 +817,16 @@ async def uniform_rate_send(
|
||||||
https://gist.github.com/njsmith/7ea44ec07e901cb78ebe1dd8dd846cb9
|
https://gist.github.com/njsmith/7ea44ec07e901cb78ebe1dd8dd846cb9
|
||||||
|
|
||||||
'''
|
'''
|
||||||
# TODO: compute the approx overhead latency per cycle
|
# ?TODO? dynamically compute the **actual** approx overhead latency per cycle
|
||||||
left_to_sleep = throttle_period = 1/rate - 0.000616
|
# instead of this magic # bidinezz?
|
||||||
|
throttle_period: float = 1/rate - 0.000616
|
||||||
|
left_to_sleep: float = throttle_period
|
||||||
|
|
||||||
# send cycle state
|
# send cycle state
|
||||||
|
first_quote: dict|None
|
||||||
first_quote = last_quote = None
|
first_quote = last_quote = None
|
||||||
last_send = time.time()
|
last_send: float = time.time()
|
||||||
diff = 0
|
diff: float = 0
|
||||||
|
|
||||||
task_status.started()
|
task_status.started()
|
||||||
ticks_by_type: dict[
|
ticks_by_type: dict[
|
||||||
|
|
@ -796,22 +837,28 @@ async def uniform_rate_send(
|
||||||
clear_types = _tick_groups['clears']
|
clear_types = _tick_groups['clears']
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
|
|
||||||
# compute the remaining time to sleep for this throttled cycle
|
# compute the remaining time to sleep for this throttled cycle
|
||||||
left_to_sleep = throttle_period - diff
|
left_to_sleep: float = throttle_period - diff
|
||||||
|
|
||||||
if left_to_sleep > 0:
|
if left_to_sleep > 0:
|
||||||
|
cs: trio.CancelScope
|
||||||
with trio.move_on_after(left_to_sleep) as cs:
|
with trio.move_on_after(left_to_sleep) as cs:
|
||||||
|
sym: str
|
||||||
|
last_quote: dict
|
||||||
try:
|
try:
|
||||||
sym, last_quote = await quote_stream.receive()
|
sym, last_quote = await quote_stream.receive()
|
||||||
except trio.EndOfChannel:
|
except trio.EndOfChannel:
|
||||||
log.exception(f"feed for {stream} ended?")
|
log.exception(
|
||||||
|
f'Live stream for feed for ended?\n'
|
||||||
|
f'<=c\n'
|
||||||
|
f' |_[{stream!r}\n'
|
||||||
|
)
|
||||||
break
|
break
|
||||||
|
|
||||||
diff = time.time() - last_send
|
diff: float = time.time() - last_send
|
||||||
|
|
||||||
if not first_quote:
|
if not first_quote:
|
||||||
first_quote = last_quote
|
first_quote: float = last_quote
|
||||||
# first_quote['tbt'] = ticks_by_type
|
# first_quote['tbt'] = ticks_by_type
|
||||||
|
|
||||||
if (throttle_period - diff) > 0:
|
if (throttle_period - diff) > 0:
|
||||||
|
|
@ -872,7 +919,9 @@ async def uniform_rate_send(
|
||||||
# TODO: now if only we could sync this to the display
|
# TODO: now if only we could sync this to the display
|
||||||
# rate timing exactly lul
|
# rate timing exactly lul
|
||||||
try:
|
try:
|
||||||
await stream.send({sym: first_quote})
|
await stream.send({
|
||||||
|
sym: first_quote
|
||||||
|
})
|
||||||
except tractor.RemoteActorError as rme:
|
except tractor.RemoteActorError as rme:
|
||||||
if rme.type is not tractor._exceptions.StreamOverrun:
|
if rme.type is not tractor._exceptions.StreamOverrun:
|
||||||
raise
|
raise
|
||||||
|
|
@ -883,19 +932,28 @@ async def uniform_rate_send(
|
||||||
f'{sym}:{ctx.cid}@{chan.uid}'
|
f'{sym}:{ctx.cid}@{chan.uid}'
|
||||||
)
|
)
|
||||||
|
|
||||||
except (
|
# NOTE: any of these can be raised by `tractor`'s IPC
|
||||||
# NOTE: any of these can be raised by ``tractor``'s IPC
|
|
||||||
# transport-layer and we want to be highly resilient
|
# transport-layer and we want to be highly resilient
|
||||||
# to consumers which crash or lose network connection.
|
# to consumers which crash or lose network connection.
|
||||||
# I.e. we **DO NOT** want to crash and propagate up to
|
# I.e. we **DO NOT** want to crash and propagate up to
|
||||||
# ``pikerd`` these kinds of errors!
|
# ``pikerd`` these kinds of errors!
|
||||||
trio.ClosedResourceError,
|
except (
|
||||||
trio.BrokenResourceError,
|
|
||||||
ConnectionResetError,
|
ConnectionResetError,
|
||||||
):
|
) + Sampler.bcast_errors as ipc_err:
|
||||||
|
match ipc_err:
|
||||||
|
case trio.EndOfChannel():
|
||||||
|
log.info(
|
||||||
|
f'{stream} terminated by peer,\n'
|
||||||
|
f'{ipc_err!r}'
|
||||||
|
)
|
||||||
|
case _:
|
||||||
# if the feed consumer goes down then drop
|
# if the feed consumer goes down then drop
|
||||||
# out of this rate limiter
|
# out of this rate limiter
|
||||||
log.warning(f'{stream} closed')
|
log.warning(
|
||||||
|
f'{stream} closed due to,\n'
|
||||||
|
f'{ipc_err!r}'
|
||||||
|
)
|
||||||
|
|
||||||
await stream.aclose()
|
await stream.aclose()
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ from pathlib import Path
|
||||||
from pprint import pformat
|
from pprint import pformat
|
||||||
from typing import (
|
from typing import (
|
||||||
Any,
|
Any,
|
||||||
|
Callable,
|
||||||
Sequence,
|
Sequence,
|
||||||
Hashable,
|
Hashable,
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
|
|
@ -56,7 +57,7 @@ from piker.brokers import (
|
||||||
)
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from ..accounting import (
|
from piker.accounting import (
|
||||||
Asset,
|
Asset,
|
||||||
MktPair,
|
MktPair,
|
||||||
)
|
)
|
||||||
|
|
@ -149,19 +150,36 @@ class SymbologyCache(Struct):
|
||||||
'Implement `Client.get_assets()`!'
|
'Implement `Client.get_assets()`!'
|
||||||
)
|
)
|
||||||
|
|
||||||
if get_mkt_pairs := getattr(client, 'get_mkt_pairs', None):
|
get_mkt_pairs: Callable|None = getattr(
|
||||||
|
client,
|
||||||
|
'get_mkt_pairs',
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if not get_mkt_pairs:
|
||||||
|
log.warning(
|
||||||
|
'No symbology cache `Pair` support for `{provider}`..\n'
|
||||||
|
'Implement `Client.get_mkt_pairs()`!'
|
||||||
|
)
|
||||||
|
return self
|
||||||
|
|
||||||
pairs: dict[str, Struct] = await get_mkt_pairs()
|
pairs: dict[str, Struct] = await get_mkt_pairs()
|
||||||
for bs_fqme, pair in pairs.items():
|
if not pairs:
|
||||||
|
log.warning(
|
||||||
|
'No pairs from intial {provider!r} sym-cache request?\n\n'
|
||||||
|
'`Client.get_mkt_pairs()` -> {pairs!r} ?'
|
||||||
|
)
|
||||||
|
return self
|
||||||
|
|
||||||
# NOTE: every backend defined pair should
|
for bs_fqme, pair in pairs.items():
|
||||||
# declare it's ns path for roundtrip
|
|
||||||
# serialization lookup.
|
|
||||||
if not getattr(pair, 'ns_path', None):
|
if not getattr(pair, 'ns_path', None):
|
||||||
|
# XXX: every backend defined pair must declare
|
||||||
|
# a `.ns_path: tractor.NamespacePath` to enable
|
||||||
|
# roundtrip serialization lookup from a local
|
||||||
|
# cache file.
|
||||||
raise TypeError(
|
raise TypeError(
|
||||||
f'Pair-struct for {self.mod.name} MUST define a '
|
f'Pair-struct for {self.mod.name} MUST define a '
|
||||||
'`.ns_path: str`!\n'
|
'`.ns_path: str`!\n\n'
|
||||||
f'{pair}'
|
f'{pair!r}'
|
||||||
)
|
)
|
||||||
|
|
||||||
entry = await self.mod.get_mkt_info(pair.bs_fqme)
|
entry = await self.mod.get_mkt_info(pair.bs_fqme)
|
||||||
|
|
@ -195,12 +213,6 @@ class SymbologyCache(Struct):
|
||||||
pair,
|
pair,
|
||||||
)
|
)
|
||||||
|
|
||||||
else:
|
|
||||||
log.warning(
|
|
||||||
'No symbology cache `Pair` support for `{provider}`..\n'
|
|
||||||
'Implement `Client.get_mkt_pairs()`!'
|
|
||||||
)
|
|
||||||
|
|
||||||
return self
|
return self
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|
|
||||||
|
|
@ -786,7 +786,6 @@ async def install_brokerd_search(
|
||||||
|
|
||||||
@acm
|
@acm
|
||||||
async def maybe_open_feed(
|
async def maybe_open_feed(
|
||||||
|
|
||||||
fqmes: list[str],
|
fqmes: list[str],
|
||||||
loglevel: str | None = None,
|
loglevel: str | None = None,
|
||||||
|
|
||||||
|
|
@ -840,7 +839,6 @@ async def maybe_open_feed(
|
||||||
|
|
||||||
@acm
|
@acm
|
||||||
async def open_feed(
|
async def open_feed(
|
||||||
|
|
||||||
fqmes: list[str],
|
fqmes: list[str],
|
||||||
|
|
||||||
loglevel: str|None = None,
|
loglevel: str|None = None,
|
||||||
|
|
|
||||||
|
|
@ -36,10 +36,10 @@ from ._sharedmem import (
|
||||||
ShmArray,
|
ShmArray,
|
||||||
_Token,
|
_Token,
|
||||||
)
|
)
|
||||||
|
from piker.accounting import MktPair
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from ..accounting import MktPair
|
from piker.data.feed import Feed
|
||||||
from .feed import Feed
|
|
||||||
|
|
||||||
|
|
||||||
class Flume(Struct):
|
class Flume(Struct):
|
||||||
|
|
|
||||||
|
|
@ -113,9 +113,9 @@ def validate_backend(
|
||||||
)
|
)
|
||||||
if ep is None:
|
if ep is None:
|
||||||
log.warning(
|
log.warning(
|
||||||
f'Provider backend {mod.name} is missing '
|
f'Provider backend {mod.name!r} is missing '
|
||||||
f'{daemon_name} support :(\n'
|
f'{daemon_name!r} support?\n'
|
||||||
f'The following endpoint is missing: {name}'
|
f'|_module endpoint-func missing: {name!r}\n'
|
||||||
)
|
)
|
||||||
|
|
||||||
inits: list[
|
inits: list[
|
||||||
|
|
|
||||||
30
piker/log.py
30
piker/log.py
|
|
@ -19,6 +19,10 @@ Log like a forester!
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import json
|
import json
|
||||||
|
import reprlib
|
||||||
|
from typing import (
|
||||||
|
Callable,
|
||||||
|
)
|
||||||
|
|
||||||
import tractor
|
import tractor
|
||||||
from pygments import (
|
from pygments import (
|
||||||
|
|
@ -84,3 +88,29 @@ def colorize_json(
|
||||||
# likeable styles: algol_nu, tango, monokai
|
# likeable styles: algol_nu, tango, monokai
|
||||||
formatters.TerminalTrueColorFormatter(style=style)
|
formatters.TerminalTrueColorFormatter(style=style)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# TODO, eventually defer to the version in `modden` once
|
||||||
|
# it becomes a dep!
|
||||||
|
def mk_repr(
|
||||||
|
**repr_kws,
|
||||||
|
) -> Callable[[str], str]:
|
||||||
|
'''
|
||||||
|
Allocate and deliver a `repr.Repr` instance with provided input
|
||||||
|
settings using the std-lib's `reprlib` mod,
|
||||||
|
* https://docs.python.org/3/library/reprlib.html
|
||||||
|
|
||||||
|
------ Ex. ------
|
||||||
|
An up to 6-layer-nested `dict` as multi-line:
|
||||||
|
- https://stackoverflow.com/a/79102479
|
||||||
|
- https://docs.python.org/3/library/reprlib.html#reprlib.Repr.maxlevel
|
||||||
|
|
||||||
|
'''
|
||||||
|
def_kws: dict[str, int] = dict(
|
||||||
|
indent=2,
|
||||||
|
maxlevel=6, # recursion levels
|
||||||
|
maxstring=66, # match editor line-len limit
|
||||||
|
)
|
||||||
|
def_kws |= repr_kws
|
||||||
|
reprr = reprlib.Repr(**def_kws)
|
||||||
|
return reprr.repr
|
||||||
|
|
|
||||||
|
|
@ -130,4 +130,5 @@ pyqtgraph = { git = "https://github.com/pikers/pyqtgraph.git" }
|
||||||
asyncvnc = { git = "https://github.com/pikers/asyncvnc.git", branch = "main" }
|
asyncvnc = { git = "https://github.com/pikers/asyncvnc.git", branch = "main" }
|
||||||
tomlkit = { git = "https://github.com/pikers/tomlkit.git", branch ="piker_pin" }
|
tomlkit = { git = "https://github.com/pikers/tomlkit.git", branch ="piker_pin" }
|
||||||
msgspec = { git = "https://github.com/jcrist/msgspec.git" }
|
msgspec = { git = "https://github.com/jcrist/msgspec.git" }
|
||||||
tractor = { path = "../tractor", editable = true }
|
tractor = { git = "https://pikers.dev/goodboy/tractor", branch = "piker_pin" }
|
||||||
|
# tractor = { path = "../tractor", editable = true }
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue