Formalize transaction normalizer func signature
Since each broker backend generally needs to define a specific field-name-schema to determine the exact instantiation arguments to `Transaction`, we generally need each backend to define an endpoint function to conduct this transaction from an input `dict[str, Any]` received either directly from provided ledger APIs or from previously stored `.accounting._ledger` saved trades ledger TOML files. To accomplish this we now require backends to declare a new routine: ```python def norm_trade( tid: str, # the uuid for the transaction txdict: dict, # the input record-dict # a table of mkt-symbols to backend # struct objects which define the (meta-data) for the backend specific # venue's symbology pairs: dict[str, Struct], ) -> Transaction: ... ``` which implements that record conversion (at least for trades) and can thus be used in `TransactionLedger.iter_txns()` which requires "some code" to implement the loading from a serialization format (aka the input `dict` record) to our local `Transaction` struct, normally also using a `Pair`-struct table defined (and maybe previously cached) by the specific backend such our (normalization layer's) `MktPair`'s fields can be set. For the case of our `.clearing._paper_engine` we def the routine to simply extract the exact same fields from the TOML ledger records that we previously had written (to it) and define it in that module. Also, we always pass `pairs=SymbologyCache.pairs: dict[str, Struct]` on norm trade calls such that offline ledger and accounting processing clients can use a previously cached symbology set without having to necessarily start the async-actor runtime to query the actual backend API if the data has already been saved locally on the system B) Other related: - always passthrough kwargs in overridden `.to_dict()` method. - only do fqme related trade record field name rewrites/names when operating on a paper ledger; normally a backend's records don't contain these. - fix `pendulum.DateTime` type annots. - just deliver `Transaction`s from `.iter_txns()`account_tests
parent
da206f5242
commit
494b3faa9b
|
@ -32,9 +32,7 @@ from typing import (
|
||||||
)
|
)
|
||||||
|
|
||||||
from pendulum import (
|
from pendulum import (
|
||||||
datetime,
|
DateTime,
|
||||||
# DateTime,
|
|
||||||
# parse,
|
|
||||||
)
|
)
|
||||||
import tomli_w # for fast ledger writing
|
import tomli_w # for fast ledger writing
|
||||||
|
|
||||||
|
@ -70,7 +68,7 @@ class Transaction(Struct, frozen=True):
|
||||||
size: float
|
size: float
|
||||||
price: float
|
price: float
|
||||||
cost: float # commisions or other additional costs
|
cost: float # commisions or other additional costs
|
||||||
dt: datetime
|
dt: DateTime
|
||||||
|
|
||||||
# the "event type" in terms of "market events" see
|
# the "event type" in terms of "market events" see
|
||||||
# https://github.com/pikers/piker/issues/510 for where we're
|
# https://github.com/pikers/piker/issues/510 for where we're
|
||||||
|
@ -80,7 +78,7 @@ class Transaction(Struct, frozen=True):
|
||||||
# TODO: we can drop this right since we
|
# TODO: we can drop this right since we
|
||||||
# can instead expect the backend to provide this
|
# can instead expect the backend to provide this
|
||||||
# via the `MktPair`?
|
# via the `MktPair`?
|
||||||
expiry: datetime | None = None
|
expiry: DateTime | None = None
|
||||||
|
|
||||||
# (optional) key-id defined by the broker-service backend which
|
# (optional) key-id defined by the broker-service backend which
|
||||||
# ensures the instrument-symbol market key for this record is unique
|
# ensures the instrument-symbol market key for this record is unique
|
||||||
|
@ -89,8 +87,11 @@ class Transaction(Struct, frozen=True):
|
||||||
# service.
|
# service.
|
||||||
bs_mktid: str | int | None = None
|
bs_mktid: str | int | None = None
|
||||||
|
|
||||||
def to_dict(self) -> dict:
|
def to_dict(
|
||||||
dct: dict[str, Any] = super().to_dict()
|
self,
|
||||||
|
**kwargs,
|
||||||
|
) -> dict:
|
||||||
|
dct: dict[str, Any] = super().to_dict(**kwargs)
|
||||||
|
|
||||||
# ensure we use a pendulum formatted
|
# ensure we use a pendulum formatted
|
||||||
# ISO style str here!@
|
# ISO style str here!@
|
||||||
|
@ -107,6 +108,8 @@ class TransactionLedger(UserDict):
|
||||||
outside.
|
outside.
|
||||||
|
|
||||||
'''
|
'''
|
||||||
|
# NOTE: see `open_trade_ledger()` for defaults, this should
|
||||||
|
# never be constructed manually!
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
ledger_dict: dict,
|
ledger_dict: dict,
|
||||||
|
@ -138,6 +141,10 @@ class TransactionLedger(UserDict):
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def symcache(self) -> SymbologyCache:
|
def symcache(self) -> SymbologyCache:
|
||||||
|
'''
|
||||||
|
Read-only ref to backend's ``SymbologyCache``.
|
||||||
|
|
||||||
|
'''
|
||||||
return self._symcache
|
return self._symcache
|
||||||
|
|
||||||
def update_from_t(
|
def update_from_t(
|
||||||
|
@ -157,7 +164,7 @@ class TransactionLedger(UserDict):
|
||||||
symcache: SymbologyCache | None = None,
|
symcache: SymbologyCache | None = None,
|
||||||
|
|
||||||
) -> Generator[
|
) -> Generator[
|
||||||
tuple[str, Transaction],
|
Transaction,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
]:
|
]:
|
||||||
|
@ -175,9 +182,13 @@ class TransactionLedger(UserDict):
|
||||||
norm_trade = self.mod.norm_trade
|
norm_trade = self.mod.norm_trade
|
||||||
|
|
||||||
# datetime-sort and pack into txs
|
# datetime-sort and pack into txs
|
||||||
for txdict in self.tx_sort(self.data.values()):
|
for tid, txdict in self.tx_sort(self.data.items()):
|
||||||
txn = norm_trade(txdict)
|
txn: Transaction = norm_trade(
|
||||||
yield txn.tid, txn
|
tid,
|
||||||
|
txdict,
|
||||||
|
pairs=symcache.pairs,
|
||||||
|
)
|
||||||
|
yield txn
|
||||||
|
|
||||||
def to_txns(
|
def to_txns(
|
||||||
self,
|
self,
|
||||||
|
@ -188,18 +199,19 @@ class TransactionLedger(UserDict):
|
||||||
Return entire output from ``.iter_txns()`` in a ``dict``.
|
Return entire output from ``.iter_txns()`` in a ``dict``.
|
||||||
|
|
||||||
'''
|
'''
|
||||||
return dict(self.iter_txns(symcache=symcache))
|
return {
|
||||||
|
t.tid: t for t in self.iter_txns(symcache=symcache)
|
||||||
|
}
|
||||||
|
|
||||||
def write_config(
|
def write_config(self) -> None:
|
||||||
self,
|
|
||||||
|
|
||||||
) -> None:
|
|
||||||
'''
|
'''
|
||||||
Render the self.data ledger dict to its TOML file form.
|
Render the self.data ledger dict to its TOML file form.
|
||||||
|
|
||||||
ALWAYS order datetime sorted!
|
ALWAYS order datetime sorted!
|
||||||
|
|
||||||
'''
|
'''
|
||||||
|
is_paper: bool = self.account == 'paper'
|
||||||
|
|
||||||
towrite: dict[str, Any] = {}
|
towrite: dict[str, Any] = {}
|
||||||
for tid, txdict in self.tx_sort(self.data.copy()):
|
for tid, txdict in self.tx_sort(self.data.copy()):
|
||||||
# write blank-str expiry for non-expiring assets
|
# write blank-str expiry for non-expiring assets
|
||||||
|
@ -210,42 +222,44 @@ class TransactionLedger(UserDict):
|
||||||
txdict['expiry'] = ''
|
txdict['expiry'] = ''
|
||||||
|
|
||||||
# (maybe) re-write old acro-key
|
# (maybe) re-write old acro-key
|
||||||
fqme: str = txdict.pop('fqsn', None) or txdict['fqme']
|
if is_paper:
|
||||||
bs_mktid: str | None = txdict.get('bs_mktid')
|
fqme: str = txdict.pop('fqsn', None) or txdict['fqme']
|
||||||
|
bs_mktid: str | None = txdict.get('bs_mktid')
|
||||||
|
|
||||||
if (
|
if (
|
||||||
fqme not in self._symcache.mktmaps
|
fqme not in self._symcache.mktmaps
|
||||||
or (
|
or (
|
||||||
# also try to see if this is maybe a paper
|
# also try to see if this is maybe a paper
|
||||||
# engine ledger in which case the bs_mktid
|
# engine ledger in which case the bs_mktid
|
||||||
# should be the fqme as well!
|
# should be the fqme as well!
|
||||||
self.account == 'paper'
|
bs_mktid
|
||||||
and bs_mktid
|
and fqme != bs_mktid
|
||||||
and fqme != bs_mktid
|
|
||||||
)
|
|
||||||
):
|
|
||||||
# always take any (paper) bs_mktid if defined and
|
|
||||||
# in the backend's cache key set.
|
|
||||||
if bs_mktid in self._symcache.mktmaps:
|
|
||||||
fqme: str = bs_mktid
|
|
||||||
else:
|
|
||||||
best_fqme: str = list(self._symcache.search(fqme))[0]
|
|
||||||
log.warning(
|
|
||||||
f'Could not find FQME: {fqme} in qualified set?\n'
|
|
||||||
f'Qualifying and expanding {fqme} -> {best_fqme}'
|
|
||||||
)
|
)
|
||||||
fqme = best_fqme
|
):
|
||||||
|
# always take any (paper) bs_mktid if defined and
|
||||||
|
# in the backend's cache key set.
|
||||||
|
if bs_mktid in self._symcache.mktmaps:
|
||||||
|
fqme: str = bs_mktid
|
||||||
|
else:
|
||||||
|
best_fqme: str = list(self._symcache.search(fqme))[0]
|
||||||
|
log.warning(
|
||||||
|
f'Could not find FQME: {fqme} in qualified set?\n'
|
||||||
|
f'Qualifying and expanding {fqme} -> {best_fqme}'
|
||||||
|
)
|
||||||
|
fqme = best_fqme
|
||||||
|
|
||||||
if (
|
if (
|
||||||
self.account == 'paper'
|
bs_mktid
|
||||||
and bs_mktid
|
and bs_mktid != fqme
|
||||||
and bs_mktid != fqme
|
):
|
||||||
):
|
# in paper account case always make sure both the
|
||||||
# in paper account case always make sure both the
|
# fqme and bs_mktid are fully qualified..
|
||||||
# fqme and bs_mktid are fully qualified..
|
txdict['bs_mktid'] = fqme
|
||||||
txdict['bs_mktid'] = fqme
|
|
||||||
|
# in paper ledgers always write the latest
|
||||||
|
# symbology key field: an FQME.
|
||||||
|
txdict['fqme'] = fqme
|
||||||
|
|
||||||
txdict['fqme'] = fqme
|
|
||||||
towrite[tid] = txdict
|
towrite[tid] = txdict
|
||||||
|
|
||||||
with self.file_path.open(mode='wb') as fp:
|
with self.file_path.open(mode='wb') as fp:
|
||||||
|
@ -256,6 +270,9 @@ def load_ledger(
|
||||||
brokername: str,
|
brokername: str,
|
||||||
acctid: str,
|
acctid: str,
|
||||||
|
|
||||||
|
# for testing or manual load from file
|
||||||
|
dirpath: Path | None = None,
|
||||||
|
|
||||||
) -> tuple[dict, Path]:
|
) -> tuple[dict, Path]:
|
||||||
'''
|
'''
|
||||||
Load a ledger (TOML) file from user's config directory:
|
Load a ledger (TOML) file from user's config directory:
|
||||||
|
@ -270,7 +287,11 @@ def load_ledger(
|
||||||
except ModuleNotFoundError:
|
except ModuleNotFoundError:
|
||||||
import tomli as tomllib
|
import tomli as tomllib
|
||||||
|
|
||||||
ldir: Path = config._config_dir / 'accounting' / 'ledgers'
|
ldir: Path = (
|
||||||
|
dirpath
|
||||||
|
or
|
||||||
|
config._config_dir / 'accounting' / 'ledgers'
|
||||||
|
)
|
||||||
if not ldir.is_dir():
|
if not ldir.is_dir():
|
||||||
ldir.mkdir()
|
ldir.mkdir()
|
||||||
|
|
||||||
|
@ -303,6 +324,9 @@ def open_trade_ledger(
|
||||||
tx_sort: Callable = iter_by_dt,
|
tx_sort: Callable = iter_by_dt,
|
||||||
rewrite: bool = False,
|
rewrite: bool = False,
|
||||||
|
|
||||||
|
# for testing or manual load from file
|
||||||
|
_fp: Path | None = None,
|
||||||
|
|
||||||
) -> Generator[TransactionLedger, None, None]:
|
) -> Generator[TransactionLedger, None, None]:
|
||||||
'''
|
'''
|
||||||
Indempotently create and read in a trade log file from the
|
Indempotently create and read in a trade log file from the
|
||||||
|
@ -316,7 +340,11 @@ def open_trade_ledger(
|
||||||
from ..brokers import get_brokermod
|
from ..brokers import get_brokermod
|
||||||
mod: ModuleType = get_brokermod(broker)
|
mod: ModuleType = get_brokermod(broker)
|
||||||
|
|
||||||
ledger_dict, fpath = load_ledger(broker, account)
|
ledger_dict, fpath = load_ledger(
|
||||||
|
broker,
|
||||||
|
account,
|
||||||
|
dirpath=_fp,
|
||||||
|
)
|
||||||
cpy = ledger_dict.copy()
|
cpy = ledger_dict.copy()
|
||||||
|
|
||||||
# XXX NOTE: if not provided presume we are being called from
|
# XXX NOTE: if not provided presume we are being called from
|
||||||
|
|
Loading…
Reference in New Issue