Formalize transaction normalizer func signature

Since each broker backend generally needs to define a specific
field-name-schema to determine the exact instantiation arguments to
`Transaction`, we generally need each backend to define an endpoint
function to conduct this transaction from an input `dict[str, Any]`
received either directly from provided ledger APIs or from previously
stored `.accounting._ledger` saved trades ledger TOML files.

To accomplish this we now require backends to declare a new routine:

```python
def norm_trade(
    tid: str,  # the uuid for the transaction
    txdict: dict,  # the input record-dict

    # a table of mkt-symbols to backend
    # struct objects which define the (meta-data) for the backend specific
    # venue's symbology
    pairs: dict[str, Struct],

) -> Transaction:
    ...
```

which implements that record conversion (at least for trades)
and can thus be used in `TransactionLedger.iter_txns()` which requires
"some code" to implement the loading from a serialization format (aka
the input `dict` record) to our local `Transaction` struct, normally
also using a `Pair`-struct table defined (and maybe previously cached)
by the specific backend such our (normalization layer's) `MktPair`'s
fields can be set.

For the case of our `.clearing._paper_engine` we def the routine to
simply extract the exact same fields from the TOML ledger records that
we previously had written (to it) and define it in that module.

Also, we always pass `pairs=SymbologyCache.pairs: dict[str, Struct]` on
norm trade calls such that offline ledger and accounting processing
clients can use a previously cached symbology set without having to
necessarily start the async-actor runtime to query the actual backend API
if the data has already been saved locally on the system B)

Other related:
- always passthrough kwargs in overridden `.to_dict()` method.
- only do fqme related trade record field name rewrites/names when
  operating on a paper ledger; normally a backend's records don't
  contain these.
- fix `pendulum.DateTime` type annots.
- just deliver `Transaction`s from `.iter_txns()`
account_tests
Tyler Goodlet 2023-07-14 14:11:49 -04:00
parent da206f5242
commit 494b3faa9b
1 changed files with 78 additions and 50 deletions

View File

@ -32,9 +32,7 @@ from typing import (
) )
from pendulum import ( from pendulum import (
datetime, DateTime,
# DateTime,
# parse,
) )
import tomli_w # for fast ledger writing import tomli_w # for fast ledger writing
@ -70,7 +68,7 @@ class Transaction(Struct, frozen=True):
size: float size: float
price: float price: float
cost: float # commisions or other additional costs cost: float # commisions or other additional costs
dt: datetime dt: DateTime
# the "event type" in terms of "market events" see # the "event type" in terms of "market events" see
# https://github.com/pikers/piker/issues/510 for where we're # https://github.com/pikers/piker/issues/510 for where we're
@ -80,7 +78,7 @@ class Transaction(Struct, frozen=True):
# TODO: we can drop this right since we # TODO: we can drop this right since we
# can instead expect the backend to provide this # can instead expect the backend to provide this
# via the `MktPair`? # via the `MktPair`?
expiry: datetime | None = None expiry: DateTime | None = None
# (optional) key-id defined by the broker-service backend which # (optional) key-id defined by the broker-service backend which
# ensures the instrument-symbol market key for this record is unique # ensures the instrument-symbol market key for this record is unique
@ -89,8 +87,11 @@ class Transaction(Struct, frozen=True):
# service. # service.
bs_mktid: str | int | None = None bs_mktid: str | int | None = None
def to_dict(self) -> dict: def to_dict(
dct: dict[str, Any] = super().to_dict() self,
**kwargs,
) -> dict:
dct: dict[str, Any] = super().to_dict(**kwargs)
# ensure we use a pendulum formatted # ensure we use a pendulum formatted
# ISO style str here!@ # ISO style str here!@
@ -107,6 +108,8 @@ class TransactionLedger(UserDict):
outside. outside.
''' '''
# NOTE: see `open_trade_ledger()` for defaults, this should
# never be constructed manually!
def __init__( def __init__(
self, self,
ledger_dict: dict, ledger_dict: dict,
@ -138,6 +141,10 @@ class TransactionLedger(UserDict):
@property @property
def symcache(self) -> SymbologyCache: def symcache(self) -> SymbologyCache:
'''
Read-only ref to backend's ``SymbologyCache``.
'''
return self._symcache return self._symcache
def update_from_t( def update_from_t(
@ -157,7 +164,7 @@ class TransactionLedger(UserDict):
symcache: SymbologyCache | None = None, symcache: SymbologyCache | None = None,
) -> Generator[ ) -> Generator[
tuple[str, Transaction], Transaction,
None, None,
None, None,
]: ]:
@ -175,9 +182,13 @@ class TransactionLedger(UserDict):
norm_trade = self.mod.norm_trade norm_trade = self.mod.norm_trade
# datetime-sort and pack into txs # datetime-sort and pack into txs
for txdict in self.tx_sort(self.data.values()): for tid, txdict in self.tx_sort(self.data.items()):
txn = norm_trade(txdict) txn: Transaction = norm_trade(
yield txn.tid, txn tid,
txdict,
pairs=symcache.pairs,
)
yield txn
def to_txns( def to_txns(
self, self,
@ -188,18 +199,19 @@ class TransactionLedger(UserDict):
Return entire output from ``.iter_txns()`` in a ``dict``. Return entire output from ``.iter_txns()`` in a ``dict``.
''' '''
return dict(self.iter_txns(symcache=symcache)) return {
t.tid: t for t in self.iter_txns(symcache=symcache)
}
def write_config( def write_config(self) -> None:
self,
) -> None:
''' '''
Render the self.data ledger dict to its TOML file form. Render the self.data ledger dict to its TOML file form.
ALWAYS order datetime sorted! ALWAYS order datetime sorted!
''' '''
is_paper: bool = self.account == 'paper'
towrite: dict[str, Any] = {} towrite: dict[str, Any] = {}
for tid, txdict in self.tx_sort(self.data.copy()): for tid, txdict in self.tx_sort(self.data.copy()):
# write blank-str expiry for non-expiring assets # write blank-str expiry for non-expiring assets
@ -210,42 +222,44 @@ class TransactionLedger(UserDict):
txdict['expiry'] = '' txdict['expiry'] = ''
# (maybe) re-write old acro-key # (maybe) re-write old acro-key
fqme: str = txdict.pop('fqsn', None) or txdict['fqme'] if is_paper:
bs_mktid: str | None = txdict.get('bs_mktid') fqme: str = txdict.pop('fqsn', None) or txdict['fqme']
bs_mktid: str | None = txdict.get('bs_mktid')
if ( if (
fqme not in self._symcache.mktmaps fqme not in self._symcache.mktmaps
or ( or (
# also try to see if this is maybe a paper # also try to see if this is maybe a paper
# engine ledger in which case the bs_mktid # engine ledger in which case the bs_mktid
# should be the fqme as well! # should be the fqme as well!
self.account == 'paper' bs_mktid
and bs_mktid and fqme != bs_mktid
and fqme != bs_mktid
)
):
# always take any (paper) bs_mktid if defined and
# in the backend's cache key set.
if bs_mktid in self._symcache.mktmaps:
fqme: str = bs_mktid
else:
best_fqme: str = list(self._symcache.search(fqme))[0]
log.warning(
f'Could not find FQME: {fqme} in qualified set?\n'
f'Qualifying and expanding {fqme} -> {best_fqme}'
) )
fqme = best_fqme ):
# always take any (paper) bs_mktid if defined and
# in the backend's cache key set.
if bs_mktid in self._symcache.mktmaps:
fqme: str = bs_mktid
else:
best_fqme: str = list(self._symcache.search(fqme))[0]
log.warning(
f'Could not find FQME: {fqme} in qualified set?\n'
f'Qualifying and expanding {fqme} -> {best_fqme}'
)
fqme = best_fqme
if ( if (
self.account == 'paper' bs_mktid
and bs_mktid and bs_mktid != fqme
and bs_mktid != fqme ):
): # in paper account case always make sure both the
# in paper account case always make sure both the # fqme and bs_mktid are fully qualified..
# fqme and bs_mktid are fully qualified.. txdict['bs_mktid'] = fqme
txdict['bs_mktid'] = fqme
# in paper ledgers always write the latest
# symbology key field: an FQME.
txdict['fqme'] = fqme
txdict['fqme'] = fqme
towrite[tid] = txdict towrite[tid] = txdict
with self.file_path.open(mode='wb') as fp: with self.file_path.open(mode='wb') as fp:
@ -256,6 +270,9 @@ def load_ledger(
brokername: str, brokername: str,
acctid: str, acctid: str,
# for testing or manual load from file
dirpath: Path | None = None,
) -> tuple[dict, Path]: ) -> tuple[dict, Path]:
''' '''
Load a ledger (TOML) file from user's config directory: Load a ledger (TOML) file from user's config directory:
@ -270,7 +287,11 @@ def load_ledger(
except ModuleNotFoundError: except ModuleNotFoundError:
import tomli as tomllib import tomli as tomllib
ldir: Path = config._config_dir / 'accounting' / 'ledgers' ldir: Path = (
dirpath
or
config._config_dir / 'accounting' / 'ledgers'
)
if not ldir.is_dir(): if not ldir.is_dir():
ldir.mkdir() ldir.mkdir()
@ -303,6 +324,9 @@ def open_trade_ledger(
tx_sort: Callable = iter_by_dt, tx_sort: Callable = iter_by_dt,
rewrite: bool = False, rewrite: bool = False,
# for testing or manual load from file
_fp: Path | None = None,
) -> Generator[TransactionLedger, None, None]: ) -> Generator[TransactionLedger, None, None]:
''' '''
Indempotently create and read in a trade log file from the Indempotently create and read in a trade log file from the
@ -316,7 +340,11 @@ def open_trade_ledger(
from ..brokers import get_brokermod from ..brokers import get_brokermod
mod: ModuleType = get_brokermod(broker) mod: ModuleType = get_brokermod(broker)
ledger_dict, fpath = load_ledger(broker, account) ledger_dict, fpath = load_ledger(
broker,
account,
dirpath=_fp,
)
cpy = ledger_dict.copy() cpy = ledger_dict.copy()
# XXX NOTE: if not provided presume we are being called from # XXX NOTE: if not provided presume we are being called from