pikers
/
piker
3
0
Fork 0
Code Issues 2 Pull Requests 4 Packages Projects Releases Wiki Activity

Formalize transaction normalizer func signature 

Since each broker backend generally needs to define a specific
field-name-schema to determine the exact instantiation arguments to
`Transaction`, we generally need each backend to define an endpoint
function to conduct this transaction from an input `dict[str, Any]`
received either directly from provided ledger APIs or from previously
stored `.accounting._ledger` saved trades ledger TOML files.

To accomplish this we now require backends to declare a new routine:

```python
def norm_trade(
    tid: str,  # the uuid for the transaction
    txdict: dict,  # the input record-dict

    # a table of mkt-symbols to backend
    # struct objects which define the (meta-data) for the backend specific
    # venue's symbology
    pairs: dict[str, Struct],

) -> Transaction:
    ...
```

which implements that record conversion (at least for trades)
and can thus be used in `TransactionLedger.iter_txns()` which requires
"some code" to implement the loading from a serialization format (aka
the input `dict` record) to our local `Transaction` struct, normally
also using a `Pair`-struct table defined (and maybe previously cached)
by the specific backend such our (normalization layer's) `MktPair`'s
fields can be set.

For the case of our `.clearing._paper_engine` we def the routine to
simply extract the exact same fields from the TOML ledger records that
we previously had written (to it) and define it in that module.

Also, we always pass `pairs=SymbologyCache.pairs: dict[str, Struct]` on
norm trade calls such that offline ledger and accounting processing
clients can use a previously cached symbology set without having to
necessarily start the async-actor runtime to query the actual backend API
if the data has already been saved locally on the system B)

Other related:
- always passthrough kwargs in overridden `.to_dict()` method.
- only do fqme related trade record field name rewrites/names when
  operating on a paper ledger; normally a backend's records don't
  contain these.
- fix `pendulum.DateTime` type annots.
- just deliver `Transaction`s from `.iter_txns()`
account_tests
Tyler Goodlet 2023-07-14 14:11:49 -04:00
parent da206f5242
commit 494b3faa9b
1 changed files with 78 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
piker/accounting/_ledger.py
View File

@ -32,9 +32,7 @@ from typing import (
)
from pendulum import (
datetime,
# DateTime,
# parse,
DateTime,
)
import tomli_w # for fast ledger writing
@ -70,7 +68,7 @@ class Transaction(Struct, frozen=True):
size: float
price: float
cost: float # commisions or other additional costs
dt: datetime
dt: DateTime
# the "event type" in terms of "market events" see
# https://github.com/pikers/piker/issues/510 for where we're
@ -80,7 +78,7 @@ class Transaction(Struct, frozen=True):
# TODO: we can drop this right since we
# can instead expect the backend to provide this
# via the `MktPair`?
expiry: datetime | None = None
expiry: DateTime | None = None
# (optional) key-id defined by the broker-service backend which
# ensures the instrument-symbol market key for this record is unique
@ -89,8 +87,11 @@ class Transaction(Struct, frozen=True):
# service.
bs_mktid: str | int | None = None
def to_dict(self) -> dict:
dct: dict[str, Any] = super().to_dict()
def to_dict(
self,
**kwargs,
) -> dict:
dct: dict[str, Any] = super().to_dict(**kwargs)
# ensure we use a pendulum formatted
# ISO style str here!@
@ -107,6 +108,8 @@ class TransactionLedger(UserDict):
outside.
'''
# NOTE: see `open_trade_ledger()` for defaults, this should
# never be constructed manually!
def __init__(
self,
ledger_dict: dict,
@ -138,6 +141,10 @@ class TransactionLedger(UserDict):
@property
def symcache(self) -> SymbologyCache:
'''
Read-only ref to backend's ``SymbologyCache``.
'''
return self._symcache
def update_from_t(
@ -157,7 +164,7 @@ class TransactionLedger(UserDict):
symcache: SymbologyCache | None = None,
) -> Generator[
tuple[str, Transaction],
Transaction,
None,
None,
]:
@ -175,9 +182,13 @@ class TransactionLedger(UserDict):
norm_trade = self.mod.norm_trade
# datetime-sort and pack into txs
for txdict in self.tx_sort(self.data.values()):
txn = norm_trade(txdict)
yield txn.tid, txn
for tid, txdict in self.tx_sort(self.data.items()):
txn: Transaction = norm_trade(
tid,
txdict,
pairs=symcache.pairs,
)
yield txn
def to_txns(
self,
@ -188,18 +199,19 @@ class TransactionLedger(UserDict):
Return entire output from ``.iter_txns()`` in a ``dict``.
'''
return dict(self.iter_txns(symcache=symcache))
return {
t.tid: t for t in self.iter_txns(symcache=symcache)
}
def write_config(
self,
) -> None:
def write_config(self) -> None:
'''
Render the self.data ledger dict to its TOML file form.
ALWAYS order datetime sorted!
'''
is_paper: bool = self.account == 'paper'
towrite: dict[str, Any] = {}
for tid, txdict in self.tx_sort(self.data.copy()):
# write blank-str expiry for non-expiring assets
@ -210,42 +222,44 @@ class TransactionLedger(UserDict):
txdict['expiry'] = ''
# (maybe) re-write old acro-key
fqme: str = txdict.pop('fqsn', None) or txdict['fqme']
bs_mktid: str | None = txdict.get('bs_mktid')
if is_paper:
fqme: str = txdict.pop('fqsn', None) or txdict['fqme']
bs_mktid: str | None = txdict.get('bs_mktid')
if (
fqme not in self._symcache.mktmaps
or (
# also try to see if this is maybe a paper
# engine ledger in which case the bs_mktid
# should be the fqme as well!
self.account == 'paper'
and bs_mktid
and fqme != bs_mktid
)
):
# always take any (paper) bs_mktid if defined and
# in the backend's cache key set.
if bs_mktid in self._symcache.mktmaps:
fqme: str = bs_mktid
else:
best_fqme: str = list(self._symcache.search(fqme))[0]
log.warning(
f'Could not find FQME: {fqme} in qualified set?\n'
f'Qualifying and expanding {fqme} -> {best_fqme}'
if (
fqme not in self._symcache.mktmaps
or (
# also try to see if this is maybe a paper
# engine ledger in which case the bs_mktid
# should be the fqme as well!
bs_mktid
and fqme != bs_mktid
)
fqme = best_fqme
):
# always take any (paper) bs_mktid if defined and
# in the backend's cache key set.
if bs_mktid in self._symcache.mktmaps:
fqme: str = bs_mktid
else:
best_fqme: str = list(self._symcache.search(fqme))[0]
log.warning(
f'Could not find FQME: {fqme} in qualified set?\n'
f'Qualifying and expanding {fqme} -> {best_fqme}'
)
fqme = best_fqme
if (
self.account == 'paper'
and bs_mktid
and bs_mktid != fqme
):
# in paper account case always make sure both the
# fqme and bs_mktid are fully qualified..
txdict['bs_mktid'] = fqme
if (
bs_mktid
and bs_mktid != fqme
):
# in paper account case always make sure both the
# fqme and bs_mktid are fully qualified..
txdict['bs_mktid'] = fqme
# in paper ledgers always write the latest
# symbology key field: an FQME.
txdict['fqme'] = fqme
txdict['fqme'] = fqme
towrite[tid] = txdict
with self.file_path.open(mode='wb') as fp:
@ -256,6 +270,9 @@ def load_ledger(
brokername: str,
acctid: str,
# for testing or manual load from file
dirpath: Path | None = None,
) -> tuple[dict, Path]:
'''
Load a ledger (TOML) file from user's config directory:
@ -270,7 +287,11 @@ def load_ledger(
except ModuleNotFoundError:
import tomli as tomllib
ldir: Path = config._config_dir / 'accounting' / 'ledgers'
ldir: Path = (
dirpath
or
config._config_dir / 'accounting' / 'ledgers'
)
if not ldir.is_dir():
ldir.mkdir()
@ -303,6 +324,9 @@ def open_trade_ledger(
tx_sort: Callable = iter_by_dt,
rewrite: bool = False,
# for testing or manual load from file
_fp: Path | None = None,
) -> Generator[TransactionLedger, None, None]:
'''
Indempotently create and read in a trade log file from the
@ -316,7 +340,11 @@ def open_trade_ledger(
from ..brokers import get_brokermod
mod: ModuleType = get_brokermod(broker)
ledger_dict, fpath = load_ledger(broker, account)
ledger_dict, fpath = load_ledger(
broker,
account,
dirpath=_fp,
)
cpy = ledger_dict.copy()
# XXX NOTE: if not provided presume we are being called from