Finally, just drop `Transaction.sym`

Turns out we don't really need it directly for most "txn processing" AND
if we do it's usually related to some `Account`-ing related calcs; which
means we can instead just rely on the new `SymbologyCache` lookup to get
it when needed. So, basically just get rid of it and rely instead on the
`.fqme` to be the god-key to getting `MktPair` info (from the cache).

Further, extend the `TransactionLedger` to contain much more info on the
pertaining backend:
- `.mod` mapping to the (pkg) py mod.
- `.filepath` pointing to the actual ledger TOML file.
- `_symcache` for doing any needed asset or mkt lookup as mentioned
  above.
- rename `.iter_trans()` -> `.iter_txns()` and allow passing in
  a symcache or using the init provided one.
  - rename `.to_trans()` similarly.
- delegate paper account txn processing to the `.clearing._paper_engine`
  mod's `norm_trade()` (and expect this similarly from other backends!)
- use new `SymbologyCache.search()` to find the best but
  un-fully-qualified fqme for a given `txdict` being processed when
  writing a config (aka always try to expand to the most verbose `.fqme`
  possible).
- add a `rewrite: bool` control to `open_trade_ledger()`.

piker/accounting/_ledger.py

@@ -22,16 +22,19 @@ from __future__ import annotations
 from collections import UserDict
 from contextlib import contextmanager as cm
 from pathlib import Path
+from pprint import pformat
+from types import ModuleType
 from typing import (
     Any,
     Callable,
-    Generator
+    Generator,
+    TYPE_CHECKING,
 )
 
 from pendulum import (
     datetime,
-    DateTime,
+    # DateTime,
-    parse,
+    # parse,
 )
 import tomli_w  # for fast ledger writing
 
@@ -41,27 +44,37 @@ from ..log import get_logger
 from .calc import (
     iter_by_dt,
 )
-from ._mktinfo import (
+
-    Symbol,  # legacy
+if TYPE_CHECKING:
-    MktPair,
+    from ..data._symcache import (
-    Asset,
+        SymbologyCache,
-)
+    )
 
 log = get_logger(__name__)
 
 
 class Transaction(Struct, frozen=True):
 
-    # TODO: unify this with the `MktPair`,
+    # NOTE: this is a unified acronym also used in our `MktPair`
-    # once we have that as a required field,
+    # and can stand for any of a
-    # we don't really need the fqme any more..
+    # "fully qualified <blank> endpoint":
+    # - "market" in the case of financial trades
+    #   (btcusdt.spot.binance).
+    # - "merkel (tree)" aka a blockchain system "wallet tranfers"
+    #   (btc.blockchain)
+    # - "money" for tradtitional (digital databases)
+    #   *bank accounts* (usd.swift, eur.sepa)
     fqme: str
+
     tid: str | int  # unique transaction id
     size: float
     price: float
     cost: float  # commisions or other additional costs
     dt: datetime
 
+    # the "event type" in terms of "market events" see
+    # https://github.com/pikers/piker/issues/510 for where we're
+    # probably going with this.
     etype: str = 'clear'
 
     # TODO: we can drop this right since we
@@ -69,19 +82,6 @@ class Transaction(Struct, frozen=True):
     # via the `MktPair`?
     expiry: datetime | None = None
 
-    # TODO: drop the Symbol type, construct using
-    # t.sys (the transaction system)
-
-    # the underlying "transaction system", normally one of a ``MktPair``
-    # (a description of a tradable double auction) or a ledger-recorded
-    # ("ledger" in any sense as long as you can record transfers) of any
-    # sort) ``Asset``.
-    sym: MktPair | Asset | Symbol | None = None
-
-    @property
-    def sys(self) -> Symbol:
-        return self.sym
-
     # (optional) key-id defined by the broker-service backend which
     # ensures the instrument-symbol market key for this record is unique
     # in the "their backend/system" sense; i.e. this uid for the market
@@ -90,14 +90,12 @@ class Transaction(Struct, frozen=True):
     bs_mktid: str | int | None = None
 
     def to_dict(self) -> dict:
-        dct = super().to_dict()
+        dct: dict[str, Any] = super().to_dict()
-
-        # TODO: switch to sys!
-        dct.pop('sym')
 
         # ensure we use a pendulum formatted
         # ISO style str here!@
         dct['dt'] = str(self.dt)
+
         return dct
 
 
@@ -113,13 +111,35 @@ class TransactionLedger(UserDict):
         self,
         ledger_dict: dict,
         file_path: Path,
+        account: str,
+        mod: ModuleType,  # broker mod
         tx_sort: Callable,
+        symcache: SymbologyCache,
 
     ) -> None:
-        self.file_path = file_path
+        self.account: str = account
-        self.tx_sort = tx_sort
+        self.file_path: Path = file_path
+        self.mod: ModuleType = mod
+        self.tx_sort: Callable = tx_sort
+
+        self._symcache: SymbologyCache = symcache
+
+        # any added txns we keep in that form for meta-data
+        # gathering purposes
+        self._txns: dict[str, Transaction] = {}
+
         super().__init__(ledger_dict)
 
+    def __repr__(self) -> str:
+        return (
+            f'TransactionLedger: {len(self)}\n'
+            f'{pformat(list(self.data))}'
+        )
+
+    @property
+    def symcache(self) -> SymbologyCache:
+        return self._symcache
+
     def update_from_t(
         self,
         t: Transaction,
@@ -130,11 +150,11 @@ class TransactionLedger(UserDict):
 
         '''
         self.data[t.tid] = t.to_dict()
+        self._txns[t.tid] = t
 
-    def iter_trans(
+    def iter_txns(
         self,
-        mkt_by_fqme: dict[str, MktPair],
+        symcache: SymbologyCache | None = None,
-        broker: str = 'paper',
 
     ) -> Generator[
         tuple[str, Transaction],
@@ -146,73 +166,42 @@ class TransactionLedger(UserDict):
         form via generator.
 
         '''
-        if broker != 'paper':
+        symcache = symcache or self._symcache
-            raise NotImplementedError('Per broker support not dun yet!')
 
-            # TODO: lookup some standard normalizer
+        if self.account == 'paper':
-            # func in the backend?
+            from piker.clearing import _paper_engine
-            # from ..brokers import get_brokermod
+            norm_trade = _paper_engine.norm_trade
-            # mod = get_brokermod(broker)
+        else:
-            # trans_dict = mod.norm_trade_records(self.data)
+            norm_trade = self.mod.norm_trade
-
-            # NOTE: instead i propose the normalizer is
-            # a one shot routine (that can be lru cached)
-            # and instead call it for each entry incrementally:
-            # normer = mod.norm_trade_record(txdict)
 
         # datetime-sort and pack into txs
         for txdict in self.tx_sort(self.data.values()):
+            txn = norm_trade(txdict)
+            yield txn.tid, txn
 
-            # special field handling for datetimes
+    def to_txns(
-            # to ensure pendulum is used!
-            tid: str = txdict['tid']
-            fqme: str = txdict.get('fqme') or txdict['fqsn']
-            dt: DateTime = parse(txdict['dt'])
-            expiry: str | None = txdict.get('expiry')
-
-            if not (mkt := mkt_by_fqme.get(fqme)):
-                # we can't build a trans if we don't have
-                # the ``.sys: MktPair`` info, so skip.
-                continue
-
-            tx = Transaction(
-                fqme=fqme,
-                tid=txdict['tid'],
-                dt=dt,
-                price=txdict['price'],
-                size=txdict['size'],
-                cost=txdict.get('cost', 0),
-                bs_mktid=txdict['bs_mktid'],
-
-                # TODO: change to .sys!
-                sym=mkt,
-                expiry=parse(expiry) if expiry else None,
-                etype='clear',
-            )
-            yield tid, tx
-
-    def to_trans(
         self,
-        **kwargs,
+        symcache: SymbologyCache | None = None,
 
     ) -> dict[str, Transaction]:
         '''
-        Return entire output from ``.iter_trans()`` in a ``dict``.
+        Return entire output from ``.iter_txns()`` in a ``dict``.
 
         '''
-        return dict(self.iter_trans(**kwargs))
+        return dict(self.iter_txns(symcache=symcache))
 
     def write_config(
         self,
 
     ) -> None:
         '''
-        Render the self.data ledger dict to it's TOML file form.
+        Render the self.data ledger dict to its TOML file form.
+
+        ALWAYS order datetime sorted!
 
         '''
         towrite: dict[str, Any] = {}
         for tid, txdict in self.tx_sort(self.data.copy()):
-
             # write blank-str expiry for non-expiring assets
             if (
                 'expiry' in txdict
@@ -221,9 +210,18 @@ class TransactionLedger(UserDict):
                 txdict['expiry'] = ''
 
             # re-write old acro-key
-            if fqme := txdict.get('fqsn'):
-                txdict['fqme'] = fqme
 
+            fqme: str = txdict.pop('fqsn', None) or txdict['fqme']
+
+            if fqme not in self._symcache.mktmaps:
+                best_fqme: str = list(self._symcache.search(fqme))[0]
+                log.warning(
+                    f'Could not find FQME: {fqme} in qualified set?\n'
+                    f'Qualifying and expanding {fqme} -> {best_fqme}'
+                )
+                fqme = best_fqme
+
+            txdict['fqme'] = fqme
             towrite[tid] = txdict
 
         with self.file_path.open(mode='wb') as fp:
@@ -276,6 +274,7 @@ def open_trade_ledger(
 
     # default is to sort by detected datetime-ish field
     tx_sort: Callable = iter_by_dt,
+    rewrite: bool = False,
 
 ) -> Generator[TransactionLedger, None, None]:
     '''
@@ -287,18 +286,31 @@ def open_trade_ledger(
     name as defined in the user's ``brokers.toml`` config.
 
     '''
+    from ..brokers import get_brokermod
+    mod: ModuleType = get_brokermod(broker)
+
     ledger_dict, fpath = load_ledger(broker, account)
     cpy = ledger_dict.copy()
+
+    from ..data._symcache import (
+        get_symcache,
+    )
+    symcache: SymbologyCache = get_symcache(broker)
     ledger = TransactionLedger(
         ledger_dict=cpy,
         file_path=fpath,
+        account=account,
+        mod=mod,
+        symcache=symcache,
         tx_sort=tx_sort,
     )
     try:
         yield ledger
     finally:
-        if ledger.data != ledger_dict:
+        if (
-
+            ledger.data != ledger_dict
+            or rewrite
+        ):
             # TODO: show diff output?
             # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries
             log.info(f'Updating ledger for {fpath}:\n')