ib-related: cope with invalid txn timestamps

That is inside embedded `.accounting.calc.dyn_parse_to_dt()` closure add
an optional `_invalid: list` param to where we can report
bad-timestamped records which we instead override and return as
`from_timestamp(0.)` (when the parser loop falls through) and report
later (in summary ) from the `.accounting.calc.iter_by_dt()` caller. Add
some logging and an optional debug block for future tracing.

NOTE, this commit was re-edited during a conflict between the orig
branches: `dev/binance_api_3.1` & `dev/alt_tpts_for_perf`.
how_to_show_ur_pp
Tyler Goodlet 2025-09-15 18:29:19 -04:00
parent 2ca50348ce
commit b6d70d5012
1 changed files with 60 additions and 22 deletions

View File

@ -22,7 +22,9 @@ you know when you're losing money (if possible) XD
from __future__ import annotations from __future__ import annotations
from collections.abc import ValuesView from collections.abc import ValuesView
from contextlib import contextmanager as cm from contextlib import contextmanager as cm
from functools import partial
from math import copysign from math import copysign
from pprint import pformat
from typing import ( from typing import (
Any, Any,
Callable, Callable,
@ -37,12 +39,16 @@ from pendulum import (
parse, parse,
) )
from ..log import get_logger
if TYPE_CHECKING: if TYPE_CHECKING:
from ._ledger import ( from ._ledger import (
Transaction, Transaction,
TransactionLedger, TransactionLedger,
) )
log = get_logger(__name__)
def ppu( def ppu(
clears: Iterator[Transaction], clears: Iterator[Transaction],
@ -238,6 +244,9 @@ def iter_by_dt(
def dyn_parse_to_dt( def dyn_parse_to_dt(
tx: tuple[str, dict[str, Any]] | Transaction, tx: tuple[str, dict[str, Any]] | Transaction,
debug: bool = False,
_invalid: list|None = None,
) -> DateTime: ) -> DateTime:
# handle `.items()` inputs # handle `.items()` inputs
@ -250,52 +259,81 @@ def iter_by_dt(
# get best parser for this record.. # get best parser for this record..
for k in parsers: for k in parsers:
if ( if (
isdict and k in tx (v := getattr(tx, k, None))
or or
getattr(tx, k, None) (
isdict
and
(v := tx.get(k))
)
): ):
v = (
tx[k] if isdict
else tx.dt
)
assert v is not None, (
f'No valid value for `{k}`!?'
)
# only call parser on the value if not None from # only call parser on the value if not None from
# the `parsers` table above (when NOT using # the `parsers` table above (when NOT using
# `.get()`), otherwise pass through the value and # `.get()`), otherwise pass through the value and
# sort on it directly # sort on it directly
if ( if (
not isinstance(v, DateTime) not isinstance(v, DateTime)
and (parser := parsers.get(k)) and
(parser := parsers.get(k))
): ):
return parser(v) ret = parser(v)
else: else:
return v ret = v
return ret
else:
log.debug(
f'Parser-field not found in txn\n'
f'\n'
f'parser-field: {k!r}\n'
f'txn: {tx!r}\n'
f'\n'
f'Trying next..\n'
)
continue
# XXX: should never get here..
else: else:
# TODO: move to top?
from piker.log import get_logger
log = get_logger(__name__)
# XXX: we should really never get here.. # XXX: we should really never get here..
# only if a ledger record has no expected sort(able) # only if a ledger record has no expected sort(able)
# field will we likely hit this.. like with ze IB. # field will we likely hit this.. like with ze IB.
# if no sortable field just deliver epoch? # if no sortable field just deliver epoch?
log.warning( log.warning(
'No (time) sortable field for TXN:\n' 'No (time) sortable field for TXN:\n'
f'{tx}\n' f'{tx!r}\n'
) )
return from_timestamp(0) if debug:
# breakpoint() import tractor
with tractor.devx.maybe_open_crash_handler():
raise ValueError(
f'No supported time-field found in txn !?\n'
f'\n'
f'supported-time-fields: {parsers!r}\n'
f'\n'
f'txn: {tx!r}\n'
)
if _invalid is not None:
_invalid.append(tx)
return from_timestamp(0.)
entry: tuple[str, dict] | Transaction entry: tuple[str, dict]|Transaction
invalid: list = []
for entry in sorted( for entry in sorted(
records, records,
key=key or dyn_parse_to_dt, key=key or partial(
dyn_parse_to_dt,
_invalid=invalid,
),
): ):
if entry in invalid:
log.warning(
f'Ignoring txn w invalid timestamp ??\n'
f'{pformat(entry)}\n'
)
continue
# NOTE the type sig above; either pairs or txns B) # NOTE the type sig above; either pairs or txns B)
yield entry yield entry