ib-related: cope with invalid txn timestamps

That is inside embedded `.accounting.calc.dyn_parse_to_dt()` closure add
an optional `_invalid: list` param to where we can report
bad-timestamped records which we instead override and return as
`from_timestamp(0.)` (when the parser loop falls through) and report
later (in summary ) from the `.accounting.calc.iter_by_dt()` caller. Add
some logging and an optional debug block for future tracing.

NOTE, this commit was re-edited during a conflict between the orig
branches: `dev/binance_api_3.1` & `dev/alt_tpts_for_perf`.
alt_tpts_for_perf
Tyler Goodlet 2025-09-15 18:29:19 -04:00
parent fc2d727fdb
commit cffefac615
1 changed files with 60 additions and 22 deletions

View File

@ -22,7 +22,9 @@ you know when you're losing money (if possible) XD
from __future__ import annotations
from collections.abc import ValuesView
from contextlib import contextmanager as cm
from functools import partial
from math import copysign
from pprint import pformat
from typing import (
Any,
Callable,
@ -37,12 +39,16 @@ from pendulum import (
parse,
)
from ..log import get_logger
if TYPE_CHECKING:
from ._ledger import (
Transaction,
TransactionLedger,
)
log = get_logger(__name__)
def ppu(
clears: Iterator[Transaction],
@ -238,6 +244,9 @@ def iter_by_dt(
def dyn_parse_to_dt(
tx: tuple[str, dict[str, Any]] | Transaction,
debug: bool = False,
_invalid: list|None = None,
) -> DateTime:
# handle `.items()` inputs
@ -250,52 +259,81 @@ def iter_by_dt(
# get best parser for this record..
for k in parsers:
if (
isdict and k in tx
(v := getattr(tx, k, None))
or
getattr(tx, k, None)
(
isdict
and
(v := tx.get(k))
)
):
v = (
tx[k] if isdict
else tx.dt
)
assert v is not None, (
f'No valid value for `{k}`!?'
)
# only call parser on the value if not None from
# the `parsers` table above (when NOT using
# `.get()`), otherwise pass through the value and
# sort on it directly
if (
not isinstance(v, DateTime)
and (parser := parsers.get(k))
and
(parser := parsers.get(k))
):
return parser(v)
ret = parser(v)
else:
return v
ret = v
return ret
else:
log.debug(
f'Parser-field not found in txn\n'
f'\n'
f'parser-field: {k!r}\n'
f'txn: {tx!r}\n'
f'\n'
f'Trying next..\n'
)
continue
# XXX: should never get here..
else:
# TODO: move to top?
from piker.log import get_logger
log = get_logger(__name__)
# XXX: we should really never get here..
# only if a ledger record has no expected sort(able)
# field will we likely hit this.. like with ze IB.
# if no sortable field just deliver epoch?
log.warning(
'No (time) sortable field for TXN:\n'
f'{tx}\n'
f'{tx!r}\n'
)
return from_timestamp(0)
# breakpoint()
if debug:
import tractor
with tractor.devx.maybe_open_crash_handler():
raise ValueError(
f'No supported time-field found in txn !?\n'
f'\n'
f'supported-time-fields: {parsers!r}\n'
f'\n'
f'txn: {tx!r}\n'
)
if _invalid is not None:
_invalid.append(tx)
return from_timestamp(0.)
entry: tuple[str, dict] | Transaction
entry: tuple[str, dict]|Transaction
invalid: list = []
for entry in sorted(
records,
key=key or dyn_parse_to_dt,
key=key or partial(
dyn_parse_to_dt,
_invalid=invalid,
),
):
if entry in invalid:
log.warning(
f'Ignoring txn w invalid timestamp ??\n'
f'{pformat(entry)}\n'
)
continue
# NOTE the type sig above; either pairs or txns B)
yield entry