ib-related: cope with invalid txn timestamps
That is inside embedded `.accounting.calc.dyn_parse_to_dt()` closure add an optional `_invalid: list` param to where we can report bad-timestamped records which we instead override and return as `from_timestamp(0.)` (when the parser loop falls through) and report later (in summary ) from the `.accounting.calc.iter_by_dt()` caller . Add some logging and an optional debug block for future tracing.alt_tpts_for_perf
parent
9df1988aa6
commit
a45de0b710
|
@ -22,7 +22,9 @@ you know when you're losing money (if possible) XD
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
from collections.abc import ValuesView
|
from collections.abc import ValuesView
|
||||||
from contextlib import contextmanager as cm
|
from contextlib import contextmanager as cm
|
||||||
|
from functools import partial
|
||||||
from math import copysign
|
from math import copysign
|
||||||
|
from pprint import pformat
|
||||||
from typing import (
|
from typing import (
|
||||||
Any,
|
Any,
|
||||||
Callable,
|
Callable,
|
||||||
|
@ -37,12 +39,16 @@ from pendulum import (
|
||||||
parse,
|
parse,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
from ..log import get_logger
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from ._ledger import (
|
from ._ledger import (
|
||||||
Transaction,
|
Transaction,
|
||||||
TransactionLedger,
|
TransactionLedger,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
log = get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def ppu(
|
def ppu(
|
||||||
clears: Iterator[Transaction],
|
clears: Iterator[Transaction],
|
||||||
|
@ -238,6 +244,9 @@ def iter_by_dt(
|
||||||
|
|
||||||
def dyn_parse_to_dt(
|
def dyn_parse_to_dt(
|
||||||
tx: tuple[str, dict[str, Any]] | Transaction,
|
tx: tuple[str, dict[str, Any]] | Transaction,
|
||||||
|
|
||||||
|
debug: bool = False,
|
||||||
|
_invalid: list|None = None,
|
||||||
) -> DateTime:
|
) -> DateTime:
|
||||||
|
|
||||||
# handle `.items()` inputs
|
# handle `.items()` inputs
|
||||||
|
@ -250,11 +259,16 @@ def iter_by_dt(
|
||||||
# get best parser for this record..
|
# get best parser for this record..
|
||||||
for k in parsers:
|
for k in parsers:
|
||||||
if (
|
if (
|
||||||
isdict and k in tx
|
(v := getattr(tx, k, None))
|
||||||
or getattr(tx, k, None)
|
or
|
||||||
|
(
|
||||||
|
isdict
|
||||||
|
and
|
||||||
|
(v := tx.get(k))
|
||||||
|
)
|
||||||
):
|
):
|
||||||
v = tx[k] if isdict else tx.dt
|
# TODO? remove yah?
|
||||||
assert v is not None, f'No valid value for `{k}`!?'
|
# v = tx[k] if isdict else tx.dt
|
||||||
|
|
||||||
# only call parser on the value if not None from
|
# only call parser on the value if not None from
|
||||||
# the `parsers` table above (when NOT using
|
# the `parsers` table above (when NOT using
|
||||||
|
@ -262,21 +276,54 @@ def iter_by_dt(
|
||||||
# sort on it directly
|
# sort on it directly
|
||||||
if (
|
if (
|
||||||
not isinstance(v, DateTime)
|
not isinstance(v, DateTime)
|
||||||
and (parser := parsers.get(k))
|
and
|
||||||
|
(parser := parsers.get(k))
|
||||||
):
|
):
|
||||||
return parser(v)
|
ret = parser(v)
|
||||||
else:
|
else:
|
||||||
return v
|
ret = v
|
||||||
|
|
||||||
|
return ret
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
continue
|
||||||
|
|
||||||
# XXX: should never get here..
|
# XXX: should never get here..
|
||||||
breakpoint()
|
else:
|
||||||
|
if debug:
|
||||||
|
import tractor
|
||||||
|
with tractor.devx.maybe_open_crash_handler():
|
||||||
|
raise ValueError(
|
||||||
|
f'Invalid txn time ??\n'
|
||||||
|
f'txn-id: {k!r}\n'
|
||||||
|
f'{k!r}: {v!r}\n'
|
||||||
|
)
|
||||||
|
# assert v is not None, f'No valid value for `{k}`!?'
|
||||||
|
|
||||||
|
if _invalid is not None:
|
||||||
|
_invalid.append(tx)
|
||||||
|
return from_timestamp(0.)
|
||||||
|
|
||||||
|
# breakpoint()
|
||||||
|
|
||||||
entry: tuple[str, dict]|Transaction
|
entry: tuple[str, dict]|Transaction
|
||||||
|
invalid: list = []
|
||||||
for entry in sorted(
|
for entry in sorted(
|
||||||
records,
|
records,
|
||||||
key=key or dyn_parse_to_dt,
|
key=key or partial(
|
||||||
|
dyn_parse_to_dt,
|
||||||
|
_invalid=invalid,
|
||||||
|
),
|
||||||
):
|
):
|
||||||
|
if entry in invalid:
|
||||||
|
log.warning(
|
||||||
|
f'Ignoring txn w invalid timestamp ??\n'
|
||||||
|
f'{pformat(entry)}\n'
|
||||||
|
# f'txn-id: {k!r}\n'
|
||||||
|
# f'{k!r}: {v!r}\n'
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
# NOTE the type sig above; either pairs or txns B)
|
# NOTE the type sig above; either pairs or txns B)
|
||||||
yield entry
|
yield entry
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue