''' Deterministic history-backfill regressions. ''' from functools import partial from types import SimpleNamespace import numpy as np from pendulum import ( datetime, from_timestamp, ) import pytest import trio from trio import TaskStatus from piker.brokers import ( DataUnavailable, NoData, ) from piker.data._source import def_iohlcv_fields from piker.tsp._history import ( maybe_fill_null_segments, notify_backfill, publish_latest_frame, start_backfill, ) def run_exhausted_backfill(get_hist) -> None: ''' Run a provider-exhaustion case without actor services. ''' async def main() -> None: with trio.fail_after(0.5): async with trio.open_nursery() as nursery: await nursery.start(partial( start_backfill, get_hist=get_hist, def_frame_duration=None, mod=SimpleNamespace(name='fake'), mkt=SimpleNamespace(fqme='x.fake'), shm=object(), timeframe=60, backfill_from_shm_index=100, backfill_from_dt=datetime(2026, 1, 2), sampler_stream=object(), backfill_until_dt=datetime(2026, 1, 1), storage=None, write_tsdb=False, )) trio.run(main) def test_data_unavailable_completes() -> None: ''' Provider exhaustion exits without orphaned completion waits. ''' async def get_hist(*args, **kwargs): raise DataUnavailable('history exhausted') run_exhausted_backfill(get_hist) @pytest.mark.parametrize('error_type', [ NoData, DataUnavailable, ]) def test_gap_repair_failure_preserves_sibling( error_type: type[Exception], ) -> None: ''' An unavailable gap query must not cancel sibling feed tasks. Use real null detection and a provider that raises on its first query. After repair returns, release a sibling waiting in the same nursery and require its acknowledgement. This checks local failure containment, not Gateway reset or quote reconnection. ''' frame: np.ndarray = np.zeros( 5, dtype=np.dtype(def_iohlcv_fields), ) frame['index'] = np.arange(5) frame['time'] = datetime(2026, 11, 6).timestamp() + np.arange(5) frame['time'][1:3] = 0 frame['close'][0] = 100 calls: list[dict] = [] async def get_hist(timeframe: float, **kwargs) -> None: assert timeframe == 1 assert kwargs['start_dt'] < kwargs['end_dt'] calls.append(kwargs) raise error_type('provider cannot repair gap') async def main() -> None: repaired: trio.Event = trio.Event() resume: trio.Event = trio.Event() survived: trio.Event = trio.Event() send, receive = trio.open_memory_channel(1) async def sibling( task_status: TaskStatus[None] = trio.TASK_STATUS_IGNORED, ) -> None: task_status.started() await resume.wait() survived.set() async def repair() -> None: await maybe_fill_null_segments( shm=SimpleNamespace(array=frame, _array=frame), timeframe=1, get_hist=get_hist, sampler_stream=send, mkt=SimpleNamespace(fqme='mnq.cme.fake'), backfill_until_dt=datetime(2026, 11, 5), ) repaired.set() with trio.fail_after(1): async with trio.open_nursery() as nursery: await nursery.start(sibling) nursery.start_soon(repair) await repaired.wait() resume.set() await survived.wait() nursery.cancel_scope.cancel() assert len(calls) == 1 assert receive.receive_nowait() == { 'broadcast_all': { 'backfilling': ('mnq.cme.fake', 1), }, } assert (frame['time'] > 0).all() trio.run(main) def test_empty_frame_completes() -> None: ''' An empty provider frame is checked before endpoint indexing. ''' frame = np.empty( 0, dtype=np.dtype(def_iohlcv_fields), ) async def get_hist(*args, **kwargs): end_dt = kwargs['end_dt'] return frame, end_dt, end_dt run_exhausted_backfill(get_hist) def test_storage_receives_full_provider_delta() -> None: ''' SHM capacity truncation does not truncate durable history. ''' frame = np.zeros( 2, dtype=np.dtype(def_iohlcv_fields), ) frame['index'] = [0, 1] frame['time'] = [60, 120] events: list[str] = [] class Shm: def __init__(self) -> None: self.pushed: list[np.ndarray] = [] def push(self, array, **kwargs) -> None: self.pushed.append(array.copy()) events.append('shm') class Sampler: async def send(self, msg) -> None: events.append('sampler') class Storage: def __init__(self) -> None: self.frames: list[np.ndarray] = [] async def update_ohlcv( self, fqme: str, ohlcv: np.ndarray, timeframe: int, ) -> None: self.frames.append(ohlcv.copy()) events.append('storage') shm = Shm() storage = Storage() mkt = SimpleNamespace( fqme='x.test', dst=SimpleNamespace(atype='crypto'), src=SimpleNamespace(atype='crypto_currency'), get_fqme=lambda **kwargs: 'x.test', ) async def get_hist(*args, **kwargs): return ( frame, from_timestamp(60), from_timestamp(120), ) async def main() -> None: with trio.fail_after(0.5): await start_backfill( get_hist=get_hist, def_frame_duration=None, mod=SimpleNamespace(name='fake'), mkt=mkt, shm=shm, timeframe=60, backfill_from_shm_index=1, backfill_from_dt=from_timestamp(180), sampler_stream=Sampler(), backfill_until_dt=from_timestamp(60), storage=storage, write_tsdb=True, ) trio.run(main) assert shm.pushed[0]['time'].tolist() == [120] assert storage.frames[0]['time'].tolist() == [60, 120] assert events == ['storage', 'shm', 'sampler'] def test_latest_frame_is_persisted_before_shm() -> None: ''' Startup persists the most-recent provider frame before publication. ''' frame = np.zeros( 2, dtype=np.dtype(def_iohlcv_fields), ) frame['time'] = [60, 120] events: list[str] = [] class Storage: async def update_ohlcv(self, *args) -> None: events.append('storage') class Shm: def push(self, array, **kwargs) -> None: events.append('shm') async def main() -> None: with trio.fail_after(0.5): await publish_latest_frame( storage=Storage(), mkt=SimpleNamespace( fqme='x.test', dst=SimpleNamespace(atype='crypto'), src=SimpleNamespace(atype='crypto_currency'), get_fqme=lambda **kwargs: 'x.test', ), shm=Shm(), array=frame, timeframe=60, ) trio.run(main) assert events == ['storage', 'shm'] def test_backfill_notification_timeout_is_bounded( monkeypatch: pytest.MonkeyPatch, ) -> None: ''' A wedged sampler can not indefinitely shield actor teardown. ''' monkeypatch.setattr( 'piker.tsp._history._notify_timeout_s', 0.01, ) class Sampler: async def send(self, msg) -> None: await trio.sleep_forever() async def main() -> None: with trio.fail_after(0.1): await notify_backfill( Sampler(), SimpleNamespace(fqme='x.test'), 60, ) trio.run(main)