'''
Deterministic history-backfill regressions.

'''
from contextlib import asynccontextmanager
from functools import partial
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4

import numpy as np
from pendulum import (
    datetime,
    from_timestamp,
)
import polars as pl
import pytest
import tractor
import trio

from piker.brokers import DataUnavailable
from piker.brokers.ib.api import (
    _bar_load_dtype,
    _ohlc_dtype,
)
from piker.data._sharedmem import maybe_open_shm_array
from piker.data._source import def_iohlcv_fields
from piker.storage.nativedb import (
    NativeStorageClient,
    ohlc_key_map,
)
from piker.tsp._history import (
    _synthetic_gap_times,
    maybe_fill_null_segments,
    notify_backfill,
    publish_latest_frame,
    start_backfill,
    tsdb_backfill,
)


def run_exhausted_backfill(get_hist) -> None:
    '''
    Run a provider-exhaustion case without actor services.

    '''
    async def main() -> None:
        with trio.fail_after(0.5):
            async with trio.open_nursery() as nursery:
                await nursery.start(partial(
                    start_backfill,
                    get_hist=get_hist,
                    def_frame_duration=None,
                    mod=SimpleNamespace(name='fake'),
                    mkt=SimpleNamespace(fqme='x.fake'),
                    shm=object(),
                    timeframe=60,
                    backfill_from_shm_index=100,
                    backfill_from_dt=datetime(2026, 1, 2),
                    sampler_stream=object(),
                    backfill_until_dt=datetime(2026, 1, 1),
                    storage=None,
                    write_tsdb=False,
                ))

    trio.run(main)


def test_data_unavailable_completes() -> None:
    '''
    Provider exhaustion exits without orphaned completion waits.

    '''
    async def get_hist(*args, **kwargs):
        raise DataUnavailable('history exhausted')

    run_exhausted_backfill(get_hist)


def test_empty_frame_completes() -> None:
    '''
    An empty provider frame is checked before endpoint indexing.

    '''
    frame = np.empty(
        0,
        dtype=np.dtype(def_iohlcv_fields),
    )

    async def get_hist(*args, **kwargs):
        end_dt = kwargs['end_dt']
        return frame, end_dt, end_dt

    run_exhausted_backfill(get_hist)


def test_storage_receives_full_provider_delta() -> None:
    '''
    SHM drops the published endpoint without truncating storage.

    Reverse history queries include their requested end bar, which is
    already the oldest sample in SHM. Publishing the overlap consumed
    an extra physical row and made reserved null counts disagree with
    elapsed timestamps. Return a frame with the overlap and prove
    NativeDB receives the complete provider delta while SHM receives
    only bars older than its published boundary.

    '''
    frame = np.zeros(
        3,
        dtype=np.dtype(def_iohlcv_fields),
    )
    frame['index'] = [0, 1, 2]
    frame['time'] = [60, 120, 180]

    events: list[str] = []

    class Shm:
        def __init__(self) -> None:
            self.pushed: list[np.ndarray] = []

        def push(self, array, **kwargs) -> None:
            self.pushed.append(array.copy())
            events.append('shm')

    class Sampler:
        async def send(self, msg) -> None:
            events.append('sampler')

    class Storage:
        def __init__(self) -> None:
            self.frames: list[np.ndarray] = []

        async def update_ohlcv(
            self,
            fqme: str,
            ohlcv: np.ndarray,
            timeframe: int,
        ) -> None:
            self.frames.append(ohlcv.copy())
            events.append('storage')

    shm = Shm()
    storage = Storage()
    mkt = SimpleNamespace(
        fqme='x.test',
        dst=SimpleNamespace(atype='crypto'),
        src=SimpleNamespace(atype='crypto_currency'),
        get_fqme=lambda **kwargs: 'x.test',
    )

    async def get_hist(*args, **kwargs):
        return (
            frame,
            from_timestamp(60),
            from_timestamp(180),
        )

    async def main() -> None:
        with trio.fail_after(0.5):
            await start_backfill(
                get_hist=get_hist,
                def_frame_duration=None,
                mod=SimpleNamespace(name='fake'),
                mkt=mkt,
                shm=shm,
                timeframe=60,
                backfill_from_shm_index=2,
                backfill_from_dt=from_timestamp(180),
                sampler_stream=Sampler(),
                backfill_until_dt=from_timestamp(60),
                storage=storage,
                write_tsdb=True,
            )

    trio.run(main)
    assert shm.pushed[0]['time'].tolist() == [60, 120]
    assert storage.frames[0]['time'].tolist() == [60, 120, 180]
    assert events == ['storage', 'shm', 'sampler']


def test_latest_frame_is_persisted_before_shm() -> None:
    '''
    Startup persists the most-recent provider frame before publication.

    '''
    frame = np.zeros(
        2,
        dtype=np.dtype(def_iohlcv_fields),
    )
    frame['time'] = [60, 120]
    events: list[str] = []

    class Storage:
        async def update_ohlcv(self, *args) -> None:
            events.append('storage')

    class Shm:
        def push(self, array, **kwargs) -> None:
            events.append('shm')

    async def main() -> None:
        with trio.fail_after(0.5):
            await publish_latest_frame(
                storage=Storage(),
                mkt=SimpleNamespace(
                    fqme='x.test',
                    dst=SimpleNamespace(atype='crypto'),
                    src=SimpleNamespace(atype='crypto_currency'),
                    get_fqme=lambda **kwargs: 'x.test',
                ),
                shm=Shm(),
                array=frame,
                timeframe=60,
            )

    trio.run(main)
    assert events == ['storage', 'shm']


def test_tsdb_prepend_uses_actual_provider_boundary(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    '''
    Sparse venue history must not reserve wall-clock rows in SHM.

    NVDA's 60s startup converted elapsed calendar minutes between
    NativeDB and the latest IB frame into physical SHM offsets. IB
    returned only trading bars, leaving 42,720 zero rows even though
    both valid margins ended at the same timestamp. Model completed
    reverse retrieval with a provider boundary overlapping NativeDB.
    Exercise real ``ShmArray.push()`` indexing and prove the stored,
    reverse-provider, and latest frames form one contiguous lossless
    sequence without an explicit wall-clock offset.

    '''
    stored = np.zeros(
        3,
        dtype=np.dtype(def_iohlcv_fields),
    )
    stored['time'] = [60, 120, 180]
    older_reverse = np.zeros(
        3,
        dtype=np.dtype(def_iohlcv_fields),
    )
    older_reverse['time'] = [180, 240, 300]
    newer_reverse = np.zeros(
        3,
        dtype=np.dtype(def_iohlcv_fields),
    )
    newer_reverse['time'] = [300, 360, 420]
    latest = np.zeros(
        2,
        dtype=np.dtype(def_iohlcv_fields),
    )
    latest['time'] = [420, 480]
    storage_frames: list[np.ndarray] = []
    notifications: list[dict] = []
    startup_done = trio.Event()

    class Storage:
        '''
        Capture the complete provider delta written to NativeDB.

        '''
        async def update_ohlcv(
            self,
            fqme: str,
            ohlcv: np.ndarray,
            timeframe: int,
        ) -> None:
            '''
            Record one reverse provider response.

            '''
            storage_frames.append(ohlcv.copy())

        async def load(
            self,
            fqme: str,
            timeframe: int,
        ) -> tuple:
            '''
            Return overlapping NativeDB history and its boundaries.

            '''
            return (
                stored,
                from_timestamp(60),
                from_timestamp(180),
            )

    class Sampler:
        '''
        Accept deterministic backfill notifications.

        '''
        async def send(self, msg) -> None:
            '''
            Capture notification without actor IPC.

            '''
            notifications.append(msg)

    mkt = SimpleNamespace(
        fqme='nvda.nasdaq.ib',
        dst=SimpleNamespace(atype='stock'),
        src=SimpleNamespace(atype='fiat'),
        get_fqme=lambda **kwargs: 'nvda.nasdaq.ib',
    )

    async def get_hist(*args, **kwargs):
        '''
        Return inclusive reverse frames overlapping each SHM
        boundary.

        '''
        end_dt = kwargs['end_dt']
        if end_dt is None:
            return (
                latest,
                from_timestamp(420),
                from_timestamp(480),
            )

        end_t: float = end_dt.timestamp()
        reverse: np.ndarray = (
            newer_reverse
            if end_t == 420
            else older_reverse
        )
        return (
            reverse,
            from_timestamp(reverse['time'][0]),
            from_timestamp(reverse['time'][-1]),
        )

    @asynccontextmanager
    async def open_history_client(mkt):
        '''
        Yield the deterministic provider and default frame config.

        '''
        yield get_hist, {}

    async def finish_null_repair(**kwargs) -> None:
        '''
        Signal that provider and NativeDB prepends both completed.

        '''
        startup_done.set()

    monkeypatch.setattr(
        'piker.tsp._history.maybe_fill_null_segments',
        finish_null_repair,
    )

    async def main() -> None:
        shm_key = f'test_sparse_prepend_{uuid4().hex}'
        async with tractor.open_root_actor(
            name=shm_key,
            tpt_bind_addrs=[('127.0.0.1', 0)],
        ):
            shm, opened = maybe_open_shm_array(
                key=shm_key,
                size=16,
                dtype=np.dtype(def_iohlcv_fields),
                append_start_index=12,
            )
            assert opened
            storage = Storage()
            async with trio.open_nursery() as nursery:
                nursery.start_soon(partial(
                    tsdb_backfill,
                    mod=SimpleNamespace(
                        name='fake',
                        open_history_client=open_history_client,
                    ),
                    storemod=SimpleNamespace(
                        ohlc_key_map=ohlc_key_map,
                    ),
                    storage=storage,
                    mkt=mkt,
                    shm=shm,
                    timeframe=60,
                    sampler_stream=Sampler(),
                ))
                with trio.fail_after(0.5):
                    await startup_done.wait()
                nursery.cancel_scope.cancel()

            assert shm.array['time'].tolist() == [
                60,
                120,
                180,
                240,
                300,
                360,
                420,
                480,
            ]
            assert not np.any(shm.array['time'] <= 0)

    trio.run(main)
    assert len(storage_frames) == 3
    assert storage_frames[0]['time'].tolist() == [420, 480]
    assert storage_frames[1]['time'].tolist() == [300, 360, 420]
    assert storage_frames[2]['time'].tolist() == [180, 240, 300]
    assert len(notifications) == 3


def test_ib_latest_frame_round_trips_through_nativedb(
    tmp_path: Path,
) -> None:
    '''
    Persist and reload IB's provider schema through history startup.

    ``publish_latest_frame()`` previously passed IB's index-less bars
    directly to NativeDB, whose durable-schema validator rejected the
    missing derived ``index``. IB also appends ``count`` after OHLCV,
    which exposed positional Polars-to-NumPy conversion to field
    corruption. Build the actual IB load dtype with distinct values,
    run the real history publication, NativeDB Parquet, and ``ShmArray``
    paths, then hydrate a fresh IB buffer from storage. Assertions prove
    first-start publication preserves provider fields while restart maps
    canonical fields and leaves provider-only ``count`` at its default.

    '''
    frame = np.zeros(
        2,
        dtype=np.dtype(_bar_load_dtype),
    )
    frame['time'] = [60, 120]
    frame['open'] = [1.1, 2.1]
    frame['high'] = [1.2, 2.2]
    frame['low'] = [1.0, 2.0]
    frame['close'] = [1.15, 2.15]
    frame['volume'] = [10, 20]
    frame['count'] = [3, 4]

    storage = NativeStorageClient(tmp_path)
    shm_key = f'test_ib_history_{uuid4().hex}'
    mkt = SimpleNamespace(
        fqme='mnq.cme.20260918.ib',
        dst=SimpleNamespace(atype='continuous_future'),
        src=SimpleNamespace(atype='fiat'),
        get_fqme=lambda **kwargs: 'mnq.cme.20260918.ib',
    )

    async def main() -> None:
        with trio.fail_after(2):
            async with tractor.open_root_actor(
                name=shm_key,
                tpt_bind_addrs=[('127.0.0.1', 0)],
            ):
                shm, opened = maybe_open_shm_array(
                    key=f'{shm_key}_first',
                    size=16,
                    dtype=np.dtype(_ohlc_dtype),
                    append_start_index=8,
                )
                restart_shm, restart_opened = maybe_open_shm_array(
                    key=f'{shm_key}_restart',
                    size=16,
                    dtype=np.dtype(_ohlc_dtype),
                    append_start_index=8,
                )
                assert opened
                assert restart_opened

                await publish_latest_frame(
                    storage=storage,
                    mkt=mkt,
                    shm=shm,
                    array=frame,
                    timeframe=60,
                )
                loaded = await storage.read_ohlcv(
                    mkt.fqme,
                    timeframe=60,
                )
                restart_shm.push(
                    loaded,
                    prepend=True,
                    field_map=ohlc_key_map,
                )
                stored = pl.read_parquet(
                    storage.mk_path(mkt.fqme, 60)
                )
                canonical = [
                    name
                    for name, _ in def_iohlcv_fields
                ]
                canonical_schema = {
                    name: (
                        pl.Int64
                        if field_type is int
                        else pl.Float64
                    )
                    for name, field_type in def_iohlcv_fields
                }

                assert stored.columns == canonical
                assert dict(stored.schema) == canonical_schema
                assert list(loaded.dtype.fields) == canonical
                assert loaded['index'].tolist() == [0, 1]
                for field in canonical[1:]:
                    assert (
                        loaded[field].tolist()
                        ==
                        frame[field].tolist()
                    )
                    assert (
                        restart_shm.array[field].tolist()
                        ==
                        frame[field].tolist()
                    )
                assert shm.array['count'].tolist() == [3, 4]
                assert restart_shm.array['count'].tolist() == [0, 0]

    trio.run(main)


def test_backfill_notification_timeout_is_bounded(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    '''
    A wedged sampler can not indefinitely shield actor teardown.

    '''
    monkeypatch.setattr(
        'piker.tsp._history._notify_timeout_s',
        0.01,
    )

    class Sampler:
        async def send(self, msg) -> None:
            await trio.sleep_forever()

    async def main() -> None:
        with trio.fail_after(0.1):
            await notify_backfill(
                Sampler(),
                SimpleNamespace(fqme='x.test'),
                60,
            )

    trio.run(main)


def test_synthetic_gap_times_require_valid_right_boundary() -> None:
    '''
    Synthetic rows must remain strictly between valid boundary bars.

    The live QQQ reservation held one more physical zero row than its
    timestamp interval could represent. Blind fill would assign the
    null the right boundary's timestamp, creating a duplicate. Prove
    aligned bounds produce cadence while an overfull segment
    is rejected instead of inventing non-monotonic timestamps.

    '''
    aligned = _synthetic_gap_times(
        start_t=60,
        right_t=300,
        count=3,
        timeframe=60,
    )
    overfull = _synthetic_gap_times(
        start_t=60,
        right_t=240,
        count=3,
        timeframe=60,
    )
    trailing = _synthetic_gap_times(
        start_t=60,
        right_t=None,
        count=3,
        timeframe=60,
    )

    assert aligned is not None
    assert aligned.tolist() == [120, 180, 240]
    assert overfull is None
    assert trailing is None


def test_null_repair_falls_back_without_debug_pause(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    '''
    Empty provider repair must resolve interior zero rows locally.

    QQQ startup exposed 653 zero-time rows between stored history and
    an IB frame. The null repair queried IB while reverse backfill
    occupied its sole history request slot, then could
    enter an interactive debug pause before reaching forward-fill.
    Arrange an interior null run, return an empty provider frame, and
    wedge sampler notification. A short timeout and fail-fast pause
    prove local fallback needs no interactive or IPC progress.
    The assertions verify only the
    zero rows inherit the preceding close and synthesized timestamps;
    both valid boundary rows remain unchanged.

    '''
    monkeypatch.setattr(
        'piker.tsp._history._notify_timeout_s',
        0.01,
    )

    backing = np.zeros(
        105,
        dtype=np.dtype(def_iohlcv_fields),
    )
    backing['index'] = np.arange(105)
    frame = backing[100:105]
    frame['index'] = np.arange(100, 105)
    frame['time'] = [60, 0, 0, 0, 300]
    frame['open'] = [9, 0, 0, 0, 19]
    frame['high'] = [11, 0, 0, 0, 21]
    frame['low'] = [8, 0, 0, 0, 18]
    frame['close'] = [10, 0, 0, 0, 20]
    frame['volume'] = [5, 0, 0, 0, 6]

    class Shm:
        '''
        Expose the test frame through the SHM repair interface.

        '''
        def __init__(self) -> None:
            self._array = backing

        @property
        def array(self) -> np.ndarray:
            '''
            Return the readable SHM view.

            '''
            return self._array[100:105]

    class Sampler:
        '''
        Model a backpressured UI notification stream.

        '''
        async def send(self, msg) -> None:
            '''
            Block until the bounded notifier cancels this send.

            '''
            await trio.sleep_forever()

    async def get_hist(*args, **kwargs):
        '''
        Return no provider bars for the requested null segment.

        '''
        end_dt = kwargs['end_dt']
        empty = np.empty(
            0,
            dtype=np.dtype(def_iohlcv_fields),
        )
        return empty, end_dt, end_dt

    async def fail_pause() -> None:
        '''
        Reject interactive debugging in normal repair fallbacks.

        '''
        raise AssertionError('null repair entered tractor.pause()')

    monkeypatch.setattr(tractor, 'pause', fail_pause)

    async def main() -> None:
        import greenback

        async def ensure_portal() -> None:
            '''
            Avoid installing a greenback portal in this unit test.

            '''

        monkeypatch.setattr(
            greenback,
            'ensure_portal',
            ensure_portal,
        )
        with trio.fail_after(0.1):
            await maybe_fill_null_segments(
                shm=Shm(),
                timeframe=60,
                get_hist=get_hist,
                sampler_stream=Sampler(),
                mkt=SimpleNamespace(fqme='qqq.nasdaq.ib'),
                backfill_until_dt=from_timestamp(60),
            )

    trio.run(main)

    assert frame['time'].tolist() == [60, 120, 180, 240, 300]
    for field in ('open', 'high', 'low', 'close'):
        assert frame[field].tolist() == [
            frame[field][0],
            10,
            10,
            10,
            frame[field][-1],
        ]
    assert frame['volume'].tolist() == [5, 0, 0, 0, 6]
