piker/tests/test_trend_mtf_graphics.py

# piker: trading gear for hackers
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of pikers)
# SPDX-License-Identifier: AGPL-3.0-or-later

# This program is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation, either version 3
# of the License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public
# License along with this program. If not, see
# <https://www.gnu.org/licenses/>.

'''
Actual shared-tracker graphics, freshness and native feed lifecycle.

'''
from collections import deque
from collections.abc import AsyncIterator, Iterator
from contextlib import asynccontextmanager
from dataclasses import replace
import os
from types import SimpleNamespace

os.environ['QT_QPA_PLATFORM'] = 'offscreen'

import numpy as np
import pyqtgraph as pg
import pytest
import trio
from trio.testing import wait_all_tasks_blocked

from piker.fsp._jev import JevClassification, RegimeAnswer
from piker.fsp._regime import (
    MultiTimeframeTrend,
    RegimeSnapshot,
    TimeframeRegime,
    TrendReversal,
)
from piker.ui import _fsp as fsp_ui
from piker.ui._trend import (
    RegimeInterval,
    ReversalPoint,
    TrendOverlay,
)
from piker.ui._trend_mtf import (
    SnapshotTrendOverlay,
    _SnapshotLayer,
)
from piker.ui.qt import QApplication


class _ArrayShm:
    '''
    Model published source bounds over deterministic OHLC bars.

    '''
    def __init__(self, count: int = 600, visible: int = 401) -> None:
        '''
        Keep an allocated mutable suffix beyond the published range.

        '''
        self._array: np.ndarray = np.zeros(count, dtype=[
            ('index', 'i8'),
            ('time', 'f8'),
            ('close', 'f8'),
            ('low', 'f8'),
            ('high', 'f8'),
        ])
        steps: np.ndarray = np.arange(count)
        prices: np.ndarray = (
            100 + .015 * steps
            + 2 * np.sin(steps * .09)
            + .5 * np.sin(steps * .4)
        )
        self._array['index'] = 1000 + steps
        self._array['time'] = steps
        self._array['close'] = prices
        self._array['low'] = prices - .2
        self._array['high'] = prices + .2
        self._first: SimpleNamespace = SimpleNamespace(value=0)
        self._last: SimpleNamespace = SimpleNamespace(value=visible)
        self._token: str = 'native-ohlcv'

    @property
    def array(self) -> np.ndarray:
        '''
        Expose only the current published SHM range.

        '''
        return self._array[self._first.value:self._last.value]


class _Chart:
    '''
    Use a real price plot while replacing actor and SHM allocation.

    '''
    _trend_overlays: dict[tuple[str, str], list[TrendOverlay]]

    def __init__(self, shm: _ArrayShm) -> None:
        '''
        Bind the actual base source to an offscreen plot.

        '''
        self.widget: pg.PlotWidget = pg.PlotWidget()
        self.viz: SimpleNamespace = SimpleNamespace(
            plot=self.widget.plotItem,
            shm=shm,
            index_field='index',
        )

    def get_viz(self, fqme: str) -> SimpleNamespace|None:
        '''
        Resolve the only market owned by this chart.

        '''
        return self.viz if fqme == 'mtf.test' else None

    def close(self) -> None:
        '''
        Release the disposable Qt widget.

        '''
        self.widget.close()
        self.widget.deleteLater()


@pytest.fixture(scope='session')
def qapp() -> Iterator[QApplication]:
    '''
    Keep one offscreen application alive for native graphics checks.

    '''
    app: QApplication|None = QApplication.instance()
    if app is None:
        app = QApplication(['piker-mtf-tests'])
    app.setQuitOnLastWindowClosed(False)
    yield app
    app.processEvents()


def _replay(
    tracker: MultiTimeframeTrend,
    shm: _ArrayShm,
    overlay: SnapshotTrendOverlay|None = None,
    start: int = 0,

) -> RegimeSnapshot:
    '''
    Replay final closes through the production shared recurrence.

    '''
    snapshot: RegimeSnapshot
    row: np.void
    for row in shm.array[start:-1]:
        snapshot = tracker.push(
            float(row['time']),
            float(row['close']),
            index=int(row['index']),
        )
        if overlay is not None:
            overlay.submit(snapshot)
    return snapshot


def _classification(
    snapshot: RegimeSnapshot,
    *,
    received_at: float|None = None,

) -> JevClassification:
    '''
    Build explicit synthetic provider output without any HTTP path.

    '''
    return JevClassification(
        market=snapshot.market,
        revision=snapshot.revision,
        sequence=snapshot.sequence,
        segment=snapshot.segment,
        source_as_of=snapshot.as_of,
        requested_at=snapshot.as_of,
        received_at=(
            received_at
            if received_at is not None
            else snapshot.as_of
        ),
        latency=.1,
        model='synthetic-jev-test',
        prompt_version='test',
        answers=(RegimeAnswer(
            question='regime_5',
            period=5,
            label='range',
            probability=.8,
            confidence=.6,
            probabilities=(('range', .8), ('unknown', .2)),
        ),),
        input_tokens=0,
        output_tokens=0,
    )


def test_bad_limits_do_not_allocate_graphics(
    qapp: QApplication,

) -> None:
    '''
    Reject fractional and boolean bounds before attaching Qt items.

    A failed child constructor cannot be cleaned up by the parent
    because the parent has not received it yet. Exercise the public
    scalar and snapshot constructors against an actual empty plot.

    '''
    shm: _ArrayShm = _ArrayShm()
    chart: _Chart = _Chart(shm)
    try:
        name: str
        for name in ('history_limit', 'order'):
            value: float|bool
            for value in (1.5, True):
                with pytest.raises(ValueError):
                    SnapshotTrendOverlay(
                        chart.viz, shm,
                        fqme='mtf.test', base_period=1,
                        **{name: value},
                    )
                assert not chart.widget.plotItem.items

        for name in ('max_points', 'max_rectangles', 'order'):
            value: float|bool
            for value in (1.5, True):
                with pytest.raises(ValueError):
                    TrendOverlay(
                        chart.viz, shm,
                        fqme='mtf.test', source_period_s=1,
                        chart_period_s=1,
                        **{name: value},
                    )
                assert not chart.widget.plotItem.items
    finally:
        chart.close()
        qapp.processEvents()


@pytest.mark.parametrize('periods', [
    (1., 5., 15.),
    (15.000000000000002, 1.0000000000000002, 5.000000000000001),
])
def test_bootstrap_uses_real_timeframes_and_endpoint_pivots(
    qapp: QApplication,
    periods: tuple[float, ...],

) -> None:
    '''
    Sampling 1 s, 5 s and 15 s bars must aggregate one base stream,
    rather than relabel identical computations. A real tracker and
    scalar sine-wave closes create distinct reversal sets. The
    snapshot's pivot end timestamp remains metadata while its x
    coordinate equals the final base bar's ``TrendReversal.index``.
    Bootstrap supplies no old trend-row history, so rectangles may
    begin only at each timeframe's latest proven as_of timestamp.
    Harmless decimal rounding must normalize to the tracker's exact
    duration keys even if the caller lists timeframes out of order.

    '''
    shm: _ArrayShm = _ArrayShm()
    chart: _Chart = _Chart(shm)
    tracker: MultiTimeframeTrend = MultiTimeframeTrend(
        'mtf.test', periods=periods, history_limit=256,
    )
    adapter: SnapshotTrendOverlay = SnapshotTrendOverlay(
        chart.viz, shm, fqme='mtf.test', base_period=1,
        periods=periods,
    )
    try:
        snapshot: RegimeSnapshot = _replay(tracker, shm)
        assert adapter.submit(snapshot)
        assert adapter.periods == (1., 5., 15.)
        assert len(chart.widget.plotItem.items) == 6
        point_counts: list[int] = []
        frame: TimeframeRegime
        for frame in snapshot.timeframes:
            layer: _SnapshotLayer = adapter.layers[frame.period]
            event: TrendReversal
            events: list[TrendReversal] = [
                event for event in frame.reversals
                if event.order == 1
            ]
            assert len(layer.points) == len(events)
            assert layer.points
            point_counts.append(len(layer.points))
            point: ReversalPoint
            event: TrendReversal
            for point, event in zip(
                layer.points, events, strict=True,
            ):
                assert point.x == event.index
                assert point.pivot_time == event.time
                assert point.detected_at == event.detected_time
                assert point.timestamp_convention == 'interval_end'
            assert layer.rectangles.intervals
            span: RegimeInterval  # noqa: F842
            assert all(
                span.start_time >= frame.as_of
                for span in layer.rectangles.intervals
            )
        assert len(set(point_counts)) == 3
    finally:
        adapter.remove()
        chart.close()
        qapp.processEvents()


def test_revision_segment_and_sequence_revoke_evidence(
    qapp: QApplication,

) -> None:
    '''
    A corrected source or new gap segment invalidates every previous
    pivot and model annotation before replay finishes. The adapter
    receives real tracker output, then an explicit generation change.
    Old market/revision/sequence snapshots and an earlier segment's
    model response must fail acceptance without reviving geometry.

    '''
    shm: _ArrayShm = _ArrayShm()
    chart: _Chart = _Chart(shm)
    tracker: MultiTimeframeTrend = MultiTimeframeTrend('mtf.test')
    adapter: SnapshotTrendOverlay = SnapshotTrendOverlay(
        chart.viz, shm, fqme='mtf.test', base_period=1,
    )
    try:
        snapshot: RegimeSnapshot = _replay(tracker, shm)
        assert adapter.submit(snapshot)
        result: JevClassification = _classification(snapshot)
        assert adapter.submit_classification(result)
        assert not adapter.submit(snapshot)
        assert not adapter.submit(
            replace(snapshot, market='foreign'),
        )
        adapter.invalidate(1)
        layer: _SnapshotLayer  # noqa: F842
        assert all(
            not layer.points for layer in adapter.layers.values()
        )
        assert not adapter.submit(snapshot)
        assert not adapter.submit_classification(result)

        tracker.reset(revision=1)
        replayed: RegimeSnapshot = _replay(tracker, shm)
        assert adapter.submit(replayed)
        result = _classification(replayed)
        assert adapter.submit_classification(result)
        after_gap: RegimeSnapshot = tracker.push(
            405, 101, index=1405,
        )
        assert adapter.submit(after_gap)
        assert adapter.segment == 1
        assert not adapter.submit_classification(result)
        assert all(
            not layer.points for layer in adapter.layers.values()
        )
        runs: deque[RegimeInterval]  # noqa: F842
        assert all(not runs for runs in adapter._model_runs.values())
    finally:
        adapter.remove()
        chart.close()
        qapp.processEvents()


def test_coalesced_buckets_leave_unknown_regime_hole(
    qapp: QApplication,

) -> None:
    '''
    Coalesced callbacks reveal only the latest trend classification.
    Extending ``_SnapshotLayer._runs`` to the next received snapshot
    invents historical knowledge. Skip ten actual finalized buckets
    and prove the old interval stops at its next possible decision
    while the new run starts only at ``RegimeSnapshot.as_of``.

    '''
    shm: _ArrayShm = _ArrayShm(visible=101)
    chart: _Chart = _Chart(shm)
    tracker: MultiTimeframeTrend = MultiTimeframeTrend('mtf.test')
    adapter: SnapshotTrendOverlay = SnapshotTrendOverlay(
        chart.viz, shm, fqme='mtf.test', base_period=1,
    )
    try:
        adapter.submit(_replay(tracker, shm))
        previous: float = adapter.as_of
        shm._last.value += 10
        latest: RegimeSnapshot = _replay(tracker, shm, start=100)
        adapter.submit(latest)
        runs: deque[RegimeInterval] = adapter.layers[1]._runs
        assert len(runs) == 2
        assert runs[0].end_time == previous + 1
        assert runs[1].start_time == latest.as_of
        assert runs[0].end_time < runs[1].start_time
    finally:
        adapter.remove()
        chart.close()
        qapp.processEvents()


def test_model_receipt_expiry_and_delayed_queue(
    qapp: QApplication,

) -> None:
    '''
    A Jev response must neither paint before receipt nor claim the
    live regime indefinitely. Retain a response arriving 2.5 s
    after its source snapshot, advance the real base stream, and
    verify its visible interval starts at receipt and stops after
    the default 5 s lifetime. Old first-ever responses and expired
    queued results are also rejected despite matching provenance.

    '''
    shm: _ArrayShm = _ArrayShm(visible=201)
    chart: _Chart = _Chart(shm)
    tracker: MultiTimeframeTrend = MultiTimeframeTrend('mtf.test')
    adapter: SnapshotTrendOverlay = SnapshotTrendOverlay(
        chart.viz, shm, fqme='mtf.test', base_period=1,
    )
    try:
        snapshot: RegimeSnapshot = _replay(tracker, shm)
        adapter.submit(snapshot)
        old: JevClassification = replace(
            _classification(snapshot),
            sequence=1,
            source_as_of=1,
            received_at=200,
        )
        assert not adapter.submit_classification(old)
        expired: JevClassification = _classification(
            snapshot, received_at=190,
        )
        assert not adapter.submit_classification(expired)

        result: JevClassification = _classification(
            snapshot, received_at=202.5,
        )
        assert adapter.submit_classification(result)
        layer: _SnapshotLayer = adapter.layers[5]
        assert len(layer.rectangles.rectangles) == 0
        shm._last.value = 212
        _replay(tracker, shm, adapter, start=200)
        assert layer.rectangles.intervals
        span: RegimeInterval = layer.rectangles.intervals[-1]
        assert span.observed_at == 202.5
        assert span.end_time == 207.5
        assert layer.rectangles.rectangles[-1, 0] == 1202.5
        assert not adapter.submit_classification(result)
    finally:
        adapter.remove()
        chart.close()
        qapp.processEvents()


def test_snapshot_retention_stays_bounded(
    qapp: QApplication,

) -> None:
    '''
    A long-running shared tracker must not accumulate unbounded Qt
    objects or regime intervals. Stream hundreds of real snapshots
    with a four-event display budget and assert retention at both
    the data and scene boundaries after many reversals.

    '''
    shm: _ArrayShm = _ArrayShm()
    chart: _Chart = _Chart(shm)
    tracker: MultiTimeframeTrend = MultiTimeframeTrend('mtf.test')
    adapter: SnapshotTrendOverlay = SnapshotTrendOverlay(
        chart.viz,
        shm,
        fqme='mtf.test',
        base_period=1,
        history_limit=4,
    )
    try:
        _replay(tracker, shm, adapter)
        assert len(chart.widget.plotItem.items) == 6
        layer: _SnapshotLayer
        for layer in adapter.layers.values():
            assert len(layer.points) <= 4
            assert len(layer._runs) <= 4
            assert len(layer.rectangles.rectangles) <= 4
    finally:
        adapter.remove()
        adapter.remove()
        assert chart.widget.plotItem.items == []
        chart.close()
        qapp.processEvents()


@pytest.mark.parametrize('max_source_age', [5, None])
def test_admin_consumes_native_source_once_and_cancels(
    qapp: QApplication,
    monkeypatch: pytest.MonkeyPatch,
    max_source_age: float|None,

) -> None:
    '''
    ``FspAdmin._run_multi_timeframe_trends()`` must subscribe before
    replay, share one tracker across three periods, and invalidate
    graphics while model callbacks apply their freshness gate.
    The ``sampler()`` stub yields a local Trio receive channel to
    the actual ``consume_regime_source()`` task. Explicit append and
    equal-bound correction notices, followed by
    ``wait_all_tasks_blocked()``, prove each notice is consumed
    before checking snapshot sequence and revision.
    ``FspAdmin.close_trend_overlays()`` must cancel the reader:
    ``closed`` is set only in ``sampler()`` cleanup, before this test
    closes either channel endpoint. Empty Qt items and
    ``_Chart._trend_overlays`` prove owned graphics also detach.
    This controls local task ordering without exercising actor IPC.

    '''
    shm: _ArrayShm = _ArrayShm(visible=201)
    chart: _Chart = _Chart(shm)
    submitted: list[RegimeSnapshot] = []
    invalidated: list[int] = []
    subscription_count: list[float] = []

    async def scenario() -> None:
        '''
        Control publication and cancellation with native Trio events.

        '''
        send: trio.MemorySendChannel[dict]
        receive: trio.MemoryReceiveChannel[dict]
        send, receive = trio.open_memory_channel(4)
        closed: trio.Event = trio.Event()

        @asynccontextmanager
        async def sampler(
            period_s: float,

        ) -> AsyncIterator[trio.MemoryReceiveChannel[dict]]:
            '''
            Open exactly one sampler before the real consumer starts.

            '''
            subscription_count.append(period_s)
            try:
                yield receive
            finally:
                closed.set()

        monkeypatch.setattr(fsp_ui, 'open_sample_stream', sampler)
        linked: SimpleNamespace = SimpleNamespace(
            chart=chart,
            isHidden=lambda: False,
        )
        flume: SimpleNamespace = SimpleNamespace(
            mkt=SimpleNamespace(fqme='mtf.test'),
            rt_shm=shm,
            rt_sample_period_s=1,
        )
        nursery: trio.Nursery
        async with trio.open_nursery() as nursery:
            admin: fsp_ui.FspAdmin = fsp_ui.FspAdmin(
                nursery, {}, linked, flume,
            )
            adapter: SnapshotTrendOverlay = (
                await admin.open_multi_timeframe_trends(
                    on_snapshot=submitted.append,
                    on_invalidate=invalidated.append,
                    max_source_age=max_source_age,
                )
            )
            assert (
                await admin.open_multi_timeframe_trends() is adapter
            )
            assert subscription_count == [1]
            assert len(chart.widget.plotItem.items) == 6
            assert adapter.sequence == 200
            assert bool(submitted) == (max_source_age is None)

            shm._last.value += 1
            await send.send({'period': 1})
            await wait_all_tasks_blocked()
            assert adapter.sequence == 201

            shm._array['close'][20] += 20
            await send.send({'backfilling': ('mtf.test', 1)})
            await wait_all_tasks_blocked()
            assert invalidated == [1]
            assert adapter.revision == 1
            assert adapter.sequence == 201
            admin.close_trend_overlays()
            await closed.wait()
            assert chart._trend_overlays == {}
            assert chart.widget.plotItem.items == []
            await send.aclose()
            await receive.aclose()

    try:
        trio.run(scenario)
    finally:
        chart.close()
        qapp.processEvents()