piker/tests/test_jev.py

# piker: trading gear for hackers
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of pikers)
# SPDX-License-Identifier: AGPL-3.0-or-later

# This program is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation, either version 3
# of the License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public
# License along with this program. If not, see
# <https://www.gnu.org/licenses/>.

'''
Offline TypeSafe protocol and controlled Trio scheduling regressions.

'''
from dataclasses import replace
import json

import httpx
import numpy as np
import pytest
import trio
from trio.testing import MockClock, wait_all_tasks_blocked

from piker.fsp._jev import (
    API_URL,
    DEFAULT_MODEL,
    JevClassification,
    JevClient,
    JevProtocolError,
    JevServiceError,
    RegimeClassifier,
    build_regime_request,
)
from piker.fsp._regime import (
    MultiTimeframeTrend,
    RegimeSnapshot,
    TimeframeRegime,
)


def _snapshots(count: int = 12) -> list[RegimeSnapshot]:
    '''
    Produce complete synthetic evidence without a broker or network.

    '''
    tracker: MultiTimeframeTrend = MultiTimeframeTrend(
        'synthetic.test', periods=(1, 2), windows=(2,),
    )
    snapshots: list[RegimeSnapshot] = []
    second: int
    for second in range(count):
        snapshots.append(tracker.push(second, 100 + second))
    return snapshots


def _answer_payload(request: httpx.Request) -> dict:
    '''
    Simulate the documented Choice response for every sent question.

    '''
    payload: dict = json.loads(request.content)
    answers: dict[str, dict] = {}
    name: str
    question: dict
    for name, question in payload['questions'].items():
        labels: list[str] = list(question['criteria'])
        probabilities: dict[str, float] = {}
        label: str
        for label in labels:
            probabilities[label] = (
                .8 if label == labels[0] else .2 / (len(labels) - 1)
            )
        answers[name] = {
            'type': 'choice',
            'choice': labels[0],
            'confidence': .8,
            'probabilities': probabilities,
        }
    return {
        'model': DEFAULT_MODEL,
        'answers': answers,
        'usage': {
            'input_tokens': 250,
            'output_tokens': 32,
        },
    }


def test_request_batches_only_ready_timeframes() -> None:
    '''
    Question IDs are not model input, so a timeframe-only identifier
    leaves the judgment ambiguous. Verify explicit state paths in
    each ready-frame question, one batched alignment question, and
    no request before warm-up. The provider never supplies TC state.

    '''
    snapshots: list[RegimeSnapshot] = _snapshots()
    with pytest.raises(ValueError, match='warm-up'):
        build_regime_request(snapshots[0])
    early: dict = build_regime_request(snapshots[3])
    assert set(early['questions']) == {'timeframe_0'}
    ready: dict = build_regime_request(snapshots[-1])
    assert set(ready['questions']) == {
        'timeframe_0', 'timeframe_1', 'alignment',
    }
    assert ready['model'] == DEFAULT_MODEL
    assert 'timeframes[1]' in (
        ready['questions']['timeframe_1']['instructions']
    )
    assert 'unknown' in ready['questions']['timeframe_0']['criteria']
    assert ready['state']['algorithm'] == 'trend_calculus'


def test_graphics_snapshot_projects_to_bounded_model_evidence(
) -> None:
    '''
    Graphics retains 256 observations for visible pivots, exceeding
    the default HTTP request budget if sent directly to Jev. Build
    an actual 4500-bar, four-order tracker with that rendering tail;
    model preparation must keep only its configured evidence tails
    while preserving source provenance, readiness and total counts.
    The original rendering snapshot must remain unchanged, and the
    projected request must pass through the ordinary HTTP adapter.

    '''
    tracker: MultiTimeframeTrend = MultiTimeframeTrend(
        'test.market', periods=(1, 5, 15), windows=(2,) * 4,
        history_limit=256,
    )
    second: int
    for second in range(4500):
        snapshot: RegimeSnapshot = tracker.push(
            second, 100 + float(np.sin(second / 3)),
        )
    full: dict = build_regime_request(snapshot, evidence_limit=256)
    projected: dict = build_regime_request(snapshot)
    assert len(json.dumps(full).encode()) > 65_536
    assert len(json.dumps(projected).encode()) < 65_536
    assert len(snapshot.timeframes[0].recent_closes) == 256
    assert projected['state']['revision'] == snapshot.revision
    assert projected['state']['sequence'] == snapshot.sequence
    frame: dict
    source: TimeframeRegime
    for frame, source in zip(
        projected['state']['timeframes'], snapshot.timeframes,
    ):
        assert len(frame['recent_closes']) <= 16
        assert len(frame['reversals']) <= 16
        assert frame['close_count'] == source.close_count
        assert frame['ready'] == source.ready
    calls: int = 0

    def handler(request: httpx.Request) -> httpx.Response:
        '''
        Prove projection precedes the client's byte-budget gate.

        '''
        nonlocal calls
        calls += 1
        assert len(request.content) <= 65_536
        return httpx.Response(200, json=_answer_payload(request))

    async def scenario() -> None:
        '''
        Exercise normal client admission with the larger UI snapshot.

        '''
        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            client: JevClient = JevClient(http, 'offline-test-key')
            result: JevClassification = await client.classify(
                snapshot,
            )
            assert result.sequence == snapshot.sequence
            assert len(result.answers) == 4

    trio.run(scenario)
    assert calls == 1


def test_http_contract_retains_probabilities_and_source_times(
) -> None:
    '''
    Collapsing a provider result to a label loses uncertainty and the
    source revision needed to reject stale UI updates. MockTransport
    inspects the actual HTTP body and bearer header, then responds
    using the documented schema. Every probability, source identity,
    model version and local availability time must survive parsing.

    '''
    snapshot: RegimeSnapshot = _snapshots()[-1]
    requests: list[httpx.Request] = []

    def handler(request: httpx.Request) -> httpx.Response:
        '''
        Capture the request without permitting network I/O.

        '''
        requests.append(request)
        return httpx.Response(200, json=_answer_payload(request))

    async def scenario() -> None:
        '''
        Exercise the same Trio-compatible HTTP path as live calls.

        '''
        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            client: JevClient = JevClient(http, 'offline-test-key')
            result: JevClassification = await client.classify(
                snapshot,
            )
        assert result.market == snapshot.market
        assert result.revision == snapshot.revision
        assert result.source_as_of == snapshot.as_of
        assert result.sequence == snapshot.sequence
        assert result.requested_at <= result.received_at
        assert result.latency >= 0
        assert result.model == DEFAULT_MODEL
        assert result.input_tokens == 250
        assert result.answers[0].period == 1
        assert result.answers[0].probability == .8
        assert result.answers[0].confidence == .8
        assert result.answers[-1].period is None
        distribution: dict = dict(result.answers[0].probabilities)
        assert sum(distribution.values()) == pytest.approx(1)

    trio.run(scenario)
    assert len(requests) == 1
    assert str(requests[0].url) == API_URL
    assert requests[0].headers['authorization'] == (
        'Bearer offline-test-key'
    )
    body: dict = json.loads(requests[0].content)
    assert body['state']['as_of'] == snapshot.as_of


@pytest.mark.parametrize('fault', [
    'missing_question',
    'unknown_label',
    'missing_probability',
    'nonfinite_probability',
    'negative_probability',
    'wrong_sum',
    'not_maximum',
    'invalid_confidence',
    'wrong_model',
    'negative_usage',
    'boolean_usage',
])
def test_malformed_response_cannot_publish(fault: str) -> None:
    '''
    A typed API can still fail at the network/schema boundary.
    Deliberately corrupt one otherwise valid response component and
    require a protocol error. This prevents invalid probabilities,
    incomplete classifications or unexpected model versions from
    entering the same display path as validated semantic judgments.

    '''
    def handler(request: httpx.Request) -> httpx.Response:
        '''
        Inject one isolated contract violation into a valid answer.

        '''
        body: dict = _answer_payload(request)
        answer: dict = body['answers']['timeframe_0']
        probabilities: dict = answer['probabilities']
        if fault == 'missing_question':
            del body['answers']['timeframe_0']
        elif fault == 'unknown_label':
            answer['choice'] = 'buy_now'
        elif fault == 'missing_probability':
            del probabilities['unknown']
        elif fault == 'nonfinite_probability':
            probabilities['uptrend'] = 'NaN'
        elif fault == 'negative_probability':
            probabilities['uptrend'] = -.8
        elif fault == 'wrong_sum':
            probabilities['uptrend'] = .5
        elif fault == 'not_maximum':
            answer['choice'] = 'downtrend'
        elif fault == 'invalid_confidence':
            answer['confidence'] = True
        elif fault == 'wrong_model':
            body['model'] = 'unexpected-model'
        elif fault == 'negative_usage':
            body['usage']['input_tokens'] = -1
        elif fault == 'boolean_usage':
            body['usage']['input_tokens'] = True
        return httpx.Response(200, json=body)

    async def scenario() -> None:
        '''
        Verify protocol rejection at the HTTP adapter boundary.

        '''
        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            client: JevClient = JevClient(http, 'offline-test-key')
            with pytest.raises(JevProtocolError):
                await client.classify(_snapshots()[-1])

    trio.run(scenario)


def test_request_and_response_budgets_prevent_unbounded_payloads(
) -> None:
    '''
    Bounded history alone does not bound JSON bytes when parameters
    or provider output changes. A tiny request limit must fail before
    transport invocation; a tiny response limit must reject received
    data instead of decoding or publishing an oversized answer.

    '''
    calls: list[httpx.Request] = []

    def handler(request: httpx.Request) -> httpx.Response:
        '''
        Count transport calls and return an ordinary valid answer.

        '''
        calls.append(request)
        return httpx.Response(200, json=_answer_payload(request))

    async def scenario() -> None:
        '''
        Verify the two independent budgets in one owned HTTP session.

        '''
        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            client: JevClient = JevClient(
                http, 'offline-test-key', max_request_bytes=10,
            )
            with pytest.raises(ValueError, match='byte budget'):
                await client.classify(_snapshots()[-1])
            assert not calls
            client = JevClient(
                http, 'offline-test-key', max_response_bytes=10,
            )
            with pytest.raises(
                JevProtocolError, match='byte budget',
            ):
                await client.classify(_snapshots()[-1])
            assert len(calls) == 1

    trio.run(scenario)


def test_worker_coalesces_bursts_and_discards_old_revision() -> None:
    '''
    One request per quote can create an unbounded queue, while a
    response from before a backfill can repaint corrected history.
    The ``handler()`` task sets ``entered`` before waiting on
    ``release``. While that request is suspended, ``scenario()``
    replaces ``RegimeClassifier._pending`` with a burst and revision.
    Releasing the first request must discard its old evidence and
    publish only the replacement. Request count and ``peak`` prove
    coalescing and single-flight transport without scheduler timing.

    '''
    requests: list[dict] = []
    published: list[JevClassification] = []
    active: int = 0
    peak: int = 0

    async def scenario() -> None:
        '''
        Control response order with events instead of scheduler luck.

        '''
        entered: trio.Event = trio.Event()
        release: trio.Event = trio.Event()

        async def handler(request: httpx.Request) -> httpx.Response:
            '''
            Keep the first network operation in flight during submit.

            '''
            nonlocal active, peak
            active += 1
            peak = max(peak, active)
            requests.append(json.loads(request.content))
            try:
                if len(requests) == 1:
                    entered.set()
                    await release.wait()
                return httpx.Response(
                    200, json=_answer_payload(request),
                )
            finally:
                active -= 1

        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            worker: RegimeClassifier = RegimeClassifier(
                JevClient(http, 'offline-test-key'),
                min_interval=0,
                max_source_lag=100,
            )
            snapshots: list[RegimeSnapshot] = _snapshots()
            worker.submit(snapshots[7])
            nursery: trio.Nursery
            async with trio.open_nursery() as nursery:
                async def publish(result: JevClassification) -> None:
                    '''
                    Stop after the fresh result proves publication.

                    '''
                    published.append(result)
                    nursery.cancel_scope.cancel()

                nursery.start_soon(worker.run, publish)
                await entered.wait()
                snapshot: RegimeSnapshot
                for snapshot in snapshots[8:]:
                    worker.submit(snapshot)
                newest: RegimeSnapshot = replace(
                    snapshots[-1], revision=1, sequence=1,
                )
                worker.submit(newest)
                release.set()
            assert worker.discarded == 1
            assert worker.requests == 2
            assert worker.published == 1

    trio.run(scenario)
    assert len(requests) == 2
    assert peak == 1
    assert len(published) == 1
    assert published[0].revision == 1
    assert requests[1]['state']['revision'] == 1
    assert requests[1]['state']['sequence'] == 1


@pytest.mark.parametrize('invalidation', ['lag', 'segment', 'age'])
def test_worker_rejects_stale_results(invalidation: str) -> None:
    '''
    Matching ``RegimeSnapshot.revision`` alone does not prove
    freshness: a gap, advancing event clock, or slow response can
    invalidate evidence. ``handler()`` sets ``entered`` and waits
    on ``release`` while ``scenario()`` changes exactly one of those
    conditions. Only the second response may publish. ``MockClock``
    controls elapsed age; the two events prove the first response
    cannot finish before its evidence becomes obsolete.

    '''
    published: list[JevClassification] = []

    async def scenario() -> None:
        '''
        Delay the first response while replacing source evidence.

        '''
        entered: trio.Event = trio.Event()
        release: trio.Event = trio.Event()
        calls: int = 0

        async def handler(request: httpx.Request) -> httpx.Response:
            '''
            Suspend only the first request at a cancellable event.

            '''
            nonlocal calls
            calls += 1
            if calls == 1:
                entered.set()
                await release.wait()
            return httpx.Response(200, json=_answer_payload(request))

        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            worker: RegimeClassifier = RegimeClassifier(
                JevClient(http, 'offline-test-key', timeout=20),
                min_interval=0,
                max_age=5,
                max_source_lag=1,
            )
            snapshots: list[RegimeSnapshot] = _snapshots()
            worker.submit(snapshots[7])
            nursery: trio.Nursery
            async with trio.open_nursery() as nursery:
                async def publish(result: JevClassification) -> None:
                    '''
                    Exit only after an admissible response arrives.

                    '''
                    published.append(result)
                    nursery.cancel_scope.cancel()

                nursery.start_soon(worker.run, publish)
                await entered.wait()
                newest: RegimeSnapshot = snapshots[8]
                if invalidation == 'lag':
                    newest = snapshots[-1]
                elif invalidation == 'segment':
                    newest = replace(newest, segment=1)
                else:
                    await trio.sleep(6)
                worker.submit(newest)
                release.set()
            assert worker.discarded == 1
            assert published[0].sequence == newest.sequence
            assert worker.published == 1

    trio.run(scenario, clock=MockClock(autojump_threshold=0))


def test_worker_throttles_and_honors_retry_after() -> None:
    '''
    Immediate retries amplify rate limits and may resend old market
    evidence. Return HTTP 429 with a three-second Retry-After, submit
    newer evidence while the worker waits, and use MockClock to
    prove the retry respects that delay and contains newest state.

    '''
    sent: list[tuple[float, int]] = []
    errors: list[Exception] = []

    async def scenario() -> None:
        '''
        Advance simulated time only after tasks reach their waits.

        '''
        failed: trio.Event = trio.Event()

        async def handler(request: httpx.Request) -> httpx.Response:
            '''
            Rate-limit the initial request and accept its successor.

            '''
            body: dict = json.loads(request.content)
            sent.append((
                trio.current_time(), body['state']['sequence'],
            ))
            if len(sent) == 1:
                return httpx.Response(
                    429, headers={'retry-after': '3'},
                )
            return httpx.Response(200, json=_answer_payload(request))

        async def on_error(error: Exception) -> None:
            '''
            Signal the exact rate-limit failure before new evidence.

            '''
            errors.append(error)
            failed.set()

        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            worker: RegimeClassifier = RegimeClassifier(
                JevClient(http, 'offline-test-key'),
                min_interval=2,
                max_age=10,
                max_source_lag=10,
            )
            snapshots: list[RegimeSnapshot] = _snapshots()
            worker.submit(snapshots[7])
            nursery: trio.Nursery
            async with trio.open_nursery() as nursery:
                async def publish(result: JevClassification) -> None:
                    '''
                    End the test after successful recovery.

                    '''
                    nursery.cancel_scope.cancel()

                async def run() -> None:
                    '''
                    Bind the worker's optional failure callback.

                    '''
                    await worker.run(publish, on_error=on_error)

                nursery.start_soon(run)
                await failed.wait()
                worker.submit(snapshots[-1])
            assert worker.requests == 2
            assert worker.failures == 1
            assert worker.last_error is None

    trio.run(scenario, clock=MockClock(autojump_threshold=0))
    assert sent[1][0] - sent[0][0] >= 3
    assert sent[1][1] == 12
    assert isinstance(errors[0], JevServiceError)


def test_cancellation_closes_inflight_work() -> None:
    '''
    An HTTP operation detached from its nursery can outlive a chart
    and use credentials after shutdown. ``handler()`` signals
    ``entered`` before ``trio.sleep_forever()``; ``scenario()`` waits
    for that signal before cancelling the worker nursery. The
    ``closed`` event proves the transport's finalizer runs before
    nursery exit, and ``publish()`` rejects any late output. This
    proves cooperative cancellation, not an uninterruptible syscall.

    '''
    async def scenario() -> None:
        '''
        Cancel only once the mock request is known to be in flight.

        '''
        entered: trio.Event = trio.Event()
        closed: trio.Event = trio.Event()

        async def handler(request: httpx.Request) -> httpx.Response:
            '''
            Model a cancellable stalled network operation.

            '''
            entered.set()
            try:
                await trio.sleep_forever()
            finally:
                closed.set()

        async def publish(result: JevClassification) -> None:
            '''
            Reject any publication after cancellation begins.

            '''
            pytest.fail('cancelled request published a result')

        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            worker: RegimeClassifier = RegimeClassifier(
                JevClient(http, 'offline-test-key'), min_interval=0,
            )
            worker.submit(_snapshots()[-1])
            nursery: trio.Nursery
            async with trio.open_nursery() as nursery:
                nursery.start_soon(worker.run, publish)
                await entered.wait()
                nursery.cancel_scope.cancel()
            assert closed.is_set()
            assert worker.published == 0

    trio.run(scenario)


def test_submit_rejects_wrong_market_and_nonmonotonic_source(
) -> None:
    '''
    Sharing a worker across markets or submitting old source evidence
    can overwrite its sole pending slot. Feed valid then invalid
    identities without starting the worker and prove invalid updates
    are refused before any HTTP call can be made.

    '''
    async def scenario() -> None:
        '''
        Exercise submission validation on an otherwise idle worker.

        '''
        http: httpx.AsyncClient
        async with httpx.AsyncClient() as http:
            worker: RegimeClassifier = RegimeClassifier(
                JevClient(http, 'offline-test-key'),
            )
            snapshot: RegimeSnapshot = _snapshots()[-1]
            assert worker.submit(snapshot)
            assert not worker.submit(snapshot)
            with pytest.raises(ValueError, match='market'):
                worker.submit(replace(snapshot, market='other.test'))
            with pytest.raises(ValueError, match='as_of'):
                worker.submit(replace(snapshot, sequence=13))
            assert worker.requests == 0

    trio.run(scenario)


def test_worker_publishes_bounded_lag_with_original_provenance(
) -> None:
    '''
    Rejecting every response after a faster input update starves live
    inference. Permit one second of lag, hold the first response,
    then submit its successor. Both may publish, but the older result
    must retain its own source sequence and as_of rather than being
    relabeled with the newer evidence's coordinates.

    '''
    published: list[JevClassification] = []

    async def scenario() -> None:
        '''
        Force a new source update before the first response returns.

        '''
        entered: trio.Event = trio.Event()
        release: trio.Event = trio.Event()
        calls: int = 0

        async def handler(request: httpx.Request) -> httpx.Response:
            '''
            Hold one response while leaving later requests immediate.

            '''
            nonlocal calls
            calls += 1
            if calls == 1:
                entered.set()
                await release.wait()
            return httpx.Response(200, json=_answer_payload(request))

        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            worker: RegimeClassifier = RegimeClassifier(
                JevClient(http, 'offline-test-key'),
                min_interval=0,
                max_source_lag=1,
            )
            snapshots: list[RegimeSnapshot] = _snapshots()
            worker.submit(snapshots[7])
            nursery: trio.Nursery
            async with trio.open_nursery() as nursery:
                async def publish(result: JevClassification) -> None:
                    '''
                    Stop after observing both ordered outputs.

                    '''
                    published.append(result)
                    if len(published) == 2:
                        nursery.cancel_scope.cancel()

                nursery.start_soon(worker.run, publish)
                await entered.wait()
                worker.submit(snapshots[8])
                release.set()
            assert worker.discarded == 0
            assert worker.published == 2

    trio.run(scenario)
    result: JevClassification  # noqa: F842
    assert [result.sequence for result in published] == [8, 9]
    assert [result.source_as_of for result in published] == [8, 9]


def test_worker_transient_retry_count_is_bounded() -> None:
    '''
    An unavailable service must not generate an endless retry loop
    for one snapshot. Return 503 on every call, allow two retries,
    and use MockClock to reach the third failure. Once all tasks are
    blocked, the worker must be idle until new evidence arrives;
    cancelling it must not reveal another scheduled request.

    '''
    calls: list[float] = []

    async def scenario() -> None:
        '''
        Observe exhausted retries before cancelling the idle worker.

        '''
        exhausted: trio.Event = trio.Event()
        errors: int = 0

        async def handler(request: httpx.Request) -> httpx.Response:
            '''
            Keep the service unavailable for the whole scenario.

            '''
            calls.append(trio.current_time())
            return httpx.Response(503)

        async def on_error(error: Exception) -> None:
            '''
            Signal when the final allowed attempt has failed.

            '''
            nonlocal errors
            errors += 1
            if errors == 3:
                exhausted.set()

        async def publish(result: JevClassification) -> None:
            '''
            Reject fabricated results during provider unavailability.

            '''
            pytest.fail('a failed request must not publish')

        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            worker: RegimeClassifier = RegimeClassifier(
                JevClient(http, 'offline-test-key'),
                min_interval=0,
                max_age=100,
                max_retries=2,
            )
            worker.submit(_snapshots()[-1])

            async def run() -> None:
                '''
                Attach the failure observer to the worker lifetime.

                '''
                await worker.run(publish, on_error=on_error)

            nursery: trio.Nursery
            async with trio.open_nursery() as nursery:
                nursery.start_soon(run)
                await exhausted.wait()
                await wait_all_tasks_blocked()
                assert worker.requests == 3
                assert worker.failures == 3
                nursery.cancel_scope.cancel()

    trio.run(scenario, clock=MockClock(autojump_threshold=0))
    assert calls == [0, 1, 3]


def test_worker_drops_expired_pending_work_before_network() -> None:
    '''
    Throttling can make a queued snapshot expire even if no request
    is in flight. Publish one result, enqueue its successor, then
    advance MockClock beyond its five-second age limit but to the
    next ten-second request slot. The expired successor must be
    dropped before transport invocation, not merely after a reply.

    '''
    clock: MockClock = MockClock()
    calls: int = 0

    async def scenario() -> None:
        '''
        Move time only after the worker reaches its throttle wait.

        '''
        first: trio.Event = trio.Event()

        async def handler(request: httpx.Request) -> httpx.Response:
            '''
            Count requests independently of worker statistics.

            '''
            nonlocal calls
            calls += 1
            return httpx.Response(200, json=_answer_payload(request))

        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            worker: RegimeClassifier = RegimeClassifier(
                JevClient(http, 'offline-test-key'),
                min_interval=10,
                max_age=5,
            )
            snapshots: list[RegimeSnapshot] = _snapshots()
            worker.submit(snapshots[7])

            async def publish(result: JevClassification) -> None:
                '''
                Submit fresh evidence at the initial publication.

                '''
                assert worker.published == 0
                worker.submit(snapshots[8])
                first.set()

            nursery: trio.Nursery
            async with trio.open_nursery() as nursery:
                nursery.start_soon(worker.run, publish)
                await first.wait()
                await wait_all_tasks_blocked()
                clock.jump(10)
                await wait_all_tasks_blocked()
                assert worker.requests == 1
                assert worker.discarded == 1
                nursery.cancel_scope.cancel()

    trio.run(scenario, clock=clock)
    assert calls == 1


def test_authentication_failure_propagates_without_retry() -> None:
    '''
    Repeatedly retrying an invalid credential consumes requests and
    hides configuration failures. A 401 response must escape the
    worker immediately with a sanitized status-only error; neither
    the key nor the server body should appear in its exception text.

    '''
    async def scenario() -> None:
        '''
        Return an authentication failure through the real HTTP path.

        '''
        def handler(request: httpx.Request) -> httpx.Response:
            '''
            Supply a body that should never be retained in the error.

            '''
            return httpx.Response(401, text='private server details')

        async def publish(result: JevClassification) -> None:
            '''
            Reject any result produced without authentication.

            '''
            pytest.fail('unauthenticated request published')

        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            worker: RegimeClassifier = RegimeClassifier(
                JevClient(http, 'offline-test-key'),
            )
            worker.submit(_snapshots()[-1])
            caught: pytest.ExceptionInfo[JevServiceError]
            with pytest.raises(JevServiceError) as caught:
                await worker.run(publish)
            assert caught.value.status == 401
            assert 'offline-test-key' not in str(caught.value)
            assert 'private server details' not in str(caught.value)
            assert worker.requests == 1

    trio.run(scenario)


@pytest.mark.parametrize(
    'invalidation', ['revision', 'segment', 'age'],
)
def test_blocked_publication_is_cancelled_when_evidence_changes(
    invalidation: str,
) -> None:
    '''
    Checking freshness before awaiting an output consumer is not
    enough: a blocked send can deliver obsolete results after source
    revision, gap or expiry. ``publish()`` sets ``entered`` before
    awaiting ``released``. The ``scenario()`` task then changes the
    revision/segment or advances ``MockClock`` past expiry and waits
    for ``cancelled`` before releasing the consumer. An empty sink
    proves the obsolete delivery was revoked; final provenance and
    counters prove only the replacement reaches ``published``. The
    event wait models cooperative backpressure, not a stuck syscall.

    '''
    published: list[JevClassification] = []

    async def scenario() -> None:
        '''
        Distinguish a blocked delivery from one already delivered.

        '''
        entered: trio.Event = trio.Event()
        released: trio.Event = trio.Event()
        cancelled: trio.Event = trio.Event()
        attempts: int = 0

        def handler(request: httpx.Request) -> httpx.Response:
            '''
            Complete HTTP immediately; delay the output consumer.

            '''
            return httpx.Response(200, json=_answer_payload(request))

        http: httpx.AsyncClient
        async with httpx.AsyncClient(
            transport=httpx.MockTransport(handler),
        ) as http:
            worker: RegimeClassifier = RegimeClassifier(
                JevClient(http, 'offline-test-key'),
                min_interval=0,
                max_age=5,
            )
            snapshots: list[RegimeSnapshot] = _snapshots()
            worker.submit(snapshots[7])
            nursery: trio.Nursery
            async with trio.open_nursery() as nursery:
                async def publish(result: JevClassification) -> None:
                    '''
                    Delay the consumer's first result acceptance.

                    '''
                    nonlocal attempts
                    attempts += 1
                    if attempts == 1:
                        entered.set()
                        try:
                            await released.wait()
                        except trio.Cancelled:
                            cancelled.set()
                            raise
                    else:
                        await released.wait()
                    published.append(result)
                    nursery.cancel_scope.cancel()

                nursery.start_soon(worker.run, publish)
                await entered.wait()
                newest: RegimeSnapshot = snapshots[8]
                if invalidation == 'revision':
                    worker.invalidate(1)
                    newest = replace(newest, revision=1, sequence=1)
                    assert not worker.submit(snapshots[-1])
                elif invalidation == 'segment':
                    newest = replace(newest, segment=1)
                    worker.submit(newest)
                else:
                    await trio.sleep(6)
                await cancelled.wait()
                assert not published
                released.set()
                worker.submit(newest)
            assert worker.discarded == 1
            assert worker.published == 1
            assert published[0].sequence == newest.sequence
            assert published[0].revision == newest.revision
            assert published[0].segment == newest.segment

    trio.run(scenario, clock=MockClock(autojump_threshold=0))