778 lines
27 KiB
Python
778 lines
27 KiB
Python
# piker: trading gear for hackers
|
|
# Copyright (C) Tyler Goodlet (in stewardship of piker0)
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
# This program is free software: you can redistribute it and/or
|
|
# modify it under the terms of the GNU Affero General Public License
|
|
# as published by the Free Software Foundation, either version 3
|
|
# of the License, or (at your option) any later version.
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
|
|
# You should have received a copy of the GNU Affero General Public
|
|
# License along with this program. If not, see
|
|
# <https://www.gnu.org/licenses/>.
|
|
|
|
'''
|
|
Optional, bounded TypeSafe System One judgments over trend evidence.
|
|
|
|
The HTTP contract is documented at https://docs.typesafe.ai/api .
|
|
``JevClient`` uses piker's existing HTTPX/Trio stack. Credentials are
|
|
explicit constructor inputs; importing this module opens no network
|
|
connection and reads no environment variables. Model judgments never
|
|
change deterministic TC state or route orders into the EMS.
|
|
|
|
'''
|
|
from collections.abc import Awaitable, Callable
|
|
from dataclasses import dataclass, replace
|
|
from email.utils import parsedate_to_datetime
|
|
import json
|
|
from math import isclose, isfinite
|
|
from operator import index as integer_index
|
|
import time
|
|
from types import MappingProxyType
|
|
|
|
import httpx
|
|
import numpy as np
|
|
import trio
|
|
|
|
from ._regime import RegimeSnapshot, TimeframeRegime
|
|
|
|
|
|
DEFAULT_MODEL: str = 'jev-1.13.0'
|
|
PROMPT_VERSION: str = 'piker-regimes-v1'
|
|
API_URL: str = 'https://api.typesafe.ai/v1/systemone'
|
|
|
|
# Centralized human-reviewable rubric. These are descriptive model
|
|
# judgments, not new TC rules or validated profitability signals.
|
|
REGIME_CRITERIA: MappingProxyType = MappingProxyType({
|
|
'uptrend': (
|
|
'Persistent upward movement supported by recent closes and '
|
|
'ready trend orders; small counter-moves do not dominate.'
|
|
),
|
|
'downtrend': (
|
|
'Persistent downward movement supported by recent closes '
|
|
'and ready trend orders; small retracements do not dominate.'
|
|
),
|
|
'range': (
|
|
'Recent movement repeatedly retraces within a bounded range '
|
|
'without a persistent direction, or closes remain flat.'
|
|
),
|
|
'transition': (
|
|
'Recent detected reversals and conflicting ready orders '
|
|
'suggest a change of regime whose direction is unsettled.'
|
|
),
|
|
'unknown': (
|
|
'Available ready orders and recent observations provide '
|
|
'insufficient or contradictory evidence for another label.'
|
|
),
|
|
})
|
|
ALIGNMENT_CRITERIA: MappingProxyType = MappingProxyType({
|
|
'aligned_up': 'Ready timeframes consistently support an uptrend',
|
|
'aligned_down': (
|
|
'Ready timeframes consistently support a downtrend.'
|
|
),
|
|
'mixed': (
|
|
'Ready timeframes disagree in direction or regime, such as '
|
|
'a shorter pullback within a longer directional trend.'
|
|
),
|
|
'unknown': (
|
|
'There is insufficient directional evidence to assess '
|
|
'agreement across the ready timeframes.'
|
|
),
|
|
})
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RegimeAnswer:
|
|
'''
|
|
Retain a model label and its entire probability distribution.
|
|
|
|
``RegimeAnswer.period=None`` identifies timeframe alignment.
|
|
``RegimeAnswer.probability`` is the selected label's probability;
|
|
``RegimeAnswer.confidence`` separately describes concentration.
|
|
|
|
'''
|
|
question: str
|
|
period: float|None
|
|
label: str
|
|
probability: float
|
|
confidence: float
|
|
probabilities: tuple[tuple[str, float], ...]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class JevClassification:
|
|
'''
|
|
Associate semantic judgments with evidence and availability time.
|
|
|
|
``JevClassification.source_as_of`` is an event-time bar END.
|
|
``JevClassification.requested_at`` and
|
|
``JevClassification.received_at`` are Unix wall-clock timestamps.
|
|
A live chart must not start a classification before receipt.
|
|
``JevClassification.latency`` uses a monotonic clock; wall-clock
|
|
adjustments do not change the measured request duration.
|
|
|
|
'''
|
|
market: str
|
|
revision: int
|
|
sequence: int
|
|
segment: int
|
|
source_as_of: float
|
|
requested_at: float
|
|
received_at: float
|
|
latency: float
|
|
model: str
|
|
prompt_version: str
|
|
answers: tuple[RegimeAnswer, ...]
|
|
input_tokens: int
|
|
output_tokens: int
|
|
|
|
|
|
class JevError(Exception):
|
|
'''
|
|
Describe a provider failure without echoing credential values.
|
|
|
|
'''
|
|
|
|
|
|
class JevProtocolError(JevError):
|
|
'''
|
|
Reject malformed or semantically inconsistent provider answers.
|
|
|
|
'''
|
|
|
|
|
|
class JevServiceError(JevError):
|
|
'''
|
|
Preserve an HTTP status and a bounded retry delay, not its body.
|
|
|
|
'''
|
|
def __init__(
|
|
self,
|
|
status: int,
|
|
retry_after: float = 0,
|
|
) -> None:
|
|
'''
|
|
Retain an HTTP failure status without its response body.
|
|
|
|
'''
|
|
self.status: int = status
|
|
self.retry_after: float = retry_after
|
|
super().__init__(f'TypeSafe request failed: HTTP {status}')
|
|
|
|
@property
|
|
def retryable(self) -> bool:
|
|
'''
|
|
Identify transient statuses documented by HTTP and TypeSafe.
|
|
|
|
'''
|
|
return self.status in {408, 429, 500, 502, 503, 504, 529}
|
|
|
|
|
|
def build_regime_request(
|
|
snapshot: RegimeSnapshot,
|
|
*,
|
|
model: str = DEFAULT_MODEL,
|
|
evidence_limit: int = 16,
|
|
) -> dict:
|
|
'''
|
|
Batch independent timeframe questions over one immutable state.
|
|
|
|
Question identifiers are opaque to Jev; each instruction names
|
|
its actual state path. Unready timeframes provide context only.
|
|
``evidence_limit`` bounds each timeframe's model-input tails
|
|
independently of larger histories retained for chart rendering.
|
|
|
|
'''
|
|
if not isinstance(model, str) or not model.strip():
|
|
raise ValueError('a model identifier is required')
|
|
if (
|
|
isinstance(evidence_limit, (bool, np.bool_))
|
|
or not 1 <= integer_index(evidence_limit) <= 256
|
|
):
|
|
raise ValueError('evidence_limit must be in [1, 256]')
|
|
frames: list[TimeframeRegime] = []
|
|
frame: TimeframeRegime
|
|
for frame in snapshot.timeframes:
|
|
frames.append(replace(
|
|
frame,
|
|
recent_closes=frame.recent_closes[-evidence_limit:],
|
|
reversals=frame.reversals[-evidence_limit:],
|
|
))
|
|
evidence: RegimeSnapshot = replace(
|
|
snapshot, timeframes=tuple(frames),
|
|
)
|
|
questions: dict[str, dict] = {}
|
|
number: int
|
|
for number, frame in enumerate(snapshot.timeframes):
|
|
if not frame.ready or not frame.ready[0]:
|
|
continue
|
|
questions[f'timeframe_{number}'] = {
|
|
'type': 'choice',
|
|
'instructions': (
|
|
f'Which descriptive market regime best fits '
|
|
f'`timeframes[{number}]` at its `as_of` time? '
|
|
f'Use only its observed recent closes, ready trend '
|
|
f'orders and detected reversals. TC order numbers '
|
|
f'are recursive reversal scales, not seconds. '
|
|
f'Zero is unresolved, not automatically a range. '
|
|
f'Ignore unready orders. A pivot was only known at '
|
|
f'its detected_time. Do not predict future returns '
|
|
f'or make a trading recommendation.'
|
|
),
|
|
'criteria': dict(REGIME_CRITERIA),
|
|
}
|
|
if not questions:
|
|
raise ValueError('no timeframe has completed TC warm-up')
|
|
if len(questions) > 1:
|
|
questions['alignment'] = {
|
|
'type': 'choice',
|
|
'instructions': (
|
|
'How do directions across the ready entries in '
|
|
'`timeframes` agree at `as_of`? Use only observed '
|
|
'closes and ready TC orders. Each timeframe may end '
|
|
'earlier than the overall as_of; do not invent '
|
|
'partial buckets or rely on other question answers.'
|
|
),
|
|
'criteria': dict(ALIGNMENT_CRITERIA),
|
|
}
|
|
return {
|
|
'model': model,
|
|
'state': evidence.to_state(),
|
|
'questions': questions,
|
|
}
|
|
|
|
|
|
def _probability(value: object) -> float:
|
|
'''
|
|
Enforce finite unit-interval probabilities without coercing text.
|
|
|
|
'''
|
|
if (
|
|
isinstance(value, bool)
|
|
or not isinstance(value, (float, int))
|
|
or not isfinite(value)
|
|
or not 0 <= value <= 1
|
|
):
|
|
raise JevProtocolError('invalid probability or confidence')
|
|
return float(value)
|
|
|
|
|
|
def _retry_after(value: str|None) -> float:
|
|
'''
|
|
Interpret Retry-After seconds or an HTTP date with a hard bound.
|
|
|
|
'''
|
|
if value is None:
|
|
return 0
|
|
try:
|
|
delay: float = float(value)
|
|
except ValueError:
|
|
try:
|
|
delay = parsedate_to_datetime(value).timestamp()
|
|
delay -= time.time()
|
|
except (
|
|
TypeError,
|
|
ValueError,
|
|
OverflowError,
|
|
):
|
|
return 0
|
|
if not isfinite(delay):
|
|
return 0
|
|
return min(60, max(0, delay))
|
|
|
|
|
|
class JevClient:
|
|
'''
|
|
Perform one bounded, validated HTTP evaluation on the Trio loop.
|
|
|
|
The caller owns ``http`` and must close it. Retries and pacing
|
|
belong to ``RegimeClassifier``, so a single evaluation never
|
|
silently resends stale state. Redirects are disabled to keep
|
|
the explicit credential bound to the TypeSafe API origin.
|
|
|
|
'''
|
|
def __init__(
|
|
self,
|
|
http: httpx.AsyncClient,
|
|
api_key: str,
|
|
*,
|
|
model: str = DEFAULT_MODEL,
|
|
timeout: float = 5,
|
|
max_request_bytes: int = 65_536,
|
|
max_response_bytes: int = 131_072,
|
|
evidence_limit: int = 16,
|
|
) -> None:
|
|
'''
|
|
Bind a caller-owned HTTP session, key and request limits.
|
|
|
|
The caller retains ownership of the supplied ``http`` client.
|
|
|
|
'''
|
|
if not isinstance(api_key, str) or not api_key.strip():
|
|
raise ValueError('explicit TypeSafe API key required')
|
|
if not isfinite(timeout) or timeout <= 0:
|
|
raise ValueError('timeout must be positive and finite')
|
|
if max_request_bytes <= 0 or max_response_bytes <= 0:
|
|
raise ValueError('byte limits must be positive')
|
|
if not isinstance(model, str) or not model.strip():
|
|
raise ValueError('a model identifier is required')
|
|
if (
|
|
isinstance(evidence_limit, (bool, np.bool_))
|
|
or not 1 <= integer_index(evidence_limit) <= 256
|
|
):
|
|
raise ValueError('evidence_limit must be in [1, 256]')
|
|
self._http: httpx.AsyncClient = http
|
|
self._api_key: str = api_key
|
|
self.model: str = model
|
|
self.timeout: float = timeout
|
|
self.max_request_bytes: int = max_request_bytes
|
|
self.max_response_bytes: int = max_response_bytes
|
|
self.evidence_limit: int = int(evidence_limit)
|
|
|
|
async def classify(
|
|
self,
|
|
snapshot: RegimeSnapshot,
|
|
) -> JevClassification:
|
|
'''
|
|
Validate one complete response against the exact sent rubric.
|
|
|
|
'''
|
|
payload: dict = build_regime_request(
|
|
snapshot, model=self.model,
|
|
evidence_limit=self.evidence_limit,
|
|
)
|
|
body: bytes = json.dumps(
|
|
payload,
|
|
allow_nan=False,
|
|
separators=(',', ':'),
|
|
).encode()
|
|
if len(body) > self.max_request_bytes:
|
|
raise ValueError('regime request exceeds byte budget')
|
|
requested_at: float = time.time()
|
|
started: float = trio.current_time()
|
|
data: bytearray = bytearray()
|
|
api_key: str = self._api_key
|
|
response: httpx.Response
|
|
with trio.fail_after(self.timeout):
|
|
async with self._http.stream(
|
|
'POST',
|
|
API_URL,
|
|
content=body,
|
|
headers={
|
|
'Authorization': f'Bearer {api_key}',
|
|
'Content-Type': 'application/json',
|
|
},
|
|
timeout=self.timeout,
|
|
follow_redirects=False,
|
|
) as response:
|
|
if response.status_code != 200:
|
|
raise JevServiceError(
|
|
response.status_code,
|
|
_retry_after(
|
|
response.headers.get('retry-after'),
|
|
),
|
|
)
|
|
chunk: bytes
|
|
async for chunk in response.aiter_bytes():
|
|
data.extend(chunk)
|
|
if len(data) > self.max_response_bytes:
|
|
raise JevProtocolError(
|
|
'response exceeds the byte budget',
|
|
)
|
|
received_at: float = time.time()
|
|
latency: float = trio.current_time() - started
|
|
error: ValueError|UnicodeError
|
|
try:
|
|
decoded: object = json.loads(data)
|
|
except (
|
|
ValueError,
|
|
UnicodeError,
|
|
) as error:
|
|
raise JevProtocolError('invalid JSON reply') from error
|
|
return self._parse(
|
|
snapshot,
|
|
payload,
|
|
decoded,
|
|
requested_at,
|
|
received_at,
|
|
latency,
|
|
)
|
|
|
|
def _parse(
|
|
self,
|
|
snapshot: RegimeSnapshot,
|
|
payload: dict,
|
|
decoded: object,
|
|
requested_at: float,
|
|
received_at: float,
|
|
latency: float,
|
|
) -> JevClassification:
|
|
'''
|
|
Reject missing questions, bad labels, or bad distributions.
|
|
|
|
'''
|
|
if not isinstance(decoded, dict):
|
|
raise JevProtocolError('response must be an object')
|
|
model: object = decoded.get('model')
|
|
if not isinstance(model, str) or not model:
|
|
raise JevProtocolError('response model is missing')
|
|
if (
|
|
self.model not in {'jev-latest', 'jev-preview'}
|
|
and model != self.model
|
|
):
|
|
raise JevProtocolError('response violates pinned model')
|
|
raw_answers: object = decoded.get('answers')
|
|
questions: dict = payload['questions']
|
|
if (
|
|
not isinstance(raw_answers, dict)
|
|
or set(raw_answers) != set(questions)
|
|
):
|
|
raise JevProtocolError('response question set differs')
|
|
answers: list[RegimeAnswer] = []
|
|
name: str
|
|
question: dict
|
|
for name, question in questions.items():
|
|
raw: object = raw_answers[name]
|
|
if (
|
|
not isinstance(raw, dict)
|
|
or raw.get('type') != 'choice'
|
|
):
|
|
raise JevProtocolError('expected a Choice answer')
|
|
label: object = raw.get('choice')
|
|
probabilities: object = raw.get('probabilities')
|
|
if (
|
|
not isinstance(probabilities, dict)
|
|
or set(probabilities) != set(question['criteria'])
|
|
or not isinstance(label, str)
|
|
or label not in probabilities
|
|
):
|
|
raise JevProtocolError('answer options differ')
|
|
distribution: dict[str, float] = {}
|
|
option: str
|
|
probability: object
|
|
for option, probability in probabilities.items():
|
|
distribution[option] = _probability(probability)
|
|
if not isclose(
|
|
sum(distribution.values()), 1, abs_tol=1e-5,
|
|
):
|
|
raise JevProtocolError('probabilities must sum to 1')
|
|
if distribution[label] < max(distribution.values()):
|
|
raise JevProtocolError('choice is not most probable')
|
|
period: float|None = None
|
|
if name != 'alignment':
|
|
position: int = int(name.removeprefix('timeframe_'))
|
|
period = snapshot.timeframes[position].period
|
|
answers.append(RegimeAnswer(
|
|
question=name,
|
|
period=period,
|
|
label=label,
|
|
probability=distribution[label],
|
|
confidence=_probability(raw.get('confidence')),
|
|
probabilities=tuple(distribution.items()),
|
|
))
|
|
usage: object = decoded.get('usage')
|
|
if not isinstance(usage, dict):
|
|
raise JevProtocolError('token usage is missing')
|
|
key: str
|
|
for key in ('input_tokens', 'output_tokens'):
|
|
count: object = usage.get(key)
|
|
if isinstance(count, bool) or not isinstance(count, int):
|
|
raise JevProtocolError('token usage must be integer')
|
|
if count < 0:
|
|
raise JevProtocolError('token usage must be >= 0')
|
|
return JevClassification(
|
|
market=snapshot.market,
|
|
revision=snapshot.revision,
|
|
sequence=snapshot.sequence,
|
|
segment=snapshot.segment,
|
|
source_as_of=snapshot.as_of,
|
|
requested_at=requested_at,
|
|
received_at=received_at,
|
|
latency=latency,
|
|
model=model,
|
|
prompt_version=PROMPT_VERSION,
|
|
answers=tuple(answers),
|
|
input_tokens=usage['input_tokens'],
|
|
output_tokens=usage['output_tokens'],
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _Submission:
|
|
'''
|
|
Retain one snapshot and its monotonic local submission time.
|
|
|
|
'''
|
|
snapshot: RegimeSnapshot
|
|
submitted: float
|
|
|
|
|
|
class RegimeClassifier:
|
|
'''
|
|
Coalesce producer updates into one paced, single-flight worker.
|
|
|
|
``RegimeClassifier.submit()`` replaces
|
|
``RegimeClassifier._pending`` without awaiting HTTP or consumers.
|
|
Responses must share the current revision/segment and remain
|
|
within ``RegimeClassifier.max_source_lag`` event-time seconds and
|
|
``RegimeClassifier.max_age`` local seconds of their evidence.
|
|
A small nonzero lag permits useful inference on fast feeds;
|
|
zero requires the response's exact latest event-time snapshot.
|
|
|
|
Run one worker in the owning actor's nursery. Cancellation closes
|
|
in-flight I/O through HTTPX; the caller owns the HTTP session.
|
|
|
|
'''
|
|
def __init__(
|
|
self,
|
|
client: JevClient,
|
|
*,
|
|
min_interval: float = 1,
|
|
max_age: float = 5,
|
|
max_source_lag: float = 2,
|
|
max_retries: int = 2,
|
|
) -> None:
|
|
'''
|
|
Configure bounded request pacing and evidence freshness.
|
|
|
|
'''
|
|
value: float
|
|
for value in (min_interval, max_source_lag):
|
|
if not isfinite(value) or value < 0:
|
|
raise ValueError('interval and lag must be >= 0')
|
|
if not isfinite(max_age) or max_age <= 0:
|
|
raise ValueError('max_age must be positive and finite')
|
|
if (
|
|
isinstance(max_retries, bool)
|
|
or not 0 <= integer_index(max_retries) <= 8
|
|
):
|
|
raise ValueError('max_retries must be in [0, 8]')
|
|
self.client: JevClient = client
|
|
self.min_interval: float = min_interval
|
|
self.max_age: float = max_age
|
|
self.max_source_lag: float = max_source_lag
|
|
self.max_retries: int = max_retries
|
|
self.requests: int = 0
|
|
self.published: int = 0
|
|
self.discarded: int = 0
|
|
self.failures: int = 0
|
|
self.last_error: Exception|None = None
|
|
self._latest: _Submission|None = None
|
|
self._pending: _Submission|None = None
|
|
self._wake: trio.Event = trio.Event()
|
|
self._running: bool = False
|
|
self._minimum_revision: int = 0
|
|
self._publishing: _Submission|None = None
|
|
self._publish_scope: trio.CancelScope|None = None
|
|
|
|
def invalidate(self, revision: int) -> None:
|
|
'''
|
|
Revoke older evidence immediately, before source replay.
|
|
|
|
A feed owner calls this upon receiving a history revision;
|
|
it need not wait for a replacement snapshot to become ready.
|
|
|
|
'''
|
|
if (
|
|
isinstance(revision, bool)
|
|
or integer_index(revision) < 0
|
|
):
|
|
raise ValueError('revision must be an integer >= 0')
|
|
self._minimum_revision = max(
|
|
self._minimum_revision, revision,
|
|
)
|
|
if (
|
|
self._pending is not None
|
|
and self._pending.snapshot.revision < revision
|
|
):
|
|
self._pending = None
|
|
self._cancel_stale_publication()
|
|
|
|
def _cancel_stale_publication(self) -> None:
|
|
'''
|
|
Revoke stale ``RegimeClassifier._publish_scope`` delivery.
|
|
|
|
``RegimeClassifier._publishing`` retains the exact submission
|
|
whose consumer may currently be blocked at an ``await``.
|
|
|
|
'''
|
|
if (
|
|
self._publishing is not None
|
|
and self._publish_scope is not None
|
|
and not self._fresh(self._publishing)
|
|
):
|
|
self._publish_scope.cancel()
|
|
|
|
def submit(self, snapshot: RegimeSnapshot) -> bool:
|
|
'''
|
|
Replace ``RegimeClassifier._pending`` with newer evidence.
|
|
|
|
Refuse older source coordinates or a different market before
|
|
replacing ``RegimeClassifier._latest`` and its pending work.
|
|
|
|
'''
|
|
if snapshot.revision < self._minimum_revision:
|
|
return False
|
|
latest: _Submission|None = self._latest
|
|
if latest is not None:
|
|
previous: RegimeSnapshot = latest.snapshot
|
|
if snapshot.market != previous.market:
|
|
raise ValueError('one classifier owns one market')
|
|
if (snapshot.revision, snapshot.sequence) <= (
|
|
previous.revision, previous.sequence,
|
|
):
|
|
return False
|
|
if (
|
|
snapshot.revision == previous.revision
|
|
and snapshot.as_of <= previous.as_of
|
|
):
|
|
raise ValueError('as_of must advance in a revision')
|
|
submission: _Submission = _Submission(
|
|
snapshot,
|
|
trio.current_time(),
|
|
)
|
|
self._latest = self._pending = submission
|
|
self._cancel_stale_publication()
|
|
self._wake.set()
|
|
return True
|
|
|
|
def _fresh(self, submission: _Submission) -> bool:
|
|
'''
|
|
Check revision, gap segment, source lag, and elapsed age.
|
|
|
|
'''
|
|
latest: _Submission|None = self._latest
|
|
assert latest is not None
|
|
source: RegimeSnapshot = submission.snapshot
|
|
newest: RegimeSnapshot = latest.snapshot
|
|
lag: float = newest.as_of - source.as_of
|
|
age: float = trio.current_time() - submission.submitted
|
|
return (
|
|
source.revision >= self._minimum_revision
|
|
and source.revision == newest.revision
|
|
and source.segment == newest.segment
|
|
and 0 <= lag <= self.max_source_lag
|
|
and age <= self.max_age
|
|
)
|
|
|
|
async def run(
|
|
self,
|
|
publish: Callable[[JevClassification], Awaitable[None]],
|
|
*,
|
|
on_error: Callable[[Exception], Awaitable[None]]|None = None,
|
|
task_status: trio.TaskStatus[None] = (
|
|
trio.TASK_STATUS_IGNORED
|
|
),
|
|
) -> None:
|
|
'''
|
|
Serve until cancelled, with bounded transient retries.
|
|
|
|
Authentication and malformed-response failures propagate;
|
|
transient errors call ``on_error`` and retry newest state.
|
|
Retry-After delays have a 60-second cap. Each retry rechecks
|
|
freshness; an expired snapshot is never resent.
|
|
|
|
``publish`` must cooperate with Trio cancellation. Its wait
|
|
is cancelled if evidence expires or is invalidated. Consumers
|
|
must also validate received message provenance: a delivery
|
|
already completed before invalidation cannot be revoked.
|
|
|
|
'''
|
|
if self._running:
|
|
raise RuntimeError('classifier worker already running')
|
|
self._running = True
|
|
task_status.started()
|
|
next_request: float = 0
|
|
retries: int = 0
|
|
try:
|
|
while True:
|
|
if self._pending is None:
|
|
await self._wake.wait()
|
|
self._wake = trio.Event()
|
|
await trio.sleep_until(next_request)
|
|
submission: _Submission|None = self._pending
|
|
self._pending = None
|
|
if submission is None:
|
|
continue
|
|
if not self._fresh(submission):
|
|
self.discarded += 1
|
|
continue
|
|
ready: bool = False
|
|
frame: TimeframeRegime
|
|
for frame in submission.snapshot.timeframes:
|
|
ready |= bool(frame.ready and frame.ready[0])
|
|
if not ready:
|
|
continue
|
|
next_request = (
|
|
trio.current_time() + self.min_interval
|
|
)
|
|
self.requests += 1
|
|
error: (
|
|
httpx.TransportError
|
|
|trio.TooSlowError
|
|
|JevServiceError
|
|
)
|
|
try:
|
|
result: JevClassification = await (
|
|
self.client.classify(submission.snapshot)
|
|
)
|
|
except (
|
|
httpx.TransportError,
|
|
trio.TooSlowError,
|
|
JevServiceError,
|
|
) as error:
|
|
self.failures += 1
|
|
self.last_error = error
|
|
if (
|
|
isinstance(error, JevServiceError)
|
|
and not error.retryable
|
|
):
|
|
raise
|
|
if on_error is not None:
|
|
await on_error(error)
|
|
if retries < self.max_retries:
|
|
delay: float = min(60, 2 ** retries)
|
|
if isinstance(error, JevServiceError):
|
|
delay = max(delay, error.retry_after)
|
|
next_request = max(
|
|
next_request,
|
|
trio.current_time() + delay,
|
|
)
|
|
if self._pending is None:
|
|
self._pending = self._latest
|
|
retries += 1
|
|
else:
|
|
retries = 0
|
|
continue
|
|
|
|
retries = 0
|
|
self.last_error = None
|
|
if self._fresh(submission):
|
|
deadline: float = (
|
|
submission.submitted + self.max_age
|
|
)
|
|
scope: trio.CancelScope = trio.CancelScope(
|
|
deadline=deadline,
|
|
)
|
|
with scope:
|
|
self._publish_scope = scope
|
|
self._publishing = submission
|
|
try:
|
|
await publish(result)
|
|
finally:
|
|
self._publishing = None
|
|
self._publish_scope = None
|
|
if scope.cancelled_caught:
|
|
self.discarded += 1
|
|
else:
|
|
self.published += 1
|
|
else:
|
|
self.discarded += 1
|
|
finally:
|
|
self._running = False
|