1019 lines
31 KiB
Python
1019 lines
31 KiB
Python
# piker: trading gear for hackers
|
|
# Copyright (C) Tyler Goodlet (in stewardship of pikers)
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
'''
|
|
core task logic for processing chains
|
|
|
|
'''
|
|
from __future__ import annotations
|
|
from contextlib import asynccontextmanager as acm
|
|
from functools import partial
|
|
from typing import (
|
|
Any,
|
|
AsyncIterator,
|
|
Awaitable,
|
|
cast,
|
|
Callable,
|
|
TYPE_CHECKING,
|
|
)
|
|
|
|
import numpy as np
|
|
import trio
|
|
from trio_typing import TaskStatus
|
|
import tractor
|
|
from tractor.msg import NamespacePath
|
|
from tractor.trionics import Lagged
|
|
|
|
from piker.types import Struct
|
|
from ..log import (
|
|
get_logger,
|
|
get_console_log,
|
|
)
|
|
from .. import data
|
|
from ..data.ticktools import FeedQuote
|
|
from ..data.flows import Flume
|
|
from ..data._sharedmem import NDTokenMsg
|
|
from tractor.ipc._shm import ShmArray
|
|
from ..data._sampling import (
|
|
open_sample_stream,
|
|
)
|
|
from ..accounting import MktPair
|
|
from ._api import (
|
|
Fsp,
|
|
FspFunc,
|
|
FspStream,
|
|
FspYield,
|
|
FspHistory,
|
|
FspSample,
|
|
_load_builtins,
|
|
NDToken,
|
|
)
|
|
from ..toolz import Profiler
|
|
|
|
if TYPE_CHECKING:
|
|
from ..data.feed import Feed
|
|
|
|
log = get_logger(__name__)
|
|
|
|
type QuoteFrame = dict[str, FeedQuote]
|
|
type BackfillNotice = tuple[str, float]|list[str|float]
|
|
type SampleValue = int|float|BackfillNotice
|
|
type SampleMsg = dict[str, SampleValue]
|
|
type EdgeFunc = FspFunc
|
|
|
|
|
|
def _is_relevant_sample_msg(
|
|
msg: SampleMsg,
|
|
fqme: str,
|
|
period_s: float,
|
|
|
|
) -> bool:
|
|
backfill: SampleValue|None = msg.get('backfilling')
|
|
if backfill is None:
|
|
return True
|
|
|
|
if (
|
|
not isinstance(backfill, (list, tuple))
|
|
or
|
|
len(backfill) != 2
|
|
):
|
|
return False
|
|
|
|
backfill_fqme: str | float
|
|
timeframe: str | float
|
|
backfill_fqme, timeframe = backfill
|
|
return (
|
|
backfill_fqme == fqme
|
|
and
|
|
timeframe == period_s
|
|
)
|
|
|
|
|
|
def _should_advance_dst(
|
|
msg: SampleMsg,
|
|
step_diff: int,
|
|
|
|
) -> bool:
|
|
return (
|
|
'backfilling' not in msg
|
|
and
|
|
step_diff == 1
|
|
)
|
|
|
|
|
|
def _needs_history_resync(
|
|
msg: SampleMsg,
|
|
synced: bool,
|
|
|
|
) -> bool:
|
|
return (
|
|
'backfilling' in msg
|
|
or
|
|
not synced
|
|
)
|
|
|
|
|
|
async def filter_quotes_by_sym(
|
|
|
|
sym: str,
|
|
quote_stream: AsyncIterator[QuoteFrame],
|
|
|
|
) -> AsyncIterator[FeedQuote]:
|
|
'''
|
|
Filter quote stream by target symbol.
|
|
|
|
'''
|
|
# TODO: make this the actual first quote from feed
|
|
# XXX: this allows for a single iteration to run for history
|
|
# processing without waiting on the real-time feed for a new quote
|
|
yield {}
|
|
|
|
async for quotes in quote_stream:
|
|
quote: FeedQuote|None = quotes.get(sym)
|
|
if quote:
|
|
yield quote
|
|
|
|
|
|
@acm
|
|
async def _sample_wakeups(
|
|
quotes: AsyncIterator[QuoteFrame],
|
|
fqme: str,
|
|
enabled: bool,
|
|
) -> AsyncIterator[
|
|
tuple[AsyncIterator[QuoteFrame], Callable[[], None]|None]
|
|
]:
|
|
'''
|
|
Merge quote and sampler wakeups for SHM snapshot processors.
|
|
|
|
One pending notification is enough: the processor reads every
|
|
newly finalized SHM row when woken. Keep this queue outside the
|
|
restartable compute task, so history resync cannot lose a step.
|
|
Tick-sensitive processors retain their original quote stream.
|
|
|
|
'''
|
|
if not enabled:
|
|
yield quotes, None
|
|
return
|
|
|
|
send: trio.MemorySendChannel[QuoteFrame]
|
|
receive: trio.MemoryReceiveChannel[QuoteFrame]
|
|
send, receive = trio.open_memory_channel[QuoteFrame](1)
|
|
|
|
def notify() -> None:
|
|
'''
|
|
Coalesce sampler steps in the local ``send`` channel.
|
|
|
|
'''
|
|
try:
|
|
send.send_nowait({fqme: {'ticks': []}})
|
|
except trio.WouldBlock:
|
|
pass
|
|
|
|
async def relay() -> None:
|
|
'''
|
|
Resume after quote overrun and wake the SHM snapshot reader.
|
|
|
|
'''
|
|
iterator: AsyncIterator[QuoteFrame] = aiter(quotes)
|
|
while True:
|
|
try:
|
|
frame: QuoteFrame = await anext(iterator)
|
|
except Lagged:
|
|
# Dropped quote payloads do not invalidate SHM.
|
|
# Wake the snapshot processor to catch up all rows.
|
|
notify()
|
|
await trio.lowlevel.checkpoint()
|
|
continue
|
|
except StopAsyncIteration:
|
|
return
|
|
if fqme in frame:
|
|
notify()
|
|
|
|
nursery: trio.Nursery
|
|
async with send, receive, trio.open_nursery() as nursery:
|
|
nursery.start_soon(relay)
|
|
try:
|
|
yield receive, notify
|
|
finally:
|
|
nursery.cancel_scope.cancel()
|
|
|
|
|
|
def _next_step_output(
|
|
previous: np.ndarray,
|
|
fsp: Fsp,
|
|
zero_on_step: bool,
|
|
) -> np.ndarray:
|
|
'''
|
|
Carry state into a new row without duplicating sparse events.
|
|
|
|
'''
|
|
row: np.ndarray = (
|
|
np.zeros_like(previous) if zero_on_step else previous.copy()
|
|
)
|
|
field: str
|
|
for field in fsp.event_outputs:
|
|
row[field] = np.nan
|
|
return row
|
|
|
|
|
|
class _HistoryShm:
|
|
'''
|
|
Present an immutable history view before switching to live SHM.
|
|
|
|
'''
|
|
def __init__(self, shm: ShmArray) -> None:
|
|
self._shm: ShmArray = shm
|
|
self.first: int = shm._first.value
|
|
self.last_index: int = shm._last.value
|
|
self.history: np.ndarray = shm._array[
|
|
self.first:self.last_index
|
|
].copy()
|
|
self.live: bool = False
|
|
|
|
@property
|
|
def array(self) -> np.ndarray:
|
|
if self.live:
|
|
return self._shm.array
|
|
return self.history
|
|
|
|
@property
|
|
def index(self) -> int:
|
|
if self.live:
|
|
return self._shm.index
|
|
return self.last_index % self._shm._len
|
|
|
|
def last(self, length: int = 1) -> np.ndarray:
|
|
return self.array[-length:]
|
|
|
|
def __getattr__(self, name: str) -> Any:
|
|
return getattr(self._shm, name)
|
|
|
|
|
|
async def _open_history_snapshot(
|
|
edge_func: EdgeFunc,
|
|
quote_stream: AsyncIterator[QuoteFrame],
|
|
fqme: str,
|
|
src_shm: ShmArray,
|
|
|
|
) -> tuple[
|
|
FspStream,
|
|
FspHistory,
|
|
tuple[int, int],
|
|
np.ndarray,
|
|
]:
|
|
'''
|
|
Start an FSP against an immutable source-history snapshot.
|
|
|
|
'''
|
|
history_shm: _HistoryShm = _HistoryShm(src_shm)
|
|
out_stream: FspStream = edge_func(
|
|
filter_quotes_by_sym(fqme, quote_stream),
|
|
cast(ShmArray, history_shm),
|
|
)
|
|
try:
|
|
first_yield: FspYield = await anext(out_stream)
|
|
except BaseException:
|
|
with trio.CancelScope(shield=True):
|
|
await out_stream.aclose()
|
|
raise
|
|
|
|
if isinstance(first_yield, tuple):
|
|
await out_stream.aclose()
|
|
raise TypeError(
|
|
f'FSP `{edge_func.__name__}` yielded a realtime update '
|
|
f'before its historical output'
|
|
)
|
|
|
|
history_output: FspHistory = first_yield
|
|
history_shm.live = True
|
|
return (
|
|
out_stream,
|
|
history_output,
|
|
(
|
|
history_shm.first,
|
|
history_shm.last_index,
|
|
),
|
|
history_shm.history,
|
|
)
|
|
|
|
# TODO: unifying the abstractions in this FSP subsys/layer:
|
|
# -[ ] move the `.data.flows.Flume` type into this
|
|
# module/subsys/pkg?
|
|
# -[ ] ideas for further abstractions as per
|
|
# - https://github.com/pikers/piker/issues/216,
|
|
# - https://github.com/pikers/piker/issues/270:
|
|
# - a (financial signal) ``Flow`` would be the a "collection" of such
|
|
# minmial cascades. Some engineering based jargon concepts:
|
|
# - https://en.wikipedia.org/wiki/Signal_chain
|
|
# - https://en.wikipedia.org/wiki/Daisy_chain_(electrical_engineering)
|
|
# - https://en.wikipedia.org/wiki/Audio_signal_flow
|
|
# - https://en.wikipedia.org/wiki/Digital_signal_processing#Implementation
|
|
# - https://en.wikipedia.org/wiki/Dataflow_programming
|
|
# - https://en.wikipedia.org/wiki/Signal_programming
|
|
# - https://en.wikipedia.org/wiki/Incremental_computing
|
|
# - https://en.wikipedia.org/wiki/Signal-flow_graph
|
|
# - https://en.wikipedia.org/wiki/Signal-flow_graph#Basic_components
|
|
|
|
# -[ ] we probably want to eval THE BELOW design and unify with the
|
|
# proto `TaskManager` in the `tractor` dev branch as well as with
|
|
# our below idea for `Cascade`:
|
|
# - https://github.com/goodboy/tractor/pull/363
|
|
class Cascade(Struct):
|
|
'''
|
|
As per sig-proc engineering parlance, this is a chaining of
|
|
`Flume`s, which are themselves collections of "Streams"
|
|
implemented currently via `ShmArray`s.
|
|
|
|
A `Cascade` is be the minimal "connection" of 2 `Flumes`
|
|
as per circuit parlance:
|
|
https://en.wikipedia.org/wiki/Two-port_network#Cascade_connection
|
|
|
|
TODO:
|
|
-[ ] could cover the combination of our `FspAdmin` and the
|
|
backend `.fsp._engine` related machinery to "connect" one flume
|
|
to another?
|
|
|
|
'''
|
|
# TODO: make these `Flume`s
|
|
src: Flume
|
|
dst: Flume
|
|
tn: trio.Nursery
|
|
fsp: Fsp # UI-side middleware ctl API
|
|
|
|
# filled during cascade/.bind_func() (fsp_compute) init phases
|
|
bind_func: Callable[..., Awaitable[None]]|None = None
|
|
complete: trio.Event|None = None
|
|
cs: trio.CancelScope|None = None
|
|
client_stream: tractor.MsgStream|None = None
|
|
sample_ready: trio.Event|None = None
|
|
|
|
async def resync(self) -> int:
|
|
# TODO: adopt an incremental update engine/approach
|
|
# where possible here eventually!
|
|
log.info(f're-syncing fsp {self.fsp.name} to source')
|
|
cs: trio.CancelScope|None = self.cs
|
|
complete: trio.Event|None = self.complete
|
|
bind_func = self.bind_func
|
|
client_stream = self.client_stream
|
|
assert cs is not None
|
|
assert complete is not None
|
|
assert bind_func is not None
|
|
assert client_stream is not None
|
|
|
|
cs.cancel()
|
|
await complete.wait()
|
|
index: int = await self.tn.start(bind_func)
|
|
|
|
# always trigger UI refresh after history update,
|
|
# see ``piker.ui._fsp.FspAdmin.open_chain()`` and
|
|
# ``piker.ui._display.trigger_update()``.
|
|
dst_shm: ShmArray = self.dst.rt_shm
|
|
await client_stream.send({
|
|
'fsp_update': {
|
|
'key': dst_shm.token,
|
|
'first': dst_shm._first.value,
|
|
'last': dst_shm._last.value,
|
|
}
|
|
})
|
|
return index
|
|
|
|
def is_synced(self) -> tuple[bool, int, int]:
|
|
'''
|
|
Predicate to dertmine if a destination FSP
|
|
output array is aligned to its source array.
|
|
|
|
'''
|
|
src_shm: ShmArray = self.src.rt_shm
|
|
dst_shm: ShmArray = self.dst.rt_shm
|
|
src_first = src_shm._first.value
|
|
src_last = src_shm._last.value
|
|
dst_first = dst_shm._first.value
|
|
dst_last = dst_shm._last.value
|
|
|
|
# Compare absolute bounds instead of ``ShmArray.index``, which
|
|
# wraps at the allocation size. A one-step source lead is the
|
|
# normal state before the destination sample is appended.
|
|
step_diff = src_last - dst_last
|
|
len_diff = abs(
|
|
(src_last - src_first)
|
|
- (dst_last - dst_first)
|
|
)
|
|
synced: bool = (
|
|
src_first == dst_first
|
|
and
|
|
step_diff in (0, 1)
|
|
)
|
|
if not synced:
|
|
fsp: Fsp = self.fsp
|
|
report = (
|
|
f'ns-path: {fsp.ns_path!r}\n'
|
|
f'shm-token: {src_shm.token}\n'
|
|
f'src-bounds: {(src_first, src_last)}\n'
|
|
f'dst-bounds: {(dst_first, dst_last)}\n'
|
|
f'step_diff: {step_diff}\n'
|
|
f'len_diff: {len_diff}\n'
|
|
)
|
|
if (
|
|
src_first < dst_first
|
|
and
|
|
src_last == dst_last
|
|
):
|
|
log.info(
|
|
f'FSP source history grew:\n'
|
|
f'{report}'
|
|
)
|
|
else:
|
|
log.warning(
|
|
f'***DESYNCED fsp***\n'
|
|
f'------------------\n'
|
|
f'{report}'
|
|
)
|
|
return (
|
|
synced,
|
|
step_diff,
|
|
len_diff,
|
|
)
|
|
|
|
async def poll_and_sync_to_step(self) -> int:
|
|
while True:
|
|
await self.resync()
|
|
(
|
|
synced,
|
|
step_diff,
|
|
_,
|
|
) = self.is_synced()
|
|
if synced:
|
|
return step_diff
|
|
|
|
@acm
|
|
async def open_edge(
|
|
self,
|
|
bind_func: Callable[..., Awaitable[None]],
|
|
) -> int:
|
|
self.bind_func = bind_func
|
|
index = await self.tn.start(bind_func)
|
|
try:
|
|
yield index
|
|
finally:
|
|
# Stop the current compute task, including after resync,
|
|
# before the outer scope closes its source receiver.
|
|
if self.cs is not None:
|
|
self.cs.cancel()
|
|
if self.complete is not None:
|
|
with trio.CancelScope(shield=True):
|
|
await self.complete.wait()
|
|
# TODO: what do we want on teardown/error?
|
|
# -[ ] dynamic reconnection after update?
|
|
|
|
|
|
async def _close_fsp_stream(out_stream: FspStream) -> None:
|
|
with trio.CancelScope(shield=True):
|
|
await out_stream.aclose()
|
|
|
|
|
|
async def _write_sample(
|
|
casc: Cascade,
|
|
sample: FspSample,
|
|
) -> bool:
|
|
'''
|
|
Wait for destination advancement, then publish at source index.
|
|
|
|
``cascade()`` sets ``Cascade.sample_ready`` after publishing the
|
|
new destination bound. Capture that event before checking bounds
|
|
so a sampler wakeup cannot be lost. Cancellation during history
|
|
resync aborts this wait with the obsolete compute generator.
|
|
|
|
'''
|
|
dst: ShmArray = casc.dst.rt_shm
|
|
while True:
|
|
ready: trio.Event|None = casc.sample_ready
|
|
if ready is None:
|
|
ready = casc.sample_ready = trio.Event()
|
|
first: int = dst._first.value
|
|
last: int = dst._last.value
|
|
if sample.index < first:
|
|
return False
|
|
if sample.index < last:
|
|
row: np.void = dst._array[sample.index]
|
|
field: str
|
|
value: Any
|
|
for field, value in sample.fields.items():
|
|
row[field] = value
|
|
row['time'] = sample.time
|
|
return True
|
|
await ready.wait()
|
|
|
|
|
|
def _write_history_output(
|
|
dst_shm: ShmArray,
|
|
history_output: FspHistory,
|
|
src_bounds: tuple[int, int],
|
|
src_history: np.ndarray,
|
|
func_name: str,
|
|
profiler: Profiler,
|
|
|
|
) -> int:
|
|
src_first, src_last = src_bounds
|
|
fields = dst_shm.array.dtype.fields
|
|
if fields is None:
|
|
raise TypeError('FSP destination SHM must be a structured array')
|
|
fields = fields.copy()
|
|
fields.pop('index')
|
|
fields.pop('time')
|
|
history_by_field: np.ndarray|None = None
|
|
|
|
if len(fields) > 1:
|
|
if not isinstance(history_output, dict):
|
|
raise ValueError(
|
|
f'`{func_name}` is a multi-output FSP and should '
|
|
f'yield a `dict[str, np.ndarray]` for history'
|
|
)
|
|
|
|
output_fields: list[str] = [
|
|
key
|
|
for key in fields
|
|
if key in history_output
|
|
]
|
|
if not output_fields:
|
|
raise ValueError(
|
|
f'`{func_name}` produced no declared history fields'
|
|
)
|
|
|
|
for key in output_fields:
|
|
output: np.ndarray|None = history_output[key]
|
|
if history_by_field is None:
|
|
length: int = (
|
|
len(src_history)
|
|
if output is None
|
|
else len(output)
|
|
)
|
|
|
|
# using the first output, determine the length of the
|
|
# struct-array that will be pushed to shm.
|
|
history_by_field = np.zeros(
|
|
length,
|
|
dtype=dst_shm.array.dtype,
|
|
)
|
|
|
|
if output is not None:
|
|
history_by_field[key] = output
|
|
|
|
else:
|
|
if not isinstance(history_output, np.ndarray):
|
|
raise ValueError(
|
|
f'`{func_name}` is a single output FSP and should '
|
|
f'yield an `np.ndarray` for history'
|
|
)
|
|
history_by_field = np.zeros(
|
|
len(history_output),
|
|
dtype=dst_shm.array.dtype,
|
|
)
|
|
history_by_field[func_name] = history_output
|
|
|
|
if history_by_field is None:
|
|
raise ValueError(f'`{func_name}` produced no history fields')
|
|
if len(history_by_field) != len(src_history):
|
|
raise ValueError(
|
|
f'`{func_name}` history length '
|
|
f'{len(history_by_field)} does not match source length '
|
|
f'{len(src_history)}'
|
|
)
|
|
|
|
history_by_field['time'] = src_history['time']
|
|
|
|
# TODO: XXX:
|
|
# THERE'S A BIG BUG HERE WITH THE `index` field since we're
|
|
# prepending a copy of the first value a few times to make
|
|
# sub-curves align with the parent bar chart.
|
|
# This likely needs to be fixed either by,
|
|
# - manually assigning the index and historical data
|
|
# seperately to the shm array (i.e. not using .push())
|
|
# - developing some system on top of the shared mem array that
|
|
# is `index` aware such that historical data can be indexed
|
|
# relative to the true first datum? Not sure if this is sane
|
|
# for incremental compuations.
|
|
first: int = src_first
|
|
dst_shm._first.value = first
|
|
|
|
# TODO: can we use this `start` flag instead of the manual
|
|
# setting above?
|
|
index: int = dst_shm.push(
|
|
history_by_field,
|
|
start=first,
|
|
)
|
|
|
|
profiler(f'{func_name} pushed history')
|
|
profiler.finish()
|
|
assert index == src_last
|
|
return index
|
|
|
|
|
|
async def connect_streams(
|
|
casc: Cascade,
|
|
mkt: MktPair,
|
|
quote_stream: AsyncIterator[QuoteFrame],
|
|
src: Flume,
|
|
dst: Flume,
|
|
|
|
edge_func: EdgeFunc,
|
|
|
|
# attach_stream: bool = False,
|
|
task_status: TaskStatus[None] = trio.TASK_STATUS_IGNORED,
|
|
|
|
) -> None:
|
|
'''
|
|
Stream and per-sample compute and write the cascade of
|
|
2 `Flumes`/streams given some operating `func`.
|
|
|
|
https://en.wikipedia.org/wiki/Signal-flow_graph#Basic_components
|
|
|
|
Not literally, but something like:
|
|
|
|
edge_func(Flume_in) -> Flume_out
|
|
|
|
'''
|
|
profiler = Profiler(
|
|
delayed=False,
|
|
disabled=True
|
|
)
|
|
|
|
# TODO: just pull it from src.mkt.fqme no?
|
|
# fqme: str = mkt.fqme
|
|
fqme: str = src.mkt.fqme
|
|
|
|
# HISTORY COMPUTE PHASE
|
|
# conduct a single iteration of fsp with historical bars input
|
|
# and get historical output.
|
|
history_output: FspHistory
|
|
src_shm: ShmArray = src.rt_shm
|
|
(
|
|
out_stream,
|
|
history_output,
|
|
src_bounds,
|
|
src_history,
|
|
) = await _open_history_snapshot(
|
|
edge_func,
|
|
quote_stream,
|
|
fqme,
|
|
src_shm,
|
|
)
|
|
func_name: str = edge_func.__name__
|
|
profiler(f'{func_name} generated history')
|
|
|
|
dst_shm: ShmArray = dst.rt_shm
|
|
try:
|
|
index: int = _write_history_output(
|
|
dst_shm,
|
|
history_output,
|
|
src_bounds,
|
|
src_history,
|
|
func_name,
|
|
profiler,
|
|
)
|
|
except BaseException:
|
|
await _close_fsp_stream(out_stream)
|
|
raise
|
|
|
|
# setup a respawn handle
|
|
with trio.CancelScope() as cs:
|
|
|
|
casc.cs = cs
|
|
casc.complete = trio.Event()
|
|
task_status.started(index)
|
|
|
|
profiler(f'{func_name} yield last index')
|
|
|
|
# import time
|
|
# last = time.time()
|
|
|
|
try:
|
|
|
|
async for processed in out_stream:
|
|
|
|
log.debug(f"{func_name}: {processed}")
|
|
if isinstance(processed, FspSample):
|
|
await _write_sample(casc, processed)
|
|
continue
|
|
key, output = processed
|
|
# dst.array[-1][key] = output
|
|
dst_shm.array[[key, 'time']][-1] = (
|
|
output,
|
|
# TODO: what about pushing ``time.time_ns()``
|
|
# in which case we'll need to round at the graphics
|
|
# processing / sampling layer?
|
|
src_shm.array[-1]['time']
|
|
)
|
|
|
|
# NOTE: for now we aren't streaming this to the consumer
|
|
# stream latest array index entry which basically just acts
|
|
# as trigger msg to tell the consumer to read from shm
|
|
# TODO: further this should likely be implemented much
|
|
# like our `Feed` api where there is one background
|
|
# "service" task which computes output and then sends to
|
|
# N-consumers who subscribe for the real-time output,
|
|
# which we'll likely want to implement using local-mem
|
|
# chans for the fan out?
|
|
# index = src_shm.index
|
|
# if attach_stream:
|
|
# await client_stream.send(index)
|
|
|
|
# period = time.time() - last
|
|
# hz = 1/period if period else float('nan')
|
|
# if hz > 60:
|
|
# log.info(f'FSP quote too fast: {hz}')
|
|
# last = time.time()
|
|
finally:
|
|
try:
|
|
await _close_fsp_stream(out_stream)
|
|
finally:
|
|
casc.complete.set()
|
|
|
|
|
|
@tractor.context
|
|
async def cascade(
|
|
ctx: tractor.Context,
|
|
|
|
# data feed key
|
|
fqme: str,
|
|
|
|
# flume pair cascaded using an "edge function"
|
|
src_flume_addr: dict[str, Any],
|
|
dst_flume_addr: dict[str, Any],
|
|
ns_path: NamespacePath,
|
|
|
|
shm_registry: list[
|
|
tuple[NDTokenMsg, str, NDTokenMsg]
|
|
],
|
|
|
|
zero_on_step: bool = False,
|
|
loglevel: str|None = None,
|
|
|
|
) -> None:
|
|
'''
|
|
Chain streaming signal processors and deliver output to
|
|
destination shm array buffer.
|
|
|
|
'''
|
|
profiler = Profiler(
|
|
delayed=False,
|
|
disabled=False
|
|
)
|
|
|
|
if loglevel:
|
|
log = get_console_log(
|
|
loglevel,
|
|
name=__name__,
|
|
)
|
|
# XXX TODO!
|
|
# figure out why this writes a dict to,
|
|
# `tractor._state._runtime_vars['_root_mailbox']`
|
|
# XD .. wtf
|
|
# TODO, solve this as reported in,
|
|
# https://www.pikers.dev/pikers/piker/issues/70
|
|
# await tractor.pause()
|
|
|
|
src: Flume = Flume.from_msg(src_flume_addr)
|
|
dst: Flume = Flume.from_msg(
|
|
dst_flume_addr,
|
|
readonly=False,
|
|
)
|
|
|
|
# src: ShmArray = attach_shm_array(token=src_shm_token)
|
|
# dst: ShmArray = attach_shm_array(readonly=False, token=dst_shm_token)
|
|
|
|
reg: dict[NamespacePath, Fsp] = _load_builtins()
|
|
lines = '\n'.join([f'{key.rpartition(":")[2]} => {key}' for key in reg])
|
|
log.info(
|
|
f'Registered FSP set:\n{lines}'
|
|
)
|
|
|
|
# NOTE XXX: update actorlocal flows table which registers
|
|
# readonly "instances" of this fsp for symbol/source so that
|
|
# consumer fsps can look it up by source + fsp.
|
|
# TODO: ugh i hate this wind/unwind to list over the wire but
|
|
# not sure how else to do it.
|
|
for (token, fsp_name, dst_token) in shm_registry:
|
|
Fsp._flow_registry[(
|
|
NDToken.from_msg(dict(token)),
|
|
fsp_name,
|
|
)] = NDToken.from_msg(dict(dst_token)), None
|
|
|
|
fsp: Fsp|None = reg.get(
|
|
NamespacePath(ns_path)
|
|
)
|
|
if (
|
|
fsp is None
|
|
or
|
|
not fsp.func
|
|
):
|
|
# TODO: assume it's a func target path
|
|
raise ValueError(f'Unknown fsp target: {ns_path}')
|
|
func: FspFunc = fsp.func
|
|
|
|
_fqme: str = src.mkt.fqme
|
|
assert _fqme == fqme
|
|
|
|
# open a data feed stream with requested broker
|
|
feed: Feed
|
|
async with data.feed.maybe_open_feed(
|
|
fqmes=[fqme],
|
|
loglevel=loglevel,
|
|
|
|
# TODO throttle tick outputs from *this* daemon since
|
|
# it'll emit tons of ticks due to the throttle only
|
|
# limits quote arrival periods, so the consumer of *this*
|
|
# needs to get throttled the ticks we generate.
|
|
# tick_throttle=60,
|
|
|
|
) as feed:
|
|
|
|
flume: Flume = feed.flumes[fqme]
|
|
# Cached feed consumers can replace ``Flume.stream`` when
|
|
# opening another subscription. Retain this scope's receiver
|
|
# before nursery setup gives another consumer a checkpoint.
|
|
quote_source: AsyncIterator[QuoteFrame] = flume.stream
|
|
# XXX: can't do this since flume.feed will be set XD
|
|
# assert flume == src
|
|
assert flume.mkt == src.mkt
|
|
mkt: MktPair = flume.mkt
|
|
|
|
# NOTE: FOR NOW, sanity checks around the feed as being
|
|
# always the src flume (until we get to fancier/lengthier
|
|
# chains/graphs.
|
|
assert src.rt_shm.token == flume.rt_shm.token
|
|
|
|
# XXX: won't work bc the _hist_shm_token value will be
|
|
# list[list] after IPC..
|
|
# assert flume.to_msg() == src_flume_addr
|
|
|
|
profiler(f'{func}: feed up')
|
|
|
|
func_name: str = func.__name__
|
|
tn: trio.Nursery
|
|
quote_stream: AsyncIterator[QuoteFrame]
|
|
wake_sample: Callable[[], None]|None
|
|
async with (
|
|
tractor.trionics.collapse_eg(), # avoid multi-taskc tb in console
|
|
trio.open_nursery() as tn,
|
|
_sample_wakeups(
|
|
quote_source,
|
|
fqme,
|
|
fsp.sample_on_step,
|
|
) as (quote_stream, wake_sample),
|
|
):
|
|
# TODO: might be better to just make a "restart" method where
|
|
# the target task is spawned implicitly and then the event is
|
|
# set via some higher level api? At that poing we might as well
|
|
# be writing a one-cancels-one nursery though right?
|
|
casc: Cascade = Cascade(
|
|
src,
|
|
dst,
|
|
tn,
|
|
fsp,
|
|
)
|
|
|
|
# TODO: this seems like it should be wrapped somewhere?
|
|
fsp_target: Callable[..., Awaitable[None]] = partial(
|
|
connect_streams,
|
|
casc=casc,
|
|
mkt=mkt,
|
|
quote_stream=quote_stream,
|
|
|
|
# flumes and shm passthrough
|
|
src=src,
|
|
dst=dst,
|
|
|
|
# chain function which takes src flume input(s)
|
|
# and renders dst flume output(s)
|
|
edge_func=func
|
|
)
|
|
|
|
# Subscribe before history bootstrap so in-place repairs
|
|
# with unchanged SHM bounds can not lose invalidations.
|
|
period_s: float = src.rt_sample_period_s
|
|
|
|
async with (
|
|
open_sample_stream(
|
|
period_s=period_s,
|
|
loglevel=loglevel,
|
|
) as istream,
|
|
casc.open_edge(
|
|
bind_func=fsp_target,
|
|
) as index,
|
|
):
|
|
# casc.bind_func = fsp_target
|
|
# index = await tn.start(fsp_target)
|
|
dst_shm: ShmArray = dst.rt_shm
|
|
src_shm: ShmArray = src.rt_shm
|
|
|
|
profiler(f'{func_name}: fsp up')
|
|
|
|
# sync to client-side actor
|
|
await ctx.started(index)
|
|
|
|
# XXX: rt stream with client which we MUST
|
|
# open here (and keep it open) in order to make
|
|
# incremental "updates" as history prepends take
|
|
# place.
|
|
async with ctx.open_stream() as client_stream:
|
|
casc.client_stream: tractor.MsgStream = client_stream
|
|
|
|
profiler(f'{func_name}: sample stream up')
|
|
profiler.finish()
|
|
|
|
async for sample_msg in istream:
|
|
# print(f'FSP incrementing {sample_msg}')
|
|
msg: SampleMsg = cast(
|
|
SampleMsg,
|
|
sample_msg,
|
|
)
|
|
|
|
# ``samplerd`` broadcasts history events for
|
|
# every feed and period. Only this cascade's
|
|
# source can invalidate its destination.
|
|
if not _is_relevant_sample_msg(
|
|
msg,
|
|
fqme,
|
|
period_s,
|
|
):
|
|
continue
|
|
|
|
# Respawn the compute task after source history
|
|
# revisions or an actual bounds desync.
|
|
(
|
|
synced,
|
|
step_diff,
|
|
_,
|
|
) = casc.is_synced()
|
|
if _needs_history_resync(
|
|
msg,
|
|
synced,
|
|
):
|
|
step_diff = (
|
|
await casc.poll_and_sync_to_step()
|
|
)
|
|
|
|
# Backfill broadcasts only announce source
|
|
# history revisions. They never represent a
|
|
# new sample row. Duplicate sample wakeups are
|
|
# likewise non-advancing when already aligned.
|
|
if not _should_advance_dst(
|
|
msg,
|
|
step_diff,
|
|
):
|
|
continue
|
|
|
|
array = dst_shm.array
|
|
|
|
# some metrics like vlm should be reset
|
|
# to zero every step.
|
|
last = _next_step_output(
|
|
array[-1:], fsp, zero_on_step,
|
|
)
|
|
|
|
dst.rt_shm.push(last)
|
|
|
|
# sync with source buffer's time step
|
|
src_l2 = src_shm.array[-2:]
|
|
src_li, src_lt = src_l2[-1][
|
|
['index', 'time']
|
|
]
|
|
src_2li, src_2lt = src_l2[-2][
|
|
['index', 'time']
|
|
]
|
|
dst_shm._array['time'][src_li] = src_lt
|
|
dst_shm._array['time'][src_2li] = src_2lt
|
|
|
|
ready: trio.Event|None = casc.sample_ready
|
|
casc.sample_ready = trio.Event()
|
|
if ready is not None:
|
|
ready.set()
|
|
if wake_sample is not None:
|
|
wake_sample()
|
|
|
|
# last2 = dst.array[-2:]
|
|
# if (
|
|
# last2[-1]['index'] != src_li
|
|
# or last2[-2]['index'] != src_2li
|
|
# ):
|
|
# dstl2 = list(last2)
|
|
# srcl2 = list(src_l2)
|
|
# print(
|
|
# # f'{dst.token}\n'
|
|
# f'src: {srcl2}\n'
|
|
# f'dst: {dstl2}\n'
|
|
# )
|