From fadab3d2bc1f217a95a8703304e28087c5a33761 Mon Sep 17 00:00:00 2001 From: goodboy Date: Tue, 1 Sep 2026 22:07:22 -0400 Subject: [PATCH] Target FSP history graphics by SHM token Replace whole-linked-chart refreshes after FSP history rebuilds with updates selected by normalized destination `NDToken` identity. Clear each matching `Viz` formatter and max/min cache, redraw its path, then rescale only the affected view. This keeps primary and foreign market curves stable while near-term history is backfilled. Record the concrete piker `PlotItem`, `ChartView`, `Axis`, and `YAxisLabel` relationships used by the redraw path. Prompt-IO: ai/prompt-io/opencode/20260828T195032Z_9115d24d_prompt_io.md (this patch was generated in some part by `opencode` using `gpt-5.6-sol` (`openai`)) --- .../20260828T195032Z_9115d24d_prompt_io.md | 67 ++++++++++++ ...20260828T195032Z_9115d24d_prompt_io.raw.md | 54 ++++++++++ piker/data/_sharedmem.py | 18 +++- piker/ui/_dataviz.py | 3 +- piker/ui/_fsp.py | 101 ++++++++++++++---- piker/ui/_pg_overrides.py | 17 +++ 6 files changed, 238 insertions(+), 22 deletions(-) create mode 100644 ai/prompt-io/opencode/20260828T195032Z_9115d24d_prompt_io.md create mode 100644 ai/prompt-io/opencode/20260828T195032Z_9115d24d_prompt_io.raw.md diff --git a/ai/prompt-io/opencode/20260828T195032Z_9115d24d_prompt_io.md b/ai/prompt-io/opencode/20260828T195032Z_9115d24d_prompt_io.md new file mode 100644 index 00000000..52b1cc6f --- /dev/null +++ b/ai/prompt-io/opencode/20260828T195032Z_9115d24d_prompt_io.md @@ -0,0 +1,67 @@ +--- +model: openai/gpt-5.6-sol +service: opencode +session: opencode-fsp-backfill-sync-20260828 +timestamp: 2026-08-28T19:50:32+00:00 +git_ref: 9115d24d +scope: code +substantive: true +raw_file: 20260828T195032Z_9115d24d_prompt_io.raw.md +--- + +## Prompt + +The human requested a repo-specialist role for realtime financial signal +processing, a new FSP expert skill, and a fix for repeated FSP desynchronizing +and whole-chart refreshes during near-term backfill with multiple overlaid +market feeds. Work was required in a new worktree based on the root checkout. + +Follow-up direction required explicit typing throughout the patch, exact +Qt/pyqtgraph widget types, project-style union spacing, and a final +per-behaviour multi-commit plan. + +## Response summary + +The response traced the reported `step_diff=0`, `len_diff=2000` state to +legitimate source-history prepends. It retained required historical +recomputation while preventing backfill wakeups from advancing destination +rows or refreshing unrelated charts. It introduced immutable bootstrap +snapshots, source/timeframe event filtering, absolute SHM-bound checks, +targeted graphics invalidation, and deterministic tests. + +The typing follow-up established a canonical FSP generator protocol, removed +an engine-side cast, normalized the `wma` and `rsi` built-ins, validated +historical output schemas, and made generator cleanup deterministic. A later +review replaced arbitrary redraw kwargs with an explicit control, typed the +`Viz`/`PlotItem`/`Axis`/`YAxisLabel` path, added normalized quote and tick +shapes, and corrected false iterator and NumPy/Numba annotations. + +The new provider-neutral `piker-fsp-expert` skill captures architecture, +mathematical methods, optimization guidance, roadmap context, dependency +research, and a test matrix. + +## Files changed + +- `piker/fsp/_api.py` - typed FSP decorator and generator contract. +- `piker/fsp/_engine.py` - backfill synchronization, snapshot, publication, + event filtering, and cleanup behavior. +- `piker/fsp/_momo.py` - protocol-conforming WMA and RSI streams. +- `piker/fsp/_volume.py` - typed volume FSP stream declarations. +- `piker/data/ticktools.py` - concrete quote/tick and iterator contracts. +- `piker/data/_sharedmem.py` - serializable SHM token message shape. +- `piker/ui/_dataviz.py` - piker `PlotItem` ownership on `Viz`. +- `piker/ui/_pg_overrides.py` - piker view and axis return types. +- `piker/ui/_fsp.py` - destination-token-targeted redraw and y-range refresh. +- `tests/test_fsp_sync.py` - deterministic engine, UI, and operator tests. +- `.agents/skills/piker-fsp-expert/` - reusable FSP specialist knowledge. +- `.claude/skills/piker-fsp-expert` - Claude-compatible skill link. + +## Human edits + +The human identified insufficient typing in the initial patch and required +the implementation to expose and repair the actual FSP protocol instead of +using broad or misleading casts. In later review, the human specifically +rejected `**kwargs: Any`, an optional `Viz.shm`, a broad sticky-label type, +and spaced union syntax, then directed the agent to infer reusable types from +the actual graphics widgets. The human also required a per-behaviour commit +plan for final review. No direct manual source edits were observed. diff --git a/ai/prompt-io/opencode/20260828T195032Z_9115d24d_prompt_io.raw.md b/ai/prompt-io/opencode/20260828T195032Z_9115d24d_prompt_io.raw.md new file mode 100644 index 00000000..30862fbc --- /dev/null +++ b/ai/prompt-io/opencode/20260828T195032Z_9115d24d_prompt_io.raw.md @@ -0,0 +1,54 @@ +--- +model: openai/gpt-5.6-sol +service: opencode +timestamp: 2026-08-28T19:50:32+00:00 +git_ref: 9115d24d +diff_cmd: git diff HEAD~1..HEAD +--- + +The response implemented and analyzed the requested FSP backfill fix. + +> `git diff HEAD~1..HEAD -- piker/fsp/_api.py` +> +> `git diff HEAD~1..HEAD -- piker/fsp/_engine.py` +> +> `git diff HEAD~1..HEAD -- piker/fsp/_momo.py` +> +> `git diff HEAD~1..HEAD -- piker/fsp/_volume.py` +> +> `git diff HEAD~1..HEAD -- piker/data/ticktools.py` +> +> `git diff HEAD~1..HEAD -- piker/data/_sharedmem.py` +> +> `git diff HEAD~1..HEAD -- piker/ui/_dataviz.py` +> +> `git diff HEAD~1..HEAD -- piker/ui/_pg_overrides.py` +> +> `git diff HEAD~1..HEAD -- piker/ui/_fsp.py` +> +> `git diff HEAD~1..HEAD -- tests/test_fsp_sync.py` +> +> `git diff HEAD~1..HEAD -- .agents/skills/piker-fsp-expert/` + +The generated implementation distinguishes history revisions from sample +advancement, filters foreign backfill events, computes historical output from +an immutable source snapshot, and targets FSP-backed visualizations for cache +invalidation, redraw, and y-range updates. It also establishes a typed FSP +stream protocol, normalizes the dormant scalar built-ins to that protocol, +and adds deterministic regression coverage. The typing review replaces +arbitrary redraw kwargs with an explicit boolean control, records the +concrete piker graphics object chain, and introduces reusable normalized +quote, tick, and SHM-token message shapes. + +The generated skill bundle records the FSP runtime architecture, financial +signal-processing principles, testing matrix, roadmap issues, and candidate +numerics and IPC research projects. + +Verification reported by the response: + +- focused FSP and deterministic history tests passed; +- Ruff passed for all changed Python files; +- focused `ty` checks passed for tick and scalar FSP contracts; +- compileall passed for all changed Python files; +- `git diff --check` passed; +- repeated read-only code review found no remaining actionable findings. diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index 269a2985..47578deb 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -26,6 +26,8 @@ are now imported directly from tractor throughout the codebase. ''' +from typing import TypedDict + import numpy as np from tractor.ipc._shm import ( @@ -39,6 +41,20 @@ from tractor.ipc._shm import ( from ._util import log +type DTypeField = ( + tuple[str, str] + |tuple[str, str, tuple[int, ...]] +) + + +class NDTokenMsg(TypedDict): + shm_name: str + shm_first_index_name: str + shm_last_index_name: str + dtype_descr: tuple[DTypeField, ...] + size: int + key: str|None + def _make_token( key: str, @@ -162,7 +178,7 @@ def maybe_open_shm_array( def try_read( array: np.ndarray, -) -> np.ndarray|None: +) -> np.void|None: ''' Try to read the last row from a shared mem array or ``None`` if the array read returns diff --git a/piker/ui/_dataviz.py b/piker/ui/_dataviz.py index eeece1fa..823a8d25 100644 --- a/piker/ui/_dataviz.py +++ b/piker/ui/_dataviz.py @@ -71,6 +71,7 @@ from ..toolz.profile import ( if TYPE_CHECKING: + from . import _pg_overrides as pgo from ._interaction import ChartView from ._chart import ChartPlotWidget from ._display import DisplayState @@ -282,7 +283,7 @@ class Viz(Struct): ''' name: str - plot: pg.PlotItem + plot: pgo.PlotItem _shm: ShmArray flume: Flume graphics: Curve | BarItems diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 8d18beee..939e0da0 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -20,6 +20,8 @@ FSP UI and graphics components. Financial signal processing cluster and real-time graphics management. ''' +from __future__ import annotations + from contextlib import asynccontextmanager as acm from functools import partial import inspect @@ -27,9 +29,11 @@ from itertools import cycle from typing import ( AsyncGenerator, Any, + TYPE_CHECKING, ) import msgspec +import numpy as np import pyqtgraph as pg import tractor from tractor.trionics import maybe_open_context @@ -53,7 +57,11 @@ from piker.data._sharedmem import try_read from piker.log import get_logger from piker.toolz import Profiler from piker.types import Struct -from ._axes import PriceAxis +from ._axes import ( + Axis, + PriceAxis, + YAxisLabel, +) from ..calc import humanize from ._chart import ( ChartPlotWidget, @@ -65,26 +73,27 @@ from ._forms import ( open_form_input_handling, ) +if TYPE_CHECKING: + from ._dataviz import Viz + from ._interaction import ChartView + log = get_logger(__name__) def update_fsp_chart( - viz, + viz: Viz, graphics_name: str, - array_key: str|None, - **kwargs, + array_key: str, + force_redraw: bool = False, ) -> None: - shm = viz.shm - if not shm: - return - - array = shm.array - last_row = try_read(array) + shm: ShmArray = viz.shm + array: np.ndarray = shm.array + last_row: np.void|None = try_read(array) # guard against unreadable case - if not last_row: + if last_row is None: log.warning( f'Read-race on shm array,\n' f'graphics_name: {graphics_name!r}\n' @@ -95,20 +104,71 @@ def update_fsp_chart( # update graphics # NOTE: this does a length check internally which allows it # staying above the last row check below.. - viz.update_graphics() + viz.update_graphics(force_redraw=force_redraw) # XXX: re: ``array_key``: fsp func names must be unique meaning we # can't have duplicates of the underlying data even if multiple # sub-charts reference it under different 'named charts'. # read from last calculated value and update any label - last_val_sticky = viz.plot.getAxis( - 'right')._stickies.get(graphics_name) - if last_val_sticky: - last = last_row[array_key] + axis: Axis = viz.plot.getAxis('right') + last_val_sticky: YAxisLabel|None = axis._stickies.get( + graphics_name, + ) + if last_val_sticky is not None: + last: float = float(last_row[array_key]) last_val_sticky.update_from_data(-1, last) +def update_fsp_vizs( + linked: LinkedSplits, + shm_token: NDToken, + +) -> int: + ''' + Redraw each `Viz` whose `Viz.shm` matches `shm_token`. + + The FSP engine sends its destination token after rewriting + history. Every chart or subplot reading that SHM needs its + graphics caches updated and its view rescaled. + + ''' + charts: tuple[ChartPlotWidget, ...] = ( + linked.chart, + *linked.subplots.values(), + ) + updated: int = 0 + views: list[ChartView] = [] + for chart in charts: + for viz in chart._vizs.values(): + viz: Viz + shm: ShmArray = viz.shm + if shm._token != shm_token: + continue + + viz._mxmns.clear() + # Backfills can rewrite visible values without changing + # bounds, so bypass `Viz.update_graphics()` cache checks. + update_fsp_chart( + viz, + graphics_name=viz.name, + array_key=viz.name, + force_redraw=True, + ) + updated += 1 + view: ChartView = viz.plot.vb + if view not in views: + views.append(view) + + for view in views: + view.interact_graphics_cycle( + do_linked_charts=False, + do_overlay_scaling=True, + ) + + return updated + + @acm async def open_fsp_sidepane( linked: LinkedSplits, @@ -285,7 +345,6 @@ async def run_fsp_ui( # first UI update, usually from shm pushed history viz = chart.get_viz(array_key) update_fsp_chart( - chart, viz, name, array_key=array_key, @@ -461,9 +520,11 @@ class FspAdmin: log.debug( f'Re-syncing graphics for fsp: {ns_path}' ) - self.linked.graphics_cycle( - trigger_all=True, - prepend_update_index=info['first'], + update_fsp_vizs( + self.linked, + shm_token=NDToken.from_msg( + dict(info['key']), + ), ) else: log.info(f'recved unexpected fsp engine msg: {msg}') diff --git a/piker/ui/_pg_overrides.py b/piker/ui/_pg_overrides.py index a09a1241..ad71dcc7 100644 --- a/piker/ui/_pg_overrides.py +++ b/piker/ui/_pg_overrides.py @@ -22,8 +22,20 @@ Generally, our does not require "scentific precision" for pixel perfect view transforms. """ +from __future__ import annotations + +from typing import ( + Callable, + TYPE_CHECKING, +) + import pyqtgraph as pg +if TYPE_CHECKING: + from ._axes import Axis + from ._chart import ChartPlotWidget + from ._interaction import ChartView + def invertQTransform(tr): """Return a QTransform that is the inverse of *tr*. @@ -83,6 +95,11 @@ class PlotItem(pg.PlotItem): options ''' + vb: ChartView + chart_widget: ChartPlotWidget|None + if TYPE_CHECKING: + getAxis: Callable[[str], Axis] + def __init__( self, parent=None,