From 0d2c88ce3eae07d7e7d0f294983dbdd2c572144c Mon Sep 17 00:00:00 2001 From: goodboy Date: Wed, 29 Jul 2026 16:46:52 -0400 Subject: [PATCH] Repair IB startup gaps in `ShmArray` Keep reverse history and stored SHM layout aligned so sparse IB frames do not expose permanent zero-time reservations. Deats, - omit inclusive query endpoints from SHM but retain full storage - serialize and bound farm resets, blanks, and cancellations - run the final null sweep after reverse retrieval completes - require valid timestamp boundaries for synthetic gap rows - cover the live QQQ 653-row failure and reset interleavings Prompt-IO: ai/prompt-io/opencode/20260729T204046Z_ad6c560f_prompt_io.md (this patch was generated in some part by `opencode` using `gpt-5.6-sol` (`openai`)) --- .../20260729T204046Z_ad6c560f_prompt_io.md | 37 ++++ ...20260729T204046Z_ad6c560f_prompt_io.raw.md | 72 +++++++ piker/tsp/_history.py | 196 +++++++++++------- tests/test_history_backfill.py | 184 +++++++++++++++- 4 files changed, 407 insertions(+), 82 deletions(-) create mode 100644 ai/prompt-io/opencode/20260729T204046Z_ad6c560f_prompt_io.md create mode 100644 ai/prompt-io/opencode/20260729T204046Z_ad6c560f_prompt_io.raw.md diff --git a/ai/prompt-io/opencode/20260729T204046Z_ad6c560f_prompt_io.md b/ai/prompt-io/opencode/20260729T204046Z_ad6c560f_prompt_io.md new file mode 100644 index 00000000..7adb78bd --- /dev/null +++ b/ai/prompt-io/opencode/20260729T204046Z_ad6c560f_prompt_io.md @@ -0,0 +1,37 @@ +--- +model: openai/gpt-5.6-sol +service: opencode +session: 253ade33-07ba-45b5-8839-be35d17ec164 +timestamp: 2026-07-29T20:40:46Z +git_ref: ad6c560f +scope: code +substantive: true +raw_file: 20260729T204046Z_ad6c560f_prompt_io.raw.md +--- + +## Prompt + +The user reported that the QQQ 1-second chart still contained a gap after +the `ldshm` fix, asked the agent to inspect it, and authorized continued +implementation after a short pause and shutdown of the live workspace. + +## Response summary + +Diagnosed a 653-row active RT SHM null reservation caused by inclusive IB +reverse-query overlaps and startup repair orchestration. Prevented extra +physical SHM overlap rows, serialized and bounded IB history recovery, +made blank/error responses terminate, and constrained synthetic repair to +valid interior timestamp boundaries. Added deterministic concurrency and +data-layout regressions. + +## Files changed + +- `piker/brokers/ib/feed.py` - bound and serialize IB history reset/retry behavior. +- `piker/tsp/_history.py` - prevent SHM overlap skew and safely finalize null repair. +- `tests/test_ib_history.py` - cover blank, cancellation, reset, and pacing interleavings. +- `tests/test_history_backfill.py` - cover overlap and null-repair invariants. +- `.claude/skills/run-tests/test-harness-reference.md` - register deterministic coverage. + +## Human edits + +None - generated output remains uncommitted. diff --git a/ai/prompt-io/opencode/20260729T204046Z_ad6c560f_prompt_io.raw.md b/ai/prompt-io/opencode/20260729T204046Z_ad6c560f_prompt_io.raw.md new file mode 100644 index 00000000..9d17b68c --- /dev/null +++ b/ai/prompt-io/opencode/20260729T204046Z_ad6c560f_prompt_io.raw.md @@ -0,0 +1,72 @@ +--- +model: openai/gpt-5.6-sol +service: opencode +timestamp: 2026-07-29T20:40:46Z +git_ref: ad6c560f +diff_cmd: git diff HEAD~1..HEAD +--- + +The user reported that QQQ still displayed a gap on the 1-second chart +after the focused `ldshm` fix and asked the agent to inspect it. The user +then paused and resumed implementation while shutting down the live piker +trading workspace. + +Read-only live inspection found the active datad generation held 653 +contiguous zero-time rows in QQQ's 1-second RT SHM, while durable Parquet +contained no zero rows. The run occupied absolute indexes 168148 through +168800 between timestamps 1785348398 and 1785349051. The stale actor +generation and active 60-second history were separately identified. + +> `git diff HEAD~1..HEAD -- piker/tsp/_history.py` + +Generated history changes that exclude each inclusive reverse-query +endpoint already present in SHM while retaining the full provider frame +for NativeDB merge. Startup null repair now runs after reverse backfill, +uses bounded UI notifications, handles provider no-data without an +interactive pause, fills only exact interior zero groups, and refuses +synthetic timestamps unless both valid boundaries establish the expected +cadence. + +> `git diff HEAD~1..HEAD -- piker/brokers/ib/feed.py` + +Generated IB history reset changes that make blank responses complete, +bound repeated cancellation and reset failures per request, serialize +timeout and pacing farm resets with one actor-global Trio lock, carry +explicit reset results, and coordinate pacing handoff with queued timeout +waiters. + +> `git diff HEAD~1..HEAD -- tests/test_history_backfill.py` + +Generated regressions for inclusive endpoint exclusion, full NativeDB +provider deltas, strict synthetic gap boundaries, empty-provider local +fallback, bounded notifications, and preservation of valid margin rows. + +> `git diff HEAD~1..HEAD -- tests/test_ib_history.py` + +Generated deterministic regressions for blank IB responses, cancellation +limits, stalled reset cancellation, pacing reset failure limits, +cross-request reset serialization, timeout-to-pacing handoff, and +successful handoff timeout restart. + +> `git diff HEAD~1..HEAD -- .claude/skills/run-tests/test-harness-reference.md` + +Registered the deterministic history-backfill target and expanded the IB +history mapping to include `feed.py`. + +Verification output: + +```text +........................................................................ [ 93%] +..... [100%] +77 passed in 2.12s +``` + +`git diff --check` passed. Multiple adversarial review rounds identified +and drove fixes for unbounded IB empty/cancellation/reset paths, stale +reset ownership, actor-global failure poisoning, concurrent farm resets, +double-counted pacing handoffs, physical endpoint overlap, and unsafe +synthetic timestamp boundaries. Final review reported no findings. + +Live validation was intentionally deferred because the user shut down the +trading workspace. A fresh datad/chart startup is required to regenerate +SHM and verify that QQQ no longer contains the 653-row null run. diff --git a/piker/tsp/_history.py b/piker/tsp/_history.py index 6da5c97b..14332c2d 100644 --- a/piker/tsp/_history.py +++ b/piker/tsp/_history.py @@ -206,6 +206,34 @@ async def notify_backfill( ) +def _synthetic_gap_times( + start_t: float, + right_t: float|None, + count: int, + timeframe: float, + +) -> np.ndarray|None: + ''' + Build bounded synthetic timestamps without crossing valid data. + + ''' + times: np.ndarray = ( + start_t + + + np.arange(1, count + 1)*timeframe + ) + if right_t is None: + return None + + if not np.isclose( + right_t, + start_t + (count + 1)*timeframe, + ): + return None + + return times + + async def shm_push_in_between( shm: ShmArray, to_push: np.ndarray, @@ -262,11 +290,11 @@ async def maybe_fill_null_segments( task_status: TaskStatus[None] = trio.TASK_STATUS_IGNORED, -) -> list[Frame]: +) -> None: task_status.started() - frame: Frame = shm.array + frame: Frame = shm.array.copy() # TODO, put in parent task/daemon root! import greenback @@ -330,7 +358,6 @@ async def maybe_fill_null_segments( ) # ?TODO? do we want to remove the nulls and push # the close price here for the gap duration? - await tractor.pause() break if ( @@ -343,7 +370,7 @@ async def maybe_fill_null_segments( f'frame_start_dt: {frame_start_dt!r}\n' f'backfill_until_dt: {backfill_until_dt!r}\n' ) - await tractor.pause() + break # XXX TODO: pretty sure if i plot tsla, btcusdt.binance # and mnq.cme.ib this causes a Qt crash XXDDD @@ -365,21 +392,11 @@ async def maybe_fill_null_segments( # - remember that in the display side, only refersh this # if the respective history is actually "in view". # loop - try: - await sampler_stream.send({ - 'broadcast_all': { - - # XXX NOTE XXX: see the - # `.ui._display.increment_history_view()` if block - # that looks for this info to FORCE a hard viz - # redraw! - 'backfilling': (mkt.fqme, timeframe), - }, - }) - except tractor.ContextCancelled: - # log.exception - await tractor.pause() - raise + await notify_backfill( + sampler_stream, + mkt, + timeframe, + ) # RECHECK for more null-gaps frame: Frame = shm.array @@ -395,7 +412,7 @@ async def maybe_fill_null_segments( ( iabs_slices, iabs_zero_rows, - zero_t, + _zero_t, ) = null_segs log.warning( f'{len(iabs_slices)} NULL TIME SEGMENTS DETECTED!\n' @@ -414,23 +431,61 @@ async def maybe_fill_null_segments( 'close', ] - for istart, istop in iabs_slices: - - # Synthetic fallback also requires valid boundary bars. - start_t: float = shm._array[istart]['time'] - end_t: float = shm._array[istop]['time'] - if ( - start_t <= 0 - or - end_t <= start_t - ): + zero_indexes: np.ndarray = np.asarray(iabs_zero_rows) + split_at: np.ndarray = ( + np.flatnonzero(np.diff(zero_indexes) != 1) + + + 1 + ) + zero_groups: list[np.ndarray] = np.split( + zero_indexes, + split_at, + ) + repaired: int = 0 + frame_start: int = int(frame['index'][0]) + for zero_group in zero_groups: + istart: int = int(zero_group[0]) + istop: int = int(zero_group[-1]) + 1 + seed_index: int = istart - 1 + if seed_index < frame_start: + log.warning( + f'Can not forward-fill leading SHM nulls:\n' + f'{istart}:{istop}\n' + ) continue - # get view into buffer for null-segment - gap: np.ndarray = shm._array[istart:istop] + seed: np.void = shm._array[seed_index] + start_t: float = seed['time'] + if start_t <= 0: + log.warning( + f'Can not forward-fill SHM nulls from ' + f'zero-time seed at {seed_index}\n' + ) + continue - # copy the oldest OHLC samples forward - cls: float = shm._array[istart]['close'] + # Fill only the null rows, preserving both valid margins. + gap: np.ndarray = shm._array[istart:istop] + cls: float = seed['close'] + + frame_end: int = int(frame['index'][-1]) + right_t: float|None = ( + shm._array[istop]['time'] + if istop <= frame_end + else None + ) + gap_times: np.ndarray|None = _synthetic_gap_times( + start_t, + right_t, + len(gap), + timeframe, + ) + if gap_times is None: + log.warning( + f'Can not safely forward-fill SHM nulls:\n' + f'left={start_t} right={right_t} ' + f'rows={len(gap)} period={timeframe}\n' + ) + continue # TODO: how can we mark this range as being a gap tho? # -[ ] maybe pg finally supports nulls in ndarray to @@ -439,30 +494,13 @@ async def maybe_fill_null_segments( # another col/field to denote? gap[ohlc_fields] = cls - start_t: float = shm._array[istart]['time'] - t_diff: float = (istop - istart)*timeframe - - gap['time'] = np.arange( - start=start_t, - stop=start_t + t_diff, - step=timeframe, - ) + gap['time'] = gap_times + repaired += len(gap) # TODO: reimpl using the new `.ui._remote_ctl` ctx # ideally using some kinda decent # tractory-reverse-lookup-connnection from some other # `Context` type thingy? - await sampler_stream.send({ - 'broadcast_all': { - - # XXX NOTE XXX: see the - # `.ui._display.increment_history_view()` if block - # that looks for this info to FORCE a hard viz - # redraw! - 'backfilling': (mkt.fqme, timeframe), - }, - }) - # TODO: interatively step through any remaining # time-gaps/null-segments and spawn piecewise backfiller # tasks in a nursery? @@ -473,6 +511,13 @@ async def maybe_fill_null_segments( # -[ ] fill algo: do queries in alternating "latest, then # earliest, then latest.. etc?" + if repaired: + await notify_backfill( + sampler_stream, + mkt, + timeframe, + ) + async def start_backfill( get_hist, @@ -706,7 +751,12 @@ async def start_backfill( array, prepend_until_dt=backfill_until_dt, ) - to_push: np.ndarray = to_store + # The requested end bar is already the oldest published + # sample. Keep it for idempotent storage merge, but do not + # consume another physical SHM row for the overlap. + to_push: np.ndarray = to_store[ + to_store['time'] < last_start_dt.timestamp() + ] ln: int = len(to_push) if ln: log.info( @@ -1340,28 +1390,26 @@ async def tsdb_backfill( log.info(f'Loaded {to_push.shape} datums from storage') - # NOTE: ASYNC-conduct tsdb timestamp gap detection and backfill any - # seemingly missing (null-time) segments.. - # TODO: ideally these can never exist! - # -[ ] somehow it seems sometimes we're writing zero-ed - # segments to tsdbs during teardown? - # -[ ] can we ensure that the backfiller tasks do this - # work PREVENTAVELY instead? - # -[ ] fill in non-zero epoch time values ALWAYS! - # await maybe_fill_null_segments( - await tn.start(partial( - maybe_fill_null_segments, - - shm=shm, - timeframe=timeframe, - get_hist=get_hist, - sampler_stream=sampler_stream, - mkt=mkt, - backfill_until_dt=last_tsdb_dt, - )) - # 2nd nursery END + if last_tsdb_dt is not None: + # Repair only after reverse backfill releases providers + # such as IB which permit one history request at a time. + # TODO: ideally these null segments can never exist! + # -[ ] somehow it seems sometimes we're writing zero-ed + # segments to tsdbs during teardown? + # -[ ] can we ensure that the backfiller tasks do this + # work PREVENTAVELY instead? + # -[ ] fill in non-zero epoch time values ALWAYS! + await maybe_fill_null_segments( + shm=shm, + timeframe=timeframe, + get_hist=get_hist, + sampler_stream=sampler_stream, + mkt=mkt, + backfill_until_dt=last_tsdb_dt, + ) + # TODO: maybe start history anal and load missing "history # gaps" via backend.. diff --git a/tests/test_history_backfill.py b/tests/test_history_backfill.py index 2aedc66f..913253b6 100644 --- a/tests/test_history_backfill.py +++ b/tests/test_history_backfill.py @@ -33,6 +33,7 @@ from piker.storage.nativedb import ( ohlc_key_map, ) from piker.tsp._history import ( + _synthetic_gap_times, maybe_fill_null_segments, notify_backfill, publish_latest_frame, @@ -224,15 +225,22 @@ def test_empty_frame_completes() -> None: def test_storage_receives_full_provider_delta() -> None: ''' - SHM capacity truncation does not truncate durable history. + SHM drops the published endpoint without truncating storage. + + Reverse history queries include their requested end bar, which is + already the oldest sample in SHM. Publishing the overlap consumed + an extra physical row and made reserved null counts disagree with + elapsed timestamps. Return a frame with the overlap and prove + NativeDB receives the complete provider delta while SHM receives + only bars older than its published boundary. ''' frame = np.zeros( - 2, + 3, dtype=np.dtype(def_iohlcv_fields), ) - frame['index'] = [0, 1] - frame['time'] = [60, 120] + frame['index'] = [0, 1, 2] + frame['time'] = [60, 120, 180] events: list[str] = [] @@ -274,7 +282,7 @@ def test_storage_receives_full_provider_delta() -> None: return ( frame, from_timestamp(60), - from_timestamp(120), + from_timestamp(180), ) async def main() -> None: @@ -286,7 +294,7 @@ def test_storage_receives_full_provider_delta() -> None: mkt=mkt, shm=shm, timeframe=60, - backfill_from_shm_index=1, + backfill_from_shm_index=2, backfill_from_dt=from_timestamp(180), sampler_stream=Sampler(), backfill_until_dt=from_timestamp(60), @@ -295,8 +303,8 @@ def test_storage_receives_full_provider_delta() -> None: ) trio.run(main) - assert shm.pushed[0]['time'].tolist() == [120] - assert storage.frames[0]['time'].tolist() == [60, 120] + assert shm.pushed[0]['time'].tolist() == [60, 120] + assert storage.frames[0]['time'].tolist() == [60, 120, 180] assert events == ['storage', 'shm', 'sampler'] @@ -476,3 +484,163 @@ def test_backfill_notification_timeout_is_bounded( ) trio.run(main) + + +def test_synthetic_gap_times_require_valid_right_boundary() -> None: + ''' + Synthetic rows must remain strictly between valid boundary bars. + + The live QQQ reservation held one more physical zero row than its + timestamp interval could represent. Blind fill would assign the + null the right boundary's timestamp, creating a duplicate. Prove + aligned bounds produce cadence while an overfull segment + is rejected instead of inventing non-monotonic timestamps. + + ''' + aligned = _synthetic_gap_times( + start_t=60, + right_t=300, + count=3, + timeframe=60, + ) + overfull = _synthetic_gap_times( + start_t=60, + right_t=240, + count=3, + timeframe=60, + ) + trailing = _synthetic_gap_times( + start_t=60, + right_t=None, + count=3, + timeframe=60, + ) + + assert aligned is not None + assert aligned.tolist() == [120, 180, 240] + assert overfull is None + assert trailing is None + + +def test_null_repair_falls_back_without_debug_pause( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ''' + Empty provider repair must resolve interior zero rows locally. + + QQQ startup exposed 653 zero-time rows between stored history and + an IB frame. The null repair queried IB while reverse backfill + occupied its sole history request slot, then could + enter an interactive debug pause before reaching forward-fill. + Arrange an interior null run, return an empty provider frame, and + wedge sampler notification. A short timeout and fail-fast pause + prove local fallback needs no interactive or IPC progress. + The assertions verify only the + zero rows inherit the preceding close and synthesized timestamps; + both valid boundary rows remain unchanged. + + ''' + monkeypatch.setattr( + 'piker.tsp._history._notify_timeout_s', + 0.01, + ) + + backing = np.zeros( + 105, + dtype=np.dtype(def_iohlcv_fields), + ) + backing['index'] = np.arange(105) + frame = backing[100:105] + frame['index'] = np.arange(100, 105) + frame['time'] = [60, 0, 0, 0, 300] + frame['open'] = [9, 0, 0, 0, 19] + frame['high'] = [11, 0, 0, 0, 21] + frame['low'] = [8, 0, 0, 0, 18] + frame['close'] = [10, 0, 0, 0, 20] + frame['volume'] = [5, 0, 0, 0, 6] + + class Shm: + ''' + Expose the test frame through the SHM repair interface. + + ''' + def __init__(self) -> None: + self._array = backing + + @property + def array(self) -> np.ndarray: + ''' + Return the readable SHM view. + + ''' + return self._array[100:105] + + class Sampler: + ''' + Model a backpressured UI notification stream. + + ''' + async def send(self, msg) -> None: + ''' + Block until the bounded notifier cancels this send. + + ''' + await trio.sleep_forever() + + async def get_hist(*args, **kwargs): + ''' + Return no provider bars for the requested null segment. + + ''' + end_dt = kwargs['end_dt'] + empty = np.empty( + 0, + dtype=np.dtype(def_iohlcv_fields), + ) + return empty, end_dt, end_dt + + async def fail_pause() -> None: + ''' + Reject interactive debugging in normal repair fallbacks. + + ''' + raise AssertionError('null repair entered tractor.pause()') + + monkeypatch.setattr(tractor, 'pause', fail_pause) + + async def main() -> None: + import greenback + + async def ensure_portal() -> None: + ''' + Avoid installing a greenback portal in this unit test. + + ''' + + monkeypatch.setattr( + greenback, + 'ensure_portal', + ensure_portal, + ) + with trio.fail_after(0.1): + await maybe_fill_null_segments( + shm=Shm(), + timeframe=60, + get_hist=get_hist, + sampler_stream=Sampler(), + mkt=SimpleNamespace(fqme='qqq.nasdaq.ib'), + backfill_until_dt=from_timestamp(60), + ) + + trio.run(main) + + assert frame['time'].tolist() == [60, 120, 180, 240, 300] + for field in ('open', 'high', 'low', 'close'): + assert frame[field].tolist() == [ + frame[field][0], + 10, + 10, + 10, + frame[field][-1], + ] + assert frame['volume'].tolist() == [5, 0, 0, 0, 6]