Defer SHM null repair without valid time bounds
Require positive, increasing timestamp bounds before querying a SHM gap or applying synthetic fallback. Defer unbounded edge runs without entering the debugger or rewriting their rows, and continue checking later segments. Cover leading, trailing and entirely null frames with provider, debugger and notification calls forbidden; assert the complete frame stays unchanged. (this patch was generated in some part by `Codex` using `GPT-6` (`OpenAI`))wkt/land_backfill_history
parent
1b76dd64fd
commit
8a4a1f67d8
|
|
@ -287,15 +287,20 @@ async def maybe_fill_null_segments(
|
||||||
timeframe=timeframe,
|
timeframe=timeframe,
|
||||||
):
|
):
|
||||||
|
|
||||||
# XXX NOTE: ?if we get a badly ordered timestamp
|
# Edge null runs have no valid query boundary on one side.
|
||||||
# pair, immediately stop backfilling?
|
# Defer them while allowing bounded interior runs to repair.
|
||||||
if (
|
if (
|
||||||
start_dt
|
start_t is None
|
||||||
and
|
or
|
||||||
end_dt < start_dt
|
start_t <= 0
|
||||||
|
or
|
||||||
|
end_t <= start_t
|
||||||
):
|
):
|
||||||
await tractor.pause()
|
log.warning(
|
||||||
break
|
f'Deferring SHM gap without valid time bounds:\n'
|
||||||
|
f'{start_t} -> {end_t}\n'
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
(
|
(
|
||||||
|
|
@ -411,6 +416,16 @@ async def maybe_fill_null_segments(
|
||||||
|
|
||||||
for istart, istop in iabs_slices:
|
for istart, istop in iabs_slices:
|
||||||
|
|
||||||
|
# Synthetic fallback also requires valid boundary bars.
|
||||||
|
start_t: float = shm._array[istart]['time']
|
||||||
|
end_t: float = shm._array[istop]['time']
|
||||||
|
if (
|
||||||
|
start_t <= 0
|
||||||
|
or
|
||||||
|
end_t <= start_t
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
# get view into buffer for null-segment
|
# get view into buffer for null-segment
|
||||||
gap: np.ndarray = shm._array[istart:istop]
|
gap: np.ndarray = shm._array[istart:istop]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -154,6 +154,57 @@ def test_gap_repair_failure_preserves_sibling(
|
||||||
trio.run(main)
|
trio.run(main)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('times', [
|
||||||
|
[0, 120, 180],
|
||||||
|
[60, 120, 0],
|
||||||
|
[0, 0, 0],
|
||||||
|
])
|
||||||
|
def test_gap_repair_defers_unbounded_nulls(
|
||||||
|
times: list[int],
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
'''
|
||||||
|
Edge nulls need neither provider queries nor synthetic writes.
|
||||||
|
|
||||||
|
Exercise real segment detection and both repair stages. A trailing
|
||||||
|
null used to enter `tractor.pause()`; a leading null could query
|
||||||
|
epoch zero and then overwrite data in the synthetic fallback.
|
||||||
|
|
||||||
|
'''
|
||||||
|
import greenback
|
||||||
|
|
||||||
|
frame: np.ndarray = np.ones(
|
||||||
|
len(times),
|
||||||
|
dtype=np.dtype(def_iohlcv_fields),
|
||||||
|
)
|
||||||
|
frame['index'] = np.arange(len(times))
|
||||||
|
frame['time'] = times
|
||||||
|
before: np.ndarray = frame.copy()
|
||||||
|
|
||||||
|
async def reject(*args, **kwargs) -> None:
|
||||||
|
raise AssertionError('unbounded SHM gap attempted I/O')
|
||||||
|
|
||||||
|
async def ensure_portal() -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
monkeypatch.setattr(greenback, 'ensure_portal', ensure_portal)
|
||||||
|
monkeypatch.setattr(tractor, 'pause', reject)
|
||||||
|
|
||||||
|
async def main() -> None:
|
||||||
|
with trio.fail_after(1):
|
||||||
|
await maybe_fill_null_segments(
|
||||||
|
shm=SimpleNamespace(array=frame, _array=frame),
|
||||||
|
timeframe=60,
|
||||||
|
get_hist=reject,
|
||||||
|
sampler_stream=SimpleNamespace(send=reject),
|
||||||
|
mkt=SimpleNamespace(fqme='x.fake'),
|
||||||
|
backfill_until_dt=from_timestamp(1),
|
||||||
|
)
|
||||||
|
|
||||||
|
trio.run(main)
|
||||||
|
np.testing.assert_array_equal(frame, before)
|
||||||
|
|
||||||
|
|
||||||
def test_empty_frame_completes() -> None:
|
def test_empty_frame_completes() -> None:
|
||||||
'''
|
'''
|
||||||
An empty provider frame is checked before endpoint indexing.
|
An empty provider frame is checked before endpoint indexing.
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue