From 51330af07270c69a54c073c50b3645510c14dda4 Mon Sep 17 00:00:00 2001 From: goodboy Date: Wed, 30 Sep 2026 13:21:36 -0400 Subject: [PATCH] Test reaping of shielded peers with stalled IPC Add a seventh stalled-IPC case whose peer task shields outer cancellation while subactor control IPC remains usable. Require shield entry before SIGINT and child-side pending-cancel evidence afterward, then prove cancel-ack budget expiry, runtime hard-reap, SIGKILL status and completed OS reaping before watchdog rescue. Keep the six existing cases and include the reviewed provenance pair with corrected human attribution. This models a shielded Trio task, not an uninterruptible kernel call. Prompt-IO: ai/prompt-io/opencode/20260929T222213Z_68e9c49f_shielded_peer.md (this patch was generated in some part by `opencode` using `gpt-6.1-sol` (`openai`)) --- ...20260929T222213Z_68e9c49f_shielded_peer.md | 115 ++++++++++++++++++ ...0929T222213Z_68e9c49f_shielded_peer.raw.md | 55 +++++++++ examples/advanced_faults/ipc_stalls.py | 98 +++++++++++++-- tests/test_ipc_stalls.py | 82 ++++++++++++- 4 files changed, 337 insertions(+), 13 deletions(-) create mode 100644 ai/prompt-io/opencode/20260929T222213Z_68e9c49f_shielded_peer.md create mode 100644 ai/prompt-io/opencode/20260929T222213Z_68e9c49f_shielded_peer.raw.md diff --git a/ai/prompt-io/opencode/20260929T222213Z_68e9c49f_shielded_peer.md b/ai/prompt-io/opencode/20260929T222213Z_68e9c49f_shielded_peer.md new file mode 100644 index 00000000..d74f9378 --- /dev/null +++ b/ai/prompt-io/opencode/20260929T222213Z_68e9c49f_shielded_peer.md @@ -0,0 +1,115 @@ +--- +model: openai/gpt-6.1-sol +service: opencode +timestamp: 2026-09-29T22:22:13Z +git_ref: 68e9c49f +scope: tests +substantive: true +raw_file: 20260929T222213Z_68e9c49f_shielded_peer.raw.md +--- + +## Prompt + +Implementation-purpose summary, not a verbatim human prompt: + +Implement only the approved shielded-child/stalled-IPC followup +in the existing `ipc_fault_regressions` worktree. Add a seventh +case with both parent context directions stalled, a child task +shielding outer cancellation, and usable subactor control IPC. +Prove readiness, received cooperative cancellation, finite budget +expiry, runtime SIGKILL and parent OS reaping. Preserve existing +six-case semantics, user-owned files, index and Git history. + +## Response summary + +`probe_peer()` optionally enters `trio.CancelScope(shield=True)` +before `Context.started()`. Its stdout event later observes both +`Actor._cancel_called` and cancelled outer `Context._scope` while +the inner shield remains uncancelled. No production instrumentation +is needed: `Actor.cancel()` calls `Actor.cancel_rpc_tasks()`, whose +`Actor._cancel_task()` cancels `Context._scope` and waits for RPC +completion. The inner shield prevents that completion. + +`root_main()` stalls both OOB context directions, completes a +control RPC using untouched `staller_ptl`, and reports its finite +`Portal.cancel_timeout`. The external monitor sends actual SIGINT +only after both parked-operation and child-shield readiness. +The seventh case requires the child's pending-cancel observation, +runtime cancel-ack timeout and hard-reap logs, SIGKILL status and +existing parent `os.waitpid()` proof before watchdog cleanup. + +## Files changed + +- `examples/advanced_faults/ipc_stalls.py`: seventh mode, child + shield and evidence, control usability check, mode documentation. +- `tests/test_ipc_stalls.py`: seventh parameter, CASE explanation, + event ordering checks and mode-specific hard-reap expectations. +- This new structured provenance pair; previous records untouched. + +## Verification + +Interpreter: + +`/home/goodboy/repos/tractor/wkts/ib_disconnect_cleanup/py313/bin/python` + +Trio version: `0.34.0`. Import probe confirmed `tractor` resolves +inside `ipc_fault_regressions`. Per-process `PYTHONPATH` entries: + +1. `/home/goodboy/repos/tractor/wkts/ipc_fault_regressions` +2. `/home/goodboy/repos/tractor/wkts/ib_disconnect_cleanup/py313/lib/python3.13/site-packages` +3. `/nix/store/l59n6vzkswz23y6s4pr6cmv2p4dpd5f0-python3.13-numpy-2.4.4/lib/python3.13/site-packages` + +`PYTHONDONTWRITEBYTECODE=1`; no installed dependencies changed. + +First run: + +`python -m pytest tests/test_ipc_stalls.py -q -p no:tractor._testing._reap` + +Result: **7 passed in 14.21s**, no skips or failures. + +Second run, in a fresh process: + +`tests/devx/test_tooling.py::test_shield_pause` + +Flags: `-q -p no:tractor._testing._reap --spawn-backend=trio`. +Result: **1 passed, 1 warning in 2.65s**, no skips or failures. +The warning was `PytestAssertRewriteWarning` because the in-memory +launcher imported `tractor._testing` before `pytest.main()`. + +The tooling fixture's `mk_cmd()` normally uses bare `python` and +launches an example with a default registry. An in-memory override +used the specified absolute interpreter and `runpy.run_path()`; +it bound `tractor.open_nursery` with `functools.partial` to a unique +`get_rando_addr('uds')` address and `ensure_registry=True`. No test +or example file was changed for this existing tooling invocation. +Only the tooling test's owned process handles were signalled. + +`git diff --check` passed. Source lines follow the 69-column style. + +## Human contributions + +The human endorsed the shielded-child/stalled-IPC followup with +"right, exactly what i was thinking" and "sounds good lfg!", then +asked to "gen me a handoff for an agent there to review from your +work." Detailed implementation constraints came through internal +agent delegation; they are not verbatim human instructions. +No direct manual source edits during this interaction were observed. +Acceptance remains with the human. + +Internal agent delegation text was originally mislabelled as human +instruction; corrected during review. The private original is +retained under ignored review runtime at +`.claude/review_replies/shield_review5/`. + +## Proof limits + +This exercises test-owned Trio task resistance to outer cancellation +with cancellable parent-side transport waits, Linux, TCP context +IPC and the Trio spawner. It does not model an uninterruptible +kernel syscall, full socket buffering, packet loss or half-open TCP. +Budget expiry is evidenced by the production cancel-ack timeout +diagnostic; no synthetic send-deadline metric is claimed for this +usable-control-channel case. Both existing and new assertions run +before failure-only external watchdog rescue. + +No core runtime gap was observed. Production runtime is unchanged. diff --git a/ai/prompt-io/opencode/20260929T222213Z_68e9c49f_shielded_peer.raw.md b/ai/prompt-io/opencode/20260929T222213Z_68e9c49f_shielded_peer.raw.md new file mode 100644 index 00000000..6bc3f739 --- /dev/null +++ b/ai/prompt-io/opencode/20260929T222213Z_68e9c49f_shielded_peer.raw.md @@ -0,0 +1,55 @@ +--- +model: openai/gpt-6.1-sol +service: opencode +timestamp: 2026-09-29T22:22:13Z +git_ref: 68e9c49f +scope: tests +substantive: true +diff_cmd: git diff 68e9c49f -- examples/advanced_faults/ipc_stalls.py tests/test_ipc_stalls.py +--- + +## Human instruction + +> sounds good lfg! + +Purpose summary: the user approved the shielded-child/stalled-IPC +followup after the six-case draft was committed. This is a summary +of the surrounding context, not a verbatim implementation prompt. + +## Generated code pointers + +> `git diff 68e9c49f -- examples/advanced_faults/ipc_stalls.py` + +Added `stalled_context_with_shielded_peer`. `probe_peer()` enters +its child-only inner shield before `Context.started()`, publishes +shield readiness, and observes `Actor._cancel_called` plus pending +outer `Context._scope` cancellation while its own shield remains +uncancelled. It then remains blocked until runtime hard-killing. +`root_main()` parks both OOB context directions and completes a +separate subactor control RPC before publishing root readiness. + +> `git diff 68e9c49f -- tests/test_ipc_stalls.py` + +Added the seventh parameter and explicit CASE documentation. +Require shield entry before SIGINT, child-side pending-cancel +evidence afterward, finite cancel-ack timeout diagnostics, runtime +hard-reap diagnostics, SIGKILL status, and existing parent waitpid +proof. Watchdog rescue cannot satisfy the assertions. + +## Unedited verification output + +```text +....... [100%] +7 passed in 14.21s +``` + +```text +. [100%] +=============================== warnings summary =============================== +../ib_disconnect_cleanup/py313/lib/python3.13/site-packages/_pytest/config/__init__.py:864 + /home/goodboy/repos/tractor/wkts/ib_disconnect_cleanup/py313/lib/python3.13/site-packages/_pytest/config/__init__.py:864: PytestAssertRewriteWarning: Module already imported so cannot be rewritten; tractor._testing.pytest + self.import_plugin(arg, consider_entry_points=True) + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +1 passed, 1 warning in 2.65s +``` diff --git a/examples/advanced_faults/ipc_stalls.py b/examples/advanced_faults/ipc_stalls.py index d9086496..c4de59e7 100644 --- a/examples/advanced_faults/ipc_stalls.py +++ b/examples/advanced_faults/ipc_stalls.py @@ -10,9 +10,15 @@ to the child, preserving cancellation through the subactor directions and the subactor ``Portal``'s cancel send. Registry traffic and the independent stdout control pipe remain usable. +``stalled_context_with_shielded_peer`` parks both context directions +but leaves ``staller_ptl`` usable. Only the child's ``probe_peer`` +task shields outer cancellation; this models a non-cooperative task, +not an uninterruptible kernel call. Runtime hard-reaping is expected. + ''' import argparse from collections.abc import AsyncIterator +from contextlib import nullcontext from functools import partial import json import os @@ -207,6 +213,8 @@ class StallableTransport: @tractor.context async def probe_peer( ctx: tractor.Context, + shield_peer: bool = False, + marker: str = '', ) -> None: ''' Keep the peer context alive while the caller stalls IPC. @@ -217,13 +225,51 @@ async def probe_peer( the previous response used buffered bytes. Only the external controller's SIGINT may terminate this test context. + With ``shield_peer``, enter an inner scope before publishing + ``Context.started()``. ``Actor._cancel_task()`` cancels the outer + ``Context._scope``, not this shield. Observe that pending cancel + and ``Actor._cancel_called`` inside the shield without completing + the task, so the parent must exhaust its cancel-ack budget. + ''' - await ctx.started(os.getpid()) - async with ctx.open_stream() as stream: - assert await stream.receive() == 'probe' - await stream.send('probe-1') - await stream.send('probe-2') - await trio.sleep_forever() + with ( + trio.CancelScope(shield=True) + if shield_peer else nullcontext() + ) as shield: + if shield_peer: + emit( + marker=marker, + event='shield_entered', + pid=os.getpid(), + ) + await ctx.started(os.getpid()) + async with ctx.open_stream() as stream: + assert await stream.receive() == 'probe' + await stream.send('probe-1') + await stream.send('probe-2') + if shield_peer: + actor: tractor.Actor = tractor.current_actor() + # Observe actor cancel-RPC receipt AND the outer + # context task's cancel request as distinct facts. + # Poll inside the shield so other child tasks can + # run; 0.01 is a scheduler yield, not a readiness + # delay or timeout. The controller verifies shield + # entry before SIGINT; runtime kill ends this task. + while not ( + actor._cancel_called + and ctx._scope.cancel_called + ): + await trio.sleep(0.01) + emit( + marker=marker, + event='shielded_cancel_pending', + pid=os.getpid(), + actor_cancel_received=actor._cancel_called, + outer_cancel_called=ctx._scope.cancel_called, + shield=shield.shield, + shield_cancel_called=shield.cancel_called, + ) + await trio.sleep_forever() raise AssertionError('Peer stream completed before cancellation') @@ -267,10 +313,14 @@ async def root_main( ``mode`` selects ``healthy_ipc``, ``stalled_context_receive``, ``stalled_context_send``, ``stalled_context_send_and_receive``, - ``stalled_context_cancel_send``, or + ``stalled_context_cancel_send``, + ``stalled_context_with_shielded_peer``, or ``stalled_context_and_subactor_cancel_send``. The last case parks both context directions and the subactor ``Portal``'s - cancel request. ``port`` selects this root's registry address; + cancel request. The shielded-peer case also parks both context + directions, but retains usable control IPC and shields only + ``probe_peer``; both cases require runtime hard-reaping. + ``port`` selects this root's registry address; ``marker`` frames stdout events; ``state`` retains child and parked-I/O evidence for the SIGINT result. @@ -309,10 +359,17 @@ async def root_main( # Most modes leave subactor ``Portal`` cancel usable. # ``stalled_context_and_subactor_cancel_send`` also # stalls its cancel send AND both context directions. + shield_peer: bool = mode == ( + 'stalled_context_with_shielded_peer' + ) async with ( _connect_chan(peer_addr) as peer_channel, open_portal(peer_channel) as peer_ptl, - peer_ptl.open_context(probe_peer) as (ctx, pid), + peer_ptl.open_context( + probe_peer, + shield_peer=shield_peer, + marker=marker, + ) as (ctx, pid), ): # Match the peer PID from ``Context.started()``. assert pid == proc.pid @@ -327,6 +384,7 @@ async def root_main( 'stalled_context_receive', # Suspend context send and receive. 'stalled_context_send_and_receive', + 'stalled_context_with_shielded_peer', # Also stall the subactor ``Portal`` cancel send. 'stalled_context_and_subactor_cancel_send', } @@ -338,7 +396,8 @@ async def root_main( # ?TODO, in what cases is this set vs. not? # sure would be handy to see that from this layer.. # ``stalled_context_receive``, - # ``stalled_context_send_and_receive``, and + # ``stalled_context_send_and_receive``, + # ``stalled_context_with_shielded_peer``, and # ``stalled_context_and_subactor_cancel_send`` wait # here; other cases consume both probe responses. if stall_tpt.recv_stream.enabled: @@ -358,12 +417,23 @@ async def root_main( 'stalled_context_send', # Suspend both context directions. 'stalled_context_send_and_receive', + 'stalled_context_with_shielded_peer', # Suspend ``Context.cancel()``'s timed send. 'stalled_context_cancel_send', # Also suspend the subactor ``Portal`` cancel. 'stalled_context_and_subactor_cancel_send', } control_tpt: StallableTransport|None = None + if shield_peer: + # Use the subactor ``Portal`` after shield entry, + # ordered by its ``Context.started()`` call. + # ``staller_ptl.channel`` remains usable despite + # both directions of the distinct OOB context + # ``ctx.chan`` being stalled. + assert await staller_ptl.run( + peer_address + ) == peer_addr + assert staller_ptl.cancel_timeout < float('inf') if mode == ( 'stalled_context_and_subactor_cancel_send' ): @@ -405,6 +475,13 @@ async def root_main( emit( marker=marker, event='ready', + **({ + 'shield_peer': True, + 'control_usable': True, + 'cancel_timeout': ( + staller_ptl.cancel_timeout + ), + } if shield_peer else {}), stalls=snapshot, control=state.get( 'control_fault_active' @@ -461,6 +538,7 @@ def main() -> None: 'stalled_context_send_and_receive', 'stalled_context_cancel_send', 'stalled_context_and_subactor_cancel_send', + 'stalled_context_with_shielded_peer', ), ) parser.add_argument('port', type=int, nargs='?') diff --git a/tests/test_ipc_stalls.py b/tests/test_ipc_stalls.py index 1476ab9f..79171a6d 100644 --- a/tests/test_ipc_stalls.py +++ b/tests/test_ipc_stalls.py @@ -229,6 +229,8 @@ def open_root_monitor( 'stalled_context_cancel_send', # Both context directions plus subactor ``Portal`` cancel. 'stalled_context_and_subactor_cancel_send', + # Shield the peer task; keep subactor control IPC usable. + 'stalled_context_with_shielded_peer', ], ) def test_stalled_channel_sigint( @@ -278,6 +280,28 @@ def test_stalled_channel_sigint( ``Start(self.cancel)`` send. Require deadline expiration, runtime hard-reap logs, SIGKILL exit and completed waitpid. + - Shielded peer task with usable subactor ``Portal``: + + - ``stalled_context_with_shielded_peer``: park both context + directions, but complete a control RPC after shield entry. + ``probe_peer`` enters its inner shield before publishing + ``Context.started()``. + Before SIGINT require that child's shield-entry event; after + SIGINT require its observation of ``Actor._cancel_called`` + and cancelled outer ``Context._scope`` inside the still + uncancelled shield. Require cancel-ack budget expiry in the + runtime log, runtime hard-reaping, SIGKILL and waitpid proof. + ``tractor.runtime._supervise._try_cancel_then_kill()`` calls + ``Portal.cancel_actor(raise_on_timeout=True)``. Its bounded + RPC wait raises ``ActorTooSlowError`` in + ``tractor.runtime._portal``; the supervisor logs escalation + and calls ``proc.kill()``. The Trio spawner joins via + ``tractor.spawn._spawn.soft_kill()`` with + ``trio.Process.wait``; the root's ``os.waitpid()`` check + independently proves the child was already reaped. + An unshielded task's clean exit must fail these assertions. + This models outer-cancel resistance, not a kernel syscall. + ``open_root_monitor()`` observes OS lifetime outside the actor tree. A ``Portal`` within the root cannot independently enforce a watchdog deadline or prove OS exit over stalled IPC. Missing @@ -289,6 +313,8 @@ def test_stalled_channel_sigint( signalled: bool = False result: dict[str, Any]|None = None stalls: dict[str, bool]|None = None + shield_entered: bool = False + shield_cancel_pending: bool = False with open_root_monitor(mode=mode) as monitor: proc: subprocess.Popen[bytes] = monitor.proc for event in monitor.iter_events(): @@ -312,6 +338,7 @@ def test_stalled_channel_sigint( 'stalled_context_receive', 'stalled_context_send_and_receive', 'stalled_context_and_subactor_cancel_send', + 'stalled_context_with_shielded_peer', } ) assert stalls['send'] == ( @@ -320,6 +347,7 @@ def test_stalled_channel_sigint( 'stalled_context_send_and_receive', 'stalled_context_cancel_send', 'stalled_context_and_subactor_cancel_send', + 'stalled_context_with_shielded_peer', } ) assert stalls['finite_send'] == ( @@ -334,6 +362,12 @@ def test_stalled_channel_sigint( 'finite_send': True, } assert proc.poll() is None + if mode == 'stalled_context_with_shielded_peer': + assert shield_entered + assert event['shield_peer'] is True + assert event['control_usable'] is True + assert event['control'] is None + assert 0 < event['cancel_timeout'] < 20 monitor.signal_root() signalled = True @@ -352,6 +386,27 @@ def test_stalled_channel_sigint( 'stalled_context_and_subactor_cancel_send' ) + elif event['event'] == 'shield_entered': + assert mode == 'stalled_context_with_shielded_peer' + assert not signalled + assert not shield_entered + assert monitor.child is not None + assert event['pid'] == monitor.child.pid + shield_entered = True + + elif event['event'] == 'shielded_cancel_pending': + assert mode == 'stalled_context_with_shielded_peer' + assert signalled + assert shield_entered + assert not shield_cancel_pending + assert monitor.child is not None + assert event['pid'] == monitor.child.pid + assert event['actor_cancel_received'] is True + assert event['outer_cancel_called'] is True + assert event['shield'] is True + assert event['shield_cancel_called'] is False + shield_cancel_pending = True + else: raise AssertionError(event) @@ -384,6 +439,26 @@ def test_stalled_channel_sigint( 'escalating to `proc.kill()` (hard-reap)' in line for line in monitor.transcript ), 'Missing runtime hard-reap evidence' + elif mode == 'stalled_context_with_shielded_peer': + assert shield_cancel_pending + assert result['control_fault_active'] is None + # ``_supervise._try_cancel_then_kill()`` kills the child + # when its shield prevents the cancel RPC completing: + # ``proc.kill()`` produces this negative SIGKILL status. + # ``_spawn.soft_kill()`` joins via ``trio.Process.wait``; + # ``child_reaped`` above proves the OS wait completed. + # This log identifies supervisor escalation, distinct + # from the Trio spawner's legacy ``hard_kill`` fallback. + assert result['child_returncode'] == -signal.SIGKILL + assert any( + 'did not ack its `Actor.cancel()` RPC within ' + 'bounded wait' in line + for line in monitor.transcript + ), 'Missing runtime cancel-ack budget expiry' + assert any( + 'escalating to `proc.kill()` (hard-reap)' in line + for line in monitor.transcript + ), 'Missing runtime hard-reap evidence' else: # The baseline and context-only faults retain a usable # subactor ``Portal``: cooperative child exit is clean. @@ -395,8 +470,9 @@ def test_stalled_channel_sigint( 'mode': mode, 'root_returncode': proc.returncode, 'watchdog_forced_cleanup': False, - 'runtime_hard_reap': mode == ( - 'stalled_context_and_subactor_cancel_send' - ), + 'runtime_hard_reap': mode in { + 'stalled_context_and_subactor_cancel_send', + 'stalled_context_with_shielded_peer', + }, 'control_deadline_expired': control_expired, }))