372 lines
12 KiB
Python
372 lines
12 KiB
Python
'''
|
|
Sketchy network blackoutz, ugly byzantine gens, puedes eschuchar la
|
|
cancelacion?..
|
|
|
|
'''
|
|
from contextlib import asynccontextmanager as acm
|
|
from types import ModuleType
|
|
|
|
import pytest
|
|
from _pytest.pathlib import import_path
|
|
import trio
|
|
import tractor
|
|
from tractor import (
|
|
Portal,
|
|
TransportClosed,
|
|
)
|
|
from tractor.runtime._supervise import ActorNursery
|
|
from tractor._testing import (
|
|
examples_dir,
|
|
break_ipc,
|
|
assert_child_reaped,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def reaped_children(monkeypatch: pytest.MonkeyPatch):
|
|
'''
|
|
Retain spawned process handles and check synchronous reaping.
|
|
|
|
Snapshot each ActorNursery._children entry at publication, before
|
|
failure teardown removes it. Check both backend completion and
|
|
PID disappearance after trio.run(), without an external reaper.
|
|
|
|
'''
|
|
procs = []
|
|
start_actor = ActorNursery.start_actor
|
|
|
|
async def record_child(an, *args, **kwargs):
|
|
'''
|
|
Record only children spawned by this test's actor nursery.
|
|
|
|
'''
|
|
portal = await start_actor(an, *args, **kwargs)
|
|
procs.append(an._children[portal.channel.aid.uid][1])
|
|
return portal
|
|
|
|
monkeypatch.setattr(ActorNursery, 'start_actor', record_child)
|
|
yield procs
|
|
for proc in procs:
|
|
assert_child_reaped(proc)
|
|
|
|
|
|
@pytest.fixture
|
|
def fault_contexts(monkeypatch: pytest.MonkeyPatch):
|
|
'''
|
|
Observe fault state without altering cancellation or propagation.
|
|
|
|
Collect the entered `tractor.Context`s in a `list` delivered
|
|
to the consuming test. This allows introspecting a ctx's recorded
|
|
fault even if teardown incorrectly swallows that failure.
|
|
|
|
On teardown ensure transport-loss does not set
|
|
`ContextStatus._canceller`: connection loss alone supplies no
|
|
remote cancellation or task-error provenance. For the simulated
|
|
user-interrupt cases, ensure `Context.cancel_called` is True
|
|
because teardown requested cancellation of the remote task.
|
|
|
|
'''
|
|
contexts: list[tractor.Context] = []
|
|
open_context = Portal.open_context
|
|
|
|
@acm
|
|
async def record_context(portal, func, **kwargs):
|
|
'''
|
|
Hook into all `Portal.open_context() as pair` calls, stashing
|
|
their contexts in a `list` for consumption by dependent
|
|
tests for introspection.
|
|
|
|
'''
|
|
ctx: tractor.Context|None = None
|
|
try:
|
|
async with open_context(
|
|
portal,
|
|
func,
|
|
**kwargs,
|
|
) as (ctx, sent):
|
|
if func.__name__ in {
|
|
'recv_and_spawn_net_killers',
|
|
'break_ipc_after_started',
|
|
}:
|
|
contexts.append(ctx)
|
|
yield (ctx, sent)
|
|
finally:
|
|
if (
|
|
ctx in contexts
|
|
and
|
|
isinstance(ctx.maybe_error, TransportClosed)
|
|
):
|
|
# MsgStream closure is distinct from transport loss.
|
|
# After teardown, send prioritizes the ctx error,
|
|
# while receive retains the stream's terminal marker.
|
|
stream = ctx._stream
|
|
assert stream.closed
|
|
assert not ctx.chan.connected()
|
|
with pytest.raises(TransportClosed) as err:
|
|
await stream.send('after-close')
|
|
assert err.value is ctx.maybe_error
|
|
terminal = stream._eoc or stream._closed
|
|
assert isinstance(terminal, BaseException)
|
|
with pytest.raises(type(terminal)) as err:
|
|
await stream.receive()
|
|
assert err.value is terminal
|
|
|
|
monkeypatch.setattr(Portal, 'open_context', record_context)
|
|
yield contexts
|
|
for ctx in contexts:
|
|
if isinstance(ctx.maybe_error, TransportClosed):
|
|
assert ctx.canceller is None
|
|
elif isinstance(ctx._status._local_error, KeyboardInterrupt):
|
|
assert ctx.cancel_called
|
|
|
|
|
|
def assert_transport_closed(
|
|
exc: TransportClosed|BaseExceptionGroup,
|
|
local: bool,
|
|
) -> tuple[TransportClosed, ...]:
|
|
'''
|
|
Validate every failure leaf, including its socket-level source.
|
|
|
|
Parent close interrupts receive with ClosedResourceError.
|
|
Peer close can report clean EOF or a reset, depending on buffered
|
|
traffic. No arbitrary exception or remote cancellation qualifies.
|
|
|
|
'''
|
|
if isinstance(exc, BaseExceptionGroup):
|
|
return tuple(
|
|
fault
|
|
for leaf in exc.exceptions
|
|
for fault in assert_transport_closed(leaf, local=local)
|
|
)
|
|
|
|
assert isinstance(exc, TransportClosed)
|
|
cause = exc.__cause__
|
|
assert exc.src_exc is cause
|
|
if local:
|
|
assert isinstance(cause, trio.ClosedResourceError)
|
|
assert 'closed locally' in str(exc)
|
|
elif cause is None:
|
|
assert 'gracefully closed' in str(exc)
|
|
else:
|
|
assert 'closed by peer' in str(exc)
|
|
assert isinstance(
|
|
cause,
|
|
(ValueError, trio.BrokenResourceError,
|
|
ConnectionResetError),
|
|
)
|
|
if isinstance(cause, ValueError):
|
|
assert 'unclean EOF' in str(cause)
|
|
|
|
# TODO, should we expect a certain exc-message (per tpt) too?
|
|
# e.g. cause.args[0] == 'another task closed this fd'
|
|
return (exc,)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
'pre_aclose_msgstream',
|
|
[False, True],
|
|
ids=['no_msgstream_aclose', 'pre_aclose_msgstream'],
|
|
)
|
|
@pytest.mark.parametrize(
|
|
'ipc_break',
|
|
[
|
|
# No breaks: retain the healthy Ctrl-C cancellation contract.
|
|
dict(break_parent_ipc_after=False,
|
|
break_child_ipc_after=False),
|
|
|
|
# Only parent breaks.
|
|
dict(break_parent_ipc_after=500,
|
|
break_child_ipc_after=False),
|
|
|
|
# Only child breaks.
|
|
dict(break_parent_ipc_after=False,
|
|
break_child_ipc_after=500),
|
|
|
|
# Both configured: parent breaks first.
|
|
dict(break_parent_ipc_after=500,
|
|
break_child_ipc_after=800),
|
|
|
|
# Both configured: child breaks first.
|
|
dict(break_parent_ipc_after=800,
|
|
break_child_ipc_after=500),
|
|
],
|
|
ids=[
|
|
'no_break',
|
|
'break_parent',
|
|
'break_child',
|
|
'break_both_parent_first',
|
|
'break_both_child_first',
|
|
],
|
|
)
|
|
def test_ipc_channel_break_during_stream(
|
|
debug_mode: bool,
|
|
loglevel: str,
|
|
spawn_backend: str,
|
|
ipc_break: dict,
|
|
pre_aclose_msgstream: bool,
|
|
tpt_proto: str,
|
|
reg_addr: tuple[str, int|str],
|
|
reaped_children,
|
|
fault_contexts: list[tractor.Context],
|
|
):
|
|
'''
|
|
Detect local/peer closure without waiting for a later Ctrl-C.
|
|
|
|
The old scenario required ContextCancelled and a hang even when
|
|
EOF had already notified the unfinished context. Sequence-number
|
|
thresholds choose the first breaker; the child closes before its
|
|
trigger reply, preventing the parent from advancing past it.
|
|
|
|
With pre-close, MsgStream absorbs the peer's `Stop`, but pending
|
|
Context.wait_for_result() must fail on transport loss. Check each
|
|
exception leaf and its source, bound teardown, and verify the
|
|
child is gone synchronously. No-break runs still raise synthetic
|
|
KeyboardInterrupt and reap the child through normal cancellation.
|
|
|
|
If Context._scope absorbs EOF-driven task cancellation, teardown
|
|
must raise the recorded fault despite a disconnected channel.
|
|
That path must not manufacture a cancel request or peer ack, and
|
|
must preserve the original TransportClosed object and its cause.
|
|
|
|
The fixture checks stream closure separately from transport
|
|
connectivity after unwinding, then retries send and receive to
|
|
verify their distinct error precedence and terminal identities.
|
|
|
|
'''
|
|
if spawn_backend != 'trio' and debug_mode:
|
|
pytest.skip('debug_mode only supported on trio spawner')
|
|
|
|
mod: ModuleType = import_path(
|
|
examples_dir() / 'advanced_faults'
|
|
/ 'ipc_failure_during_stream.py',
|
|
root=examples_dir(),
|
|
consider_namespace_packages=False,
|
|
)
|
|
parent = ipc_break['break_parent_ipc_after']
|
|
child = ipc_break['break_child_ipc_after']
|
|
breaking = parent or child
|
|
local = bool(
|
|
parent
|
|
and
|
|
(
|
|
(not child)
|
|
or
|
|
(parent < child)
|
|
)
|
|
)
|
|
|
|
async def run():
|
|
'''
|
|
Bound the example, including actor nursery teardown.
|
|
|
|
'''
|
|
with trio.fail_after(10):
|
|
await mod.main(
|
|
debug_mode=debug_mode,
|
|
start_method=spawn_backend,
|
|
loglevel=loglevel,
|
|
pre_close=pre_aclose_msgstream,
|
|
tpt_proto=tpt_proto,
|
|
reg_addr=reg_addr,
|
|
**ipc_break,
|
|
)
|
|
|
|
if not breaking:
|
|
with pytest.raises(KeyboardInterrupt):
|
|
trio.run(run)
|
|
else:
|
|
with pytest.raises((TransportClosed, ExceptionGroup)) as err:
|
|
trio.run(run)
|
|
ctx = fault_contexts[0]
|
|
pytest.fail(
|
|
'Transport failure was swallowed:\n'
|
|
f'{ctx.repr_outcome()}\n'
|
|
f'error={ctx.maybe_error!r}'
|
|
)
|
|
faults = assert_transport_closed(err.value, local=local)
|
|
ctx = fault_contexts[0]
|
|
assert any(fault is ctx.maybe_error for fault in faults)
|
|
if ctx._scope.cancelled_caught:
|
|
error = ctx._status._remote_error
|
|
assert any(fault is error for fault in faults)
|
|
assert ctx._status._local_error is error
|
|
assert not ctx.cancel_called
|
|
assert not ctx.cancel_acked
|
|
assert len(reaped_children) == len(fault_contexts) == 1
|
|
|
|
|
|
@tractor.context
|
|
async def break_ipc_after_started(
|
|
ctx: tractor.Context,
|
|
) -> None:
|
|
'''
|
|
Wait for the caller's stream barrier, then close unfinished IPC.
|
|
|
|
'''
|
|
await ctx.started()
|
|
async with ctx.open_stream() as stream:
|
|
assert await stream.receive() == 'ready'
|
|
|
|
# TODO: make a test which verifies the error
|
|
# for this, i.e. raises a `MsgTypeError`
|
|
# await ctx.chan.send(None)
|
|
|
|
await break_ipc(stream=stream, pre_close=True)
|
|
await trio.sleep_forever()
|
|
|
|
|
|
def test_stream_closed_right_after_ipc_break_and_zombie_lord_engages(
|
|
reg_addr: tuple[str, int|str],
|
|
tpt_proto: str,
|
|
spawn_backend: str,
|
|
loglevel: str,
|
|
reaped_children,
|
|
fault_contexts,
|
|
):
|
|
'''
|
|
Reap a child after detected close without a later user SIGINT.
|
|
|
|
Previously the parent slept, timed out its context, then raised
|
|
KeyboardInterrupt to engage process supervision ("zombie lord").
|
|
EOF now notifies the unfinished context immediately. A ready
|
|
message ensures both streams exist before the child sends `Stop`
|
|
and closes IPC; the caller then waits for the missing result.
|
|
|
|
A bounded TransportClosed exit proves fault-driven teardown, and
|
|
reaped_children checks process completion and PID disappearance
|
|
synchronously. Timer-driven cancellation cannot pass this test.
|
|
|
|
'''
|
|
async def main():
|
|
'''
|
|
Await transport failure and bound the entire nursery exit.
|
|
|
|
'''
|
|
with trio.fail_after(10):
|
|
async with tractor.open_nursery(
|
|
registry_addrs=[reg_addr],
|
|
ensure_registry=True,
|
|
enable_transports=[tpt_proto],
|
|
start_method=spawn_backend,
|
|
loglevel=loglevel,
|
|
) as an:
|
|
portal = await an.start_actor(
|
|
'ipc_breaker',
|
|
enable_modules=[__name__],
|
|
)
|
|
async with portal.open_context(
|
|
break_ipc_after_started,
|
|
) as (ctx, sent):
|
|
async with ctx.open_stream() as stream:
|
|
await stream.send('ready')
|
|
await stream.receive()
|
|
await ctx.wait_for_result()
|
|
pytest.fail('broken context returned a result')
|
|
|
|
with pytest.raises((TransportClosed, ExceptionGroup)) as err:
|
|
trio.run(main)
|
|
faults = assert_transport_closed(err.value, local=False)
|
|
ctx = fault_contexts[0]
|
|
assert any(fault is ctx.maybe_error for fault in faults)
|
|
assert len(reaped_children) == len(fault_contexts) == 1
|