`_root`: drop unused `typing` import

Allows forcing the opened actor to either obtain the passed registry
addrs or raise a runtime error.

examples/advanced_faults/ipc_failure_during_stream.py

@@ -6,47 +6,115 @@ been an outage) and we want to ensure that despite being in debug mode
 actor tree will eventually be cancelled without leaving any zombies.
 
 '''
-import trio
+from contextlib import asynccontextmanager as acm
+from functools import partial
+
 from tractor import (
     open_nursery,
     context,
     Context,
+    ContextCancelled,
     MsgStream,
+    _testing,
 )
+import trio
+import pytest
 
 
-async def break_channel_silently_then_error(
+async def break_ipc(
     stream: MsgStream,
-):
+    method: str|None = None,
-    async for msg in stream:
+    pre_close: bool = False,
-        await stream.send(msg)
 
-        # XXX: close the channel right after an error is raised
+    def_method: str = 'eof',
-        # purposely breaking the IPC transport to make sure the parent
-        # doesn't get stuck in debug or hang on the connection join.
-        # this more or less simulates an infinite msg-receive hang on
-        # the other end.
-        await stream._ctx.chan.send(None)
-        assert 0
 
+) -> None:
+    '''
+    XXX: close the channel right after an error is raised
+    purposely breaking the IPC transport to make sure the parent
+    doesn't get stuck in debug or hang on the connection join.
+    this more or less simulates an infinite msg-receive hang on
+    the other end.
 
-async def close_stream_and_error(
+    '''
-    stream: MsgStream,
+    # close channel via IPC prot msging before
-):
+    # any transport breakage
-    async for msg in stream:
+    if pre_close:
-        await stream.send(msg)
-
-        # wipe out channel right before raising
-        await stream._ctx.chan.send(None)
         await stream.aclose()
-        assert 0
+
+    method: str = method or def_method
+    print(
+        '#################################\n'
+        'Simulating CHILD-side IPC BREAK!\n'
+        f'method: {method}\n'
+        f'pre `.aclose()`: {pre_close}\n'
+        '#################################\n'
+    )
+
+    match method:
+        case 'trans_aclose':
+            await stream._ctx.chan.transport.stream.aclose()
+
+        case 'eof':
+            await stream._ctx.chan.transport.stream.send_eof()
+
+        case 'msg':
+            await stream._ctx.chan.send(None)
+
+        # TODO: the actual real-world simulated cases like
+        # transport layer hangs and/or lower layer 2-gens type
+        # scenarios..
+        #
+        # -[ ] already have some issues for this general testing
+        # area:
+        #  - https://github.com/goodboy/tractor/issues/97
+        #  - https://github.com/goodboy/tractor/issues/124
+        #   - PR from @guille:
+        #     https://github.com/goodboy/tractor/pull/149
+        # case 'hang':
+        # TODO: framework research:
+        #
+        # - https://github.com/GuoTengda1993/pynetem
+        # - https://github.com/shopify/toxiproxy
+        # - https://manpages.ubuntu.com/manpages/trusty/man1/wirefilter.1.html
+
+        case _:
+            raise RuntimeError(
+                f'IPC break method unsupported: {method}'
+            )
+
+
+async def break_ipc_then_error(
+    stream: MsgStream,
+    break_ipc_with: str|None = None,
+    pre_close: bool = False,
+):
+    await break_ipc(
+        stream=stream,
+        method=break_ipc_with,
+        pre_close=pre_close,
+    )
+    async for msg in stream:
+        await stream.send(msg)
+
+    assert 0
+
+
+async def iter_ipc_stream(
+    stream: MsgStream,
+    break_ipc_with: str|None = None,
+    pre_close: bool = False,
+):
+    async for msg in stream:
+        await stream.send(msg)
 
 
 @context
 async def recv_and_spawn_net_killers(
 
     ctx: Context,
-    break_ipc_after: bool | int = False,
+    break_ipc_after: bool|int = False,
+    pre_close: bool = False,
 
 ) -> None:
     '''
@@ -61,26 +129,53 @@ async def recv_and_spawn_net_killers(
         async for i in stream:
             print(f'child echoing {i}')
             await stream.send(i)
+
             if (
                 break_ipc_after
-                and i > break_ipc_after
+                and
+                i >= break_ipc_after
             ):
-                '#################################\n'
+                n.start_soon(
-                'Simulating child-side IPC BREAK!\n'
+                    iter_ipc_stream,
-                '#################################'
+                    stream,
-                n.start_soon(break_channel_silently_then_error, stream)
+                )
-                n.start_soon(close_stream_and_error, stream)
+                n.start_soon(
+                    partial(
+                        break_ipc_then_error,
+                        stream=stream,
+                        pre_close=pre_close,
+                    )
+                )
+
+
+@acm
+async def stuff_hangin_ctlc(timeout: float = 1) -> None:
+
+    with trio.move_on_after(timeout) as cs:
+        yield timeout
+
+    if cs.cancelled_caught:
+        # pretend to be a user seeing no streaming action
+        # thinking it's a hang, and then hitting ctl-c..
+        print(
+            f"i'm a user on the PARENT side and thingz hangin "
+            f'after timeout={timeout} ???\n\n'
+            'MASHING CTlR-C..!?\n'
+        )
+        raise KeyboardInterrupt
 
 
 async def main(
     debug_mode: bool = False,
     start_method: str = 'trio',
+    loglevel: str = 'cancel',
 
     # by default we break the parent IPC first (if configured to break
     # at all), but this can be changed so the child does first (even if
     # both are set to break).
-    break_parent_ipc_after: int | bool = False,
+    break_parent_ipc_after: int|bool = False,
-    break_child_ipc_after: int | bool = False,
+    break_child_ipc_after: int|bool = False,
+    pre_close: bool = False,
 
 ) -> None:
 
@@ -91,60 +186,123 @@ async def main(
             # NOTE: even debugger is used we shouldn't get
             # a hang since it never engages due to broken IPC
             debug_mode=debug_mode,
-            loglevel='warning',
+            loglevel=loglevel,
 
         ) as an,
     ):
+        sub_name: str = 'chitty_hijo'
         portal = await an.start_actor(
-            'chitty_hijo',
+            sub_name,
             enable_modules=[__name__],
         )
 
-        async with portal.open_context(
+        async with (
-            recv_and_spawn_net_killers,
+            stuff_hangin_ctlc(timeout=2) as timeout,
-            break_ipc_after=break_child_ipc_after,
+            _testing.expect_ctxc(
+                yay=(
+                    break_parent_ipc_after
+                    or break_child_ipc_after
+                ),
+                # TODO: we CAN'T remove this right?
+                # since we need the ctxc to bubble up from either
+                # the stream API after the `None` msg is sent
+                # (which actually implicitly cancels all remote
+                # tasks in the hijo) or from simluated
+                # KBI-mash-from-user
+                # or should we expect that a KBI triggers the ctxc
+                # and KBI in an eg?
+                reraise=True,
+            ),
 
-        ) as (ctx, sent):
+            portal.open_context(
+                recv_and_spawn_net_killers,
+                break_ipc_after=break_child_ipc_after,
+                pre_close=pre_close,
+            ) as (ctx, sent),
+        ):
+            rx_eoc: bool = False
+            ipc_break_sent: bool = False
             async with ctx.open_stream() as stream:
                 for i in range(1000):
 
                     if (
                         break_parent_ipc_after
-                        and i > break_parent_ipc_after
+                        and
+                        i > break_parent_ipc_after
+                        and
+                        not ipc_break_sent
                     ):
                         print(
                             '#################################\n'
-                            'Simulating parent-side IPC BREAK!\n'
+                            'Simulating PARENT-side IPC BREAK!\n'
-                            '#################################'
+                            '#################################\n'
                         )
-                        await stream._ctx.chan.send(None)
+
+                        # TODO: other methods? see break func above.
+                        # await stream._ctx.chan.send(None)
+                        # await stream._ctx.chan.transport.stream.send_eof()
+                        await stream._ctx.chan.transport.stream.aclose()
+
+                        ipc_break_sent = True
 
                     # it actually breaks right here in the
                     # mp_spawn/forkserver backends and thus the zombie
                     # reaper never even kicks in?
                     print(f'parent sending {i}')
-                    await stream.send(i)
+                    try:
+                        await stream.send(i)
+                    except ContextCancelled as ctxc:
+                        print(
+                            'parent received ctxc on `stream.send()`\n'
+                            f'{ctxc}\n'
+                        )
+                        assert 'root' in ctxc.canceller
+                        assert sub_name in ctx.canceller
 
-                    with trio.move_on_after(2) as cs:
+                        # TODO: is this needed or no?
+                        raise
 
+                    # timeout: int = 1
+                    # with trio.move_on_after(timeout) as cs:
+                    async with stuff_hangin_ctlc() as timeout:
+                        print(
+                            f'PARENT `stream.receive()` with timeout={timeout}\n'
+                        )
                         # NOTE: in the parent side IPC failure case this
                         # will raise an ``EndOfChannel`` after the child
                         # is killed and sends a stop msg back to it's
                         # caller/this-parent.
-                        rx = await stream.receive()
+                        try:
+                            rx = await stream.receive()
+                            print(
+                                "I'm a happy PARENT user and echoed to me is\n"
+                                f'{rx}\n'
+                            )
+                        except trio.EndOfChannel:
+                            rx_eoc: bool = True
+                            print('MsgStream got EoC for PARENT')
+                            raise
 
-                        print(f"I'm a happy user and echoed to me is {rx}")
+            print(
+                'Streaming finished and we got Eoc.\n'
+                'Canceling `.open_context()` in root with\n'
+                'CTlR-C..'
+            )
+            if rx_eoc:
+                assert stream.closed
+                try:
+                    await stream.send(i)
+                    pytest.fail('stream not closed?')
+                except (
+                    trio.ClosedResourceError,
+                    trio.EndOfChannel,
+                ) as send_err:
+                    if rx_eoc:
+                        assert send_err is stream._eoc
+                    else:
+                        assert send_err is stream._closed
 
-                    if cs.cancelled_caught:
+            raise KeyboardInterrupt
-                        # pretend to be a user seeing no streaming action
-                        # thinking it's a hang, and then hitting ctl-c..
-                        print("YOO i'm a user anddd thingz hangin..")
-
-                print(
-                    "YOO i'm mad send side dun but thingz hangin..\n"
-                    'MASHING CTlR-C Ctl-c..'
-                )
-                raise KeyboardInterrupt
 
 
 if __name__ == '__main__':

examples/debugging/debug_mode_hang.py

@@ -0,0 +1,9 @@
+'''
+Reproduce a bug where enabling debug mode for a sub-actor actually causes
+a hang on teardown...
+
+'''
+import asyncio
+
+import trio
+import tractor

examples/parallelism/concurrent_actors_primes.py

@@ -8,7 +8,10 @@ This uses no extra threads, fancy semaphores or futures; all we need
 is ``tractor``'s channels.
 
 """
-from contextlib import asynccontextmanager
+from contextlib import (
+    asynccontextmanager as acm,
+    aclosing,
+)
 from typing import Callable
 import itertools
 import math
@@ -16,7 +19,6 @@ import time
 
 import tractor
 import trio
-from async_generator import aclosing
 
 
 PRIMES = [
@@ -44,7 +46,7 @@ async def is_prime(n):
     return True
 
 
-@asynccontextmanager
+@acm
 async def worker_pool(workers=4):
     """Though it's a trivial special case for ``tractor``, the well
     known "worker pool" seems to be the defacto "but, I want this

examples/rpc_bidir_streaming.py

@@ -13,7 +13,7 @@ async def simple_rpc(
 
     '''
     # signal to parent that we're up much like
-    # ``trio_typing.TaskStatus.started()``
+    # ``trio.TaskStatus.started()``
     await ctx.started(data + 1)
 
     async with ctx.open_stream() as stream:

pyproject.toml

@@ -26,3 +26,23 @@ all_bullets = true
   directory = "trivial"
   name = "Trivial/Internal Changes"
   showcontent = true
+
+
+[tool.pytest.ini_options]
+minversion = '6.0'
+testpaths = [
+  'tests'
+]
+addopts = [
+  # TODO: figure out why this isn't working..
+  '--rootdir=./tests',
+
+  '--import-mode=importlib',
+  # don't show frickin captured logs AGAIN in the report..
+  '--show-capture=no',
+]
+log_cli = false
+
+# TODO: maybe some of these layout choices?
+# https://docs.pytest.org/en/8.0.x/explanation/goodpractices.html#choosing-a-test-layout-import-rules
+# pythonpath = "src"

requirements-test.txt

@@ -6,4 +6,3 @@ mypy
 trio_typing
 pexpect
 towncrier
-numpy

setup.py

@@ -36,18 +36,20 @@ setup(
     platforms=['linux', 'windows'],
     packages=[
         'tractor',
-        'tractor.experimental',
+        'tractor.experimental',  # wacky ideas
-        'tractor.trionics',
+        'tractor.trionics',  # trio extensions
+        'tractor.msg',  # lowlevel data types
     ],
     install_requires=[
 
         # trio related
         # proper range spec:
         # https://packaging.python.org/en/latest/discussions/install-requires-vs-requirements/#id5
-        'trio >= 0.22',
+        'trio >= 0.24',
-        'async_generator',
+
-        'trio_typing',
+        # 'async_generator',  # in stdlib mostly!
-        'exceptiongroup',
+        # 'trio_typing',  # trio==0.23.0 has type hints!
+        # 'exceptiongroup',  # in stdlib as of 3.11!
 
         # tooling
         'tricycle',

tests/conftest.py

@@ -7,91 +7,19 @@ import os
 import random
 import signal
 import platform
-import pathlib
 import time
-import inspect
-from functools import partial, wraps
 
 import pytest
-import trio
 import tractor
+from tractor._testing import (
+    examples_dir as examples_dir,
+    tractor_test as tractor_test,
+    expect_ctxc as expect_ctxc,
+)
 
+# TODO: include wtv plugin(s) we build in `._testing.pytest`?
 pytest_plugins = ['pytester']
 
-
-def tractor_test(fn):
-    """
-    Use:
-
-    @tractor_test
-    async def test_whatever():
-        await ...
-
-    If fixtures:
-
-        - ``reg_addr`` (a socket addr tuple where arbiter is listening)
-        - ``loglevel`` (logging level passed to tractor internals)
-        - ``start_method`` (subprocess spawning backend)
-
-    are defined in the `pytest` fixture space they will be automatically
-    injected to tests declaring these funcargs.
-    """
-    @wraps(fn)
-    def wrapper(
-        *args,
-        loglevel=None,
-        reg_addr=None,
-        start_method=None,
-        **kwargs
-    ):
-        # __tracebackhide__ = True
-
-        if 'reg_addr' in inspect.signature(fn).parameters:
-            # injects test suite fixture value to test as well
-            # as `run()`
-            kwargs['reg_addr'] = reg_addr
-
-        if 'loglevel' in inspect.signature(fn).parameters:
-            # allows test suites to define a 'loglevel' fixture
-            # that activates the internal logging
-            kwargs['loglevel'] = loglevel
-
-        if start_method is None:
-            if platform.system() == "Windows":
-                start_method = 'trio'
-
-        if 'start_method' in inspect.signature(fn).parameters:
-            # set of subprocess spawning backends
-            kwargs['start_method'] = start_method
-
-        if kwargs:
-
-            # use explicit root actor start
-
-            async def _main():
-                async with tractor.open_root_actor(
-                    # **kwargs,
-                    registry_addrs=[reg_addr] if reg_addr else None,
-                    loglevel=loglevel,
-                    start_method=start_method,
-
-                    # TODO: only enable when pytest is passed --pdb
-                    # debug_mode=True,
-
-                ):
-                    await fn(*args, **kwargs)
-
-            main = _main
-
-        else:
-            # use implicit root actor start
-            main = partial(fn, *args, **kwargs)
-
-        return trio.run(main)
-
-    return wrapper
-
-
 # Sending signal.SIGINT on subprocess fails on windows. Use CTRL_* alternatives
 if platform.system() == 'Windows':
     _KILL_SIGNAL = signal.CTRL_BREAK_EVENT
@@ -111,23 +39,6 @@ no_windows = pytest.mark.skipif(
 )
 
 
-def repodir() -> pathlib.Path:
-    '''
-    Return the abspath to the repo directory.
-
-    '''
-    # 2 parents up to step up through tests/<repo_dir>
-    return pathlib.Path(__file__).parent.parent.absolute()
-
-
-def examples_dir() -> pathlib.Path:
-    '''
-    Return the abspath to the examples directory as `pathlib.Path`.
-
-    '''
-    return repodir() / 'examples'
-
-
 def pytest_addoption(parser):
     parser.addoption(
         "--ll", action="store", dest='loglevel',
@@ -165,11 +76,18 @@ _ci_env: bool = os.environ.get('CI', False)
 
 @pytest.fixture(scope='session')
 def ci_env() -> bool:
-    """Detect CI envoirment.
+    '''
-    """
+    Detect CI envoirment.
+
+    '''
     return _ci_env
 
 
+# TODO: also move this to `._testing` for now?
+# -[ ] possibly generalize and re-use for multi-tree spawning
+#    along with the new stuff for multi-addrs in distribute_dis
+#    branch?
+#
 # choose randomly at import time
 _reg_addr: tuple[str, int] = (
     '127.0.0.1',
@@ -223,6 +141,7 @@ def sig_prog(proc, sig):
     assert ret
 
 
+# TODO: factor into @cm and move to `._testing`?
 @pytest.fixture
 def daemon(
     loglevel: str,

tests/test_advanced_faults.py

@@ -3,22 +3,29 @@ Sketchy network blackoutz, ugly byzantine gens, puedes eschuchar la
 cancelacion?..
 
 '''
+import itertools
 from functools import partial
+from types import ModuleType
 
 import pytest
 from _pytest.pathlib import import_path
 import trio
 import tractor
-
+from tractor._testing import (
-from conftest import (
     examples_dir,
 )
 
 
 @pytest.mark.parametrize(
-    'debug_mode',
+    'pre_aclose_msgstream',
-    [False, True],
+    [
-    ids=['no_debug_mode', 'debug_mode'],
+        False,
+        True,
+    ],
+    ids=[
+        'no_msgstream_aclose',
+        'pre_aclose_msgstream',
+    ],
 )
 @pytest.mark.parametrize(
     'ipc_break',
@@ -63,8 +70,10 @@ from conftest import (
 )
 def test_ipc_channel_break_during_stream(
     debug_mode: bool,
+    loglevel: str,
     spawn_backend: str,
-    ipc_break: dict | None,
+    ipc_break: dict|None,
+    pre_aclose_msgstream: bool,
 ):
     '''
     Ensure we can have an IPC channel break its connection during
@@ -83,70 +92,130 @@ def test_ipc_channel_break_during_stream(
         # requires the user to do ctl-c to cancel the actor tree.
         expect_final_exc = trio.ClosedResourceError
 
-    mod = import_path(
+    mod: ModuleType = import_path(
         examples_dir() / 'advanced_faults' / 'ipc_failure_during_stream.py',
         root=examples_dir(),
     )
 
-    expect_final_exc = KeyboardInterrupt
+    # by def we expect KBI from user after a simulated "hang
-
+    # period" wherein the user eventually hits ctl-c to kill the
-    # when ONLY the child breaks we expect the parent to get a closed
+    # root-actor tree.
-    # resource error on the next `MsgStream.receive()` and then fail out
+    expect_final_exc: BaseException = KeyboardInterrupt
-    # and cancel the child from there.
     if (
+        # only expect EoC if trans is broken on the child side,
+        ipc_break['break_child_ipc_after'] is not False
+        # AND we tell the child to call `MsgStream.aclose()`.
+        and pre_aclose_msgstream
+    ):
+        # expect_final_exc = trio.EndOfChannel
+        # ^XXX NOPE! XXX^ since now `.open_stream()` absorbs this
+        # gracefully!
+        expect_final_exc = KeyboardInterrupt
 
-        # only child breaks
+    # NOTE when ONLY the child breaks or it breaks BEFORE the
-        (
+    # parent we expect the parent to get a closed resource error
-            ipc_break['break_child_ipc_after']
+    # on the next `MsgStream.receive()` and then fail out and
-            and ipc_break['break_parent_ipc_after'] is False
+    # cancel the child from there.
-        )
+    #
-
+    # ONLY CHILD breaks
-        # both break but, parent breaks first
+    if (
-        or (
+        ipc_break['break_child_ipc_after']
-            ipc_break['break_child_ipc_after'] is not False
+        and
-            and (
+        ipc_break['break_parent_ipc_after'] is False
-                ipc_break['break_parent_ipc_after']
+    ):
-                > ipc_break['break_child_ipc_after']
+        # NOTE: we DO NOT expect this any more since
-            )
+        # the child side's channel will be broken silently
+        # and nothing on the parent side will indicate this!
+        # expect_final_exc = trio.ClosedResourceError
+
+        # NOTE: child will send a 'stop' msg before it breaks
+        # the transport channel BUT, that will be absorbed by the
+        # `ctx.open_stream()` block and thus the `.open_context()`
+        # should hang, after which the test script simulates
+        # a user sending ctl-c by raising a KBI.
+        if pre_aclose_msgstream:
+            expect_final_exc = KeyboardInterrupt
+
+            # XXX OLD XXX
+            # if child calls `MsgStream.aclose()` then expect EoC.
+            # ^ XXX not any more ^ since eoc is always absorbed
+            # gracefully and NOT bubbled to the `.open_context()`
+            # block!
+            # expect_final_exc = trio.EndOfChannel
+
+    # BOTH but, CHILD breaks FIRST
+    elif (
+        ipc_break['break_child_ipc_after'] is not False
+        and (
+            ipc_break['break_parent_ipc_after']
+            > ipc_break['break_child_ipc_after']
         )
+    ):
+        if pre_aclose_msgstream:
+            expect_final_exc = KeyboardInterrupt
 
+    # NOTE when the parent IPC side dies (even if the child's does as well
+    # but the child fails BEFORE the parent) we always expect the
+    # IPC layer to raise a closed-resource, NEVER do we expect
+    # a stop msg since the parent-side ctx apis will error out
+    # IMMEDIATELY before the child ever sends any 'stop' msg.
+    #
+    # ONLY PARENT breaks
+    elif (
+        ipc_break['break_parent_ipc_after']
+        and
+        ipc_break['break_child_ipc_after'] is False
     ):
         expect_final_exc = trio.ClosedResourceError
 
-    # when the parent IPC side dies (even if the child's does as well
+    # BOTH but, PARENT breaks FIRST
-    # but the child fails BEFORE the parent) we expect the channel to be
-    # sent a stop msg from the child at some point which will signal the
-    # parent that the stream has been terminated.
-    # NOTE: when the parent breaks "after" the child you get this same
-    # case as well, the child breaks the IPC channel with a stop msg
-    # before any closure takes place.
     elif (
-        # only parent breaks
+        ipc_break['break_parent_ipc_after'] is not False
-        (
+        and (
+            ipc_break['break_child_ipc_after']
+            >
             ipc_break['break_parent_ipc_after']
-            and ipc_break['break_child_ipc_after'] is False
-        )
-
-        # both break but, child breaks first
-        or (
-            ipc_break['break_parent_ipc_after'] is not False
-            and (
-                ipc_break['break_child_ipc_after']
-                > ipc_break['break_parent_ipc_after']
-            )
         )
     ):
-        expect_final_exc = trio.EndOfChannel
+        expect_final_exc = trio.ClosedResourceError
 
-    with pytest.raises(expect_final_exc):
+    with pytest.raises(
-        trio.run(
+        expected_exception=(
-            partial(
+            expect_final_exc,
-                mod.main,
+            ExceptionGroup,
-                debug_mode=debug_mode,
+        ),
-                start_method=spawn_backend,
+    ) as excinfo:
-                **ipc_break,
+        try:
+            trio.run(
+                partial(
+                    mod.main,
+                    debug_mode=debug_mode,
+                    start_method=spawn_backend,
+                    loglevel=loglevel,
+                    pre_close=pre_aclose_msgstream,
+                    **ipc_break,
+                )
+            )
+        except KeyboardInterrupt as kbi:
+            _err = kbi
+            if expect_final_exc is not KeyboardInterrupt:
+                pytest.fail(
+                    'Rxed unexpected KBI !?\n'
+                    f'{repr(kbi)}'
+                )
+
+            raise
+
+    # get raw instance from pytest wrapper
+    value = excinfo.value
+    if isinstance(value, ExceptionGroup):
+        value = next(
+            itertools.dropwhile(
+                lambda exc: not isinstance(exc, expect_final_exc),
+                value.exceptions,
             )
         )
+        assert value
 
 
 @tractor.context
@@ -169,25 +238,29 @@ def test_stream_closed_right_after_ipc_break_and_zombie_lord_engages():
 
     '''
     async def main():
-        async with tractor.open_nursery() as n:
+        with trio.fail_after(3):
-            portal = await n.start_actor(
+            async with tractor.open_nursery() as n:
-                'ipc_breaker',
+                portal = await n.start_actor(
-                enable_modules=[__name__],
+                    'ipc_breaker',
-            )
+                    enable_modules=[__name__],
+                )
 
-            with trio.move_on_after(1):
+                with trio.move_on_after(1):
-                async with (
+                    async with (
-                    portal.open_context(
+                        portal.open_context(
-                        break_ipc_after_started
+                            break_ipc_after_started
-                    ) as (ctx, sent),
+                        ) as (ctx, sent),
-                ):
+                    ):
-                    async with ctx.open_stream():
+                        async with ctx.open_stream():
-                        await trio.sleep(0.5)
+                            await trio.sleep(0.5)
 
-                    print('parent waiting on context')
+                        print('parent waiting on context')
 
-            print('parent exited context')
+                print(
-            raise KeyboardInterrupt
+                    'parent exited context\n'
+                    'parent raising KBI..\n'
+                )
+                raise KeyboardInterrupt
 
     with pytest.raises(KeyboardInterrupt):
         trio.run(main)

tests/test_advanced_streaming.py

@@ -6,6 +6,7 @@ from collections import Counter
 import itertools
 import platform
 
+import pytest
 import trio
 import tractor
 
@@ -143,8 +144,16 @@ def test_dynamic_pub_sub():
 
     try:
         trio.run(main)
-    except trio.TooSlowError:
+    except (
-        pass
+        trio.TooSlowError,
+        ExceptionGroup,
+    ) as err:
+        if isinstance(err, ExceptionGroup):
+            for suberr in err.exceptions:
+                if isinstance(suberr, trio.TooSlowError):
+                    break
+            else:
+                pytest.fail('Never got a `TooSlowError` ?')
 
 
 @tractor.context
@@ -298,44 +307,69 @@ async def inf_streamer(
 
     async with (
         ctx.open_stream() as stream,
-        trio.open_nursery() as n,
+        trio.open_nursery() as tn,
     ):
-        async def bail_on_sentinel():
+        async def close_stream_on_sentinel():
             async for msg in stream:
                 if msg == 'done':
+                    print(
+                        'streamer RXed "done" sentinel msg!\n'
+                        'CLOSING `MsgStream`!'
+                    )
                     await stream.aclose()
                 else:
                     print(f'streamer received {msg}')
+            else:
+                print('streamer exited recv loop')
 
         # start termination detector
-        n.start_soon(bail_on_sentinel)
+        tn.start_soon(close_stream_on_sentinel)
 
-        for val in itertools.count():
+        cap: int = 10000  # so that we don't spin forever when bug..
+        for val in range(cap):
             try:
+                print(f'streamer sending {val}')
                 await stream.send(val)
+                if val > cap:
+                    raise RuntimeError(
+                        'Streamer never cancelled by setinel?'
+                    )
+                await trio.sleep(0.001)
+
+            # close out the stream gracefully
             except trio.ClosedResourceError:
-                # close out the stream gracefully
+                print('transport closed on streamer side!')
+                assert stream.closed
                 break
+        else:
+            raise RuntimeError(
+                'Streamer not cancelled before finished sending?'
+            )
 
-    print('terminating streamer')
+    print('streamer exited .open_streamer() block')
 
 
-def test_local_task_fanout_from_stream():
+def test_local_task_fanout_from_stream(
+    debug_mode: bool,
+):
     '''
     Single stream with multiple local consumer tasks using the
     ``MsgStream.subscribe()` api.
 
-    Ensure all tasks receive all values after stream completes sending.
+    Ensure all tasks receive all values after stream completes
+    sending.
 
     '''
-    consumers = 22
+    consumers: int = 22
 
     async def main():
 
         counts = Counter()
 
-        async with tractor.open_nursery() as tn:
+        async with tractor.open_nursery(
-            p = await tn.start_actor(
+            debug_mode=debug_mode,
+        ) as tn:
+            p: tractor.Portal = await tn.start_actor(
                 'inf_streamer',
                 enable_modules=[__name__],
             )
@@ -343,7 +377,6 @@ def test_local_task_fanout_from_stream():
                 p.open_context(inf_streamer) as (ctx, _),
                 ctx.open_stream() as stream,
             ):
-
                 async def pull_and_count(name: str):
                     # name = trio.lowlevel.current_task().name
                     async with stream.subscribe() as recver:
@@ -352,7 +385,7 @@ def test_local_task_fanout_from_stream():
                             tractor.trionics.BroadcastReceiver
                         )
                         async for val in recver:
-                            # print(f'{name}: {val}')
+                            print(f'bx {name} rx: {val}')
                             counts[name] += 1
 
                         print(f'{name} bcaster ended')
@@ -362,10 +395,14 @@ def test_local_task_fanout_from_stream():
                 with trio.fail_after(3):
                     async with trio.open_nursery() as nurse:
                         for i in range(consumers):
-                            nurse.start_soon(pull_and_count, i)
+                            nurse.start_soon(
+                                pull_and_count,
+                                i,
+                            )
 
+                        # delay to let bcast consumers pull msgs
                         await trio.sleep(0.5)
-                        print('\nterminating')
+                        print('terminating nursery of bcast rxer consumers!')
                         await stream.send('done')
 
             print('closed stream connection')

tests/test_cancellation.py

@@ -8,15 +8,13 @@ import platform
 import time
 from itertools import repeat
 
-from exceptiongroup import (
-    BaseExceptionGroup,
-    ExceptionGroup,
-)
 import pytest
 import trio
 import tractor
-
+from tractor._testing import (
-from conftest import tractor_test, no_windows
+    tractor_test,
+)
+from conftest import no_windows
 
 
 def is_win():
@@ -48,11 +46,13 @@ async def do_nuthin():
     ids=['no_args', 'unexpected_args'],
 )
 def test_remote_error(reg_addr, args_err):
-    """Verify an error raised in a subactor that is propagated
+    '''
+    Verify an error raised in a subactor that is propagated
     to the parent nursery, contains the underlying boxed builtin
     error type info and causes cancellation and reraising all the
     way up the stack.
-    """
+
+    '''
     args, errtype = args_err
 
     async def main():
@@ -65,7 +65,9 @@ def test_remote_error(reg_addr, args_err):
             # an exception group outside the nursery since the error
             # here and the far end task error are one in the same?
             portal = await nursery.run_in_actor(
-                assert_err, name='errorer', **args
+                assert_err,
+                name='errorer',
+                **args
             )
 
             # get result(s) from main task

tests/test_child_manages_service_nursery.py

@@ -6,14 +6,15 @@ sub-sub-actor daemons.
 '''
 from typing import Optional
 import asyncio
-from contextlib import asynccontextmanager as acm
+from contextlib import (
+    asynccontextmanager as acm,
+    aclosing,
+)
 
 import pytest
 import trio
-from trio_typing import TaskStatus
 import tractor
 from tractor import RemoteActorError
-from async_generator import aclosing
 
 
 async def aio_streamer(

tests/test_clustering.py

@@ -5,9 +5,7 @@ import trio
 import tractor
 from tractor import open_actor_cluster
 from tractor.trionics import gather_contexts
-
+from tractor._testing import tractor_test
-from conftest import tractor_test
-
 
 MESSAGE = 'tractoring at full speed'
 

tests/test_context_stream_semantics.py

@@ -5,10 +5,12 @@ Verify the we raise errors when streams are opened prior to
 sync-opening a ``tractor.Context`` beforehand.
 
 '''
-# from contextlib import asynccontextmanager as acm
 from itertools import count
 import platform
-from typing import Optional
+from pprint import pformat
+from typing import (
+    Callable,
+)
 
 import pytest
 import trio
@@ -23,7 +25,10 @@ from tractor._exceptions import (
     ContextCancelled,
 )
 
-from conftest import tractor_test
+from tractor._testing import (
+    tractor_test,
+    expect_ctxc,
+)
 
 # ``Context`` semantics are as follows,
 #  ------------------------------------
@@ -69,7 +74,7 @@ _state: bool = False
 
 @tractor.context
 async def too_many_starteds(
-    ctx: tractor.Context,
+    ctx: Context,
 ) -> None:
     '''
     Call ``Context.started()`` more then once (an error).
@@ -84,7 +89,7 @@ async def too_many_starteds(
 
 @tractor.context
 async def not_started_but_stream_opened(
-    ctx: tractor.Context,
+    ctx: Context,
 ) -> None:
     '''
     Enter ``Context.open_stream()`` without calling ``.started()``.
@@ -105,11 +110,15 @@ async def not_started_but_stream_opened(
     ],
     ids='misuse_type={}'.format,
 )
-def test_started_misuse(target):
+def test_started_misuse(
-
+    target: Callable,
+    debug_mode: bool,
+):
     async def main():
-        async with tractor.open_nursery() as n:
+        async with tractor.open_nursery(
-            portal = await n.start_actor(
+            debug_mode=debug_mode,
+        ) as an:
+            portal = await an.start_actor(
                 target.__name__,
                 enable_modules=[__name__],
             )
@@ -124,7 +133,7 @@ def test_started_misuse(target):
 @tractor.context
 async def simple_setup_teardown(
 
-    ctx: tractor.Context,
+    ctx: Context,
     data: int,
     block_forever: bool = False,
 
@@ -170,6 +179,7 @@ def test_simple_context(
     error_parent,
     callee_blocks_forever,
     pointlessly_open_stream,
+    debug_mode: bool,
 ):
 
     timeout = 1.5 if not platform.system() == 'Windows' else 4
@@ -177,20 +187,22 @@ def test_simple_context(
     async def main():
 
         with trio.fail_after(timeout):
-            async with tractor.open_nursery() as nursery:
+            async with tractor.open_nursery(
-
+                debug_mode=debug_mode,
-                portal = await nursery.start_actor(
+            ) as an:
+                portal = await an.start_actor(
                     'simple_context',
                     enable_modules=[__name__],
                 )
 
                 try:
-                    async with portal.open_context(
+                    async with (
-                        simple_setup_teardown,
+                        portal.open_context(
-                        data=10,
+                            simple_setup_teardown,
-                        block_forever=callee_blocks_forever,
+                            data=10,
-                    ) as (ctx, sent):
+                            block_forever=callee_blocks_forever,
-
+                        ) as (ctx, sent),
+                    ):
                         assert sent == 11
 
                         if callee_blocks_forever:
@@ -260,6 +272,7 @@ def test_caller_cancels(
     cancel_method: str,
     chk_ctx_result_before_exit: bool,
     callee_returns_early: bool,
+    debug_mode: bool,
 ):
     '''
     Verify that when the opening side of a context (aka the caller)
@@ -268,37 +281,100 @@ def test_caller_cancels(
 
     '''
     async def check_canceller(
-        ctx: tractor.Context,
+        ctx: Context,
     ) -> None:
-        # should not raise yet return the remote
+        actor: Actor = current_actor()
-        # context cancelled error.
+        uid: tuple = actor.uid
-        res = await ctx.result()
+        _ctxc: ContextCancelled|None = None
 
+        if (
+            cancel_method == 'portal'
+            and not callee_returns_early
+        ):
+            try:
+                res = await ctx.result()
+                assert 0, 'Portal cancel should raise!'
+
+            except ContextCancelled as ctxc:
+                # with trio.CancelScope(shield=True):
+                #     await tractor.pause()
+                _ctxc = ctxc
+                assert ctx.chan._cancel_called
+                assert ctxc.canceller == uid
+                assert ctxc is ctx.maybe_error
+
+        # NOTE: should not ever raise even in the `ctx`
+        # case since self-cancellation should swallow the ctxc
+        # silently!
+        else:
+            try:
+                res = await ctx.result()
+            except ContextCancelled as ctxc:
+                pytest.fail(f'should not have raised ctxc\n{ctxc}')
+
+        # we actually get a result
         if callee_returns_early:
             assert res == 'yo'
+            assert ctx.outcome is res
+            assert ctx.maybe_error is None
 
         else:
-            err = res
+            err: Exception = ctx.outcome
             assert isinstance(err, ContextCancelled)
             assert (
                 tuple(err.canceller)
                 ==
-                current_actor().uid
+                uid
             )
+            assert (
+                err
+                is ctx.maybe_error
+                is ctx._remote_error
+            )
+            if le := ctx._local_error:
+                assert err is le
+
+            # else:
+                # TODO: what should this be then?
+                # not defined until block closes right?
+                #
+                # await tractor.pause()
+                # assert ctx._local_error is None
+
+        # TODO: don't need this right?
+        # if _ctxc:
+        #     raise _ctxc
+
 
     async def main():
-        async with tractor.open_nursery() as nursery:
+
-            portal = await nursery.start_actor(
+        async with tractor.open_nursery(
+            debug_mode=debug_mode,
+        ) as an:
+            portal = await an.start_actor(
                 'simple_context',
                 enable_modules=[__name__],
             )
-            timeout = 0.5 if not callee_returns_early else 2
+            timeout: float = (
+                0.5
+                if not callee_returns_early
+                else 2
+            )
             with trio.fail_after(timeout):
-                async with portal.open_context(
+                async with (
-                    simple_setup_teardown,
+                    expect_ctxc(
-                    data=10,
+                        yay=(
-                    block_forever=not callee_returns_early,
+                            not callee_returns_early
-                ) as (ctx, sent):
+                            and cancel_method == 'portal'
+                        )
+                    ),
+
+                    portal.open_context(
+                        simple_setup_teardown,
+                        data=10,
+                        block_forever=not callee_returns_early,
+                    ) as (ctx, sent),
+                ):
 
                     if callee_returns_early:
                         # ensure we block long enough before sending
@@ -307,10 +383,18 @@ def test_caller_cancels(
                         await trio.sleep(0.5)
 
                     if cancel_method == 'ctx':
+                        print('cancelling with `Context.cancel()`')
                         await ctx.cancel()
-                    else:
+
+                    elif cancel_method == 'portal':
+                        print('cancelling with `Portal.cancel_actor()`')
                         await portal.cancel_actor()
 
+                    else:
+                        pytest.fail(
+                            f'Unknown `cancel_method={cancel_method} ?'
+                        )
+
                     if chk_ctx_result_before_exit:
                         await check_canceller(ctx)
 
@@ -320,6 +404,23 @@ def test_caller_cancels(
             if cancel_method != 'portal':
                 await portal.cancel_actor()
 
+            # XXX NOTE XXX: non-normal yet purposeful
+            # test-specific ctxc suppression is implemented!
+            #
+            # WHY: the `.cancel_actor()` case (cancel_method='portal')
+            # will cause both:
+            #  * the `ctx.result()` inside `.open_context().__aexit__()`
+            #  * AND the `ctx.result()` inside `check_canceller()`
+            # to raise ctxc.
+            #
+            #   which should in turn cause `ctx._scope` to
+            # catch any cancellation?
+            if (
+                not callee_returns_early
+                and cancel_method != 'portal'
+            ):
+                assert not ctx._scope.cancelled_caught
+
     trio.run(main)
 
 
@@ -338,7 +439,7 @@ def test_caller_cancels(
 @tractor.context
 async def close_ctx_immediately(
 
-    ctx: tractor.Context,
+    ctx: Context,
 
 ) -> None:
 
@@ -350,17 +451,33 @@ async def close_ctx_immediately(
 
 
 @tractor_test
-async def test_callee_closes_ctx_after_stream_open():
+async def test_callee_closes_ctx_after_stream_open(
-    'callee context closes without using stream'
+    debug_mode: bool,
+):
+    '''
+    callee context closes without using stream.
 
-    async with tractor.open_nursery() as n:
+    This should result in a msg sequence
+    |_<root>_
+             |_<fast_stream_closer>
 
-        portal = await n.start_actor(
+             <= {'started': <Any>, 'cid': <str>}
+             <= {'stop': True, 'cid': <str>}
+             <= {'result': Any, ..}
+
+     (ignored by child)
+    => {'stop': True, 'cid': <str>}
+
+    '''
+    async with tractor.open_nursery(
+        debug_mode=debug_mode,
+    ) as an:
+        portal = await an.start_actor(
             'fast_stream_closer',
             enable_modules=[__name__],
         )
 
-        with trio.fail_after(2):
+        with trio.fail_after(0.5):
             async with portal.open_context(
                 close_ctx_immediately,
 
@@ -368,10 +485,9 @@ async def test_callee_closes_ctx_after_stream_open():
                 # cancel_on_exit=True,
 
             ) as (ctx, sent):
-
                 assert sent is None
 
-                with trio.fail_after(0.5):
+                with trio.fail_after(0.4):
                     async with ctx.open_stream() as stream:
 
                         # should fall through since ``StopAsyncIteration``
@@ -379,12 +495,15 @@ async def test_callee_closes_ctx_after_stream_open():
                         # a ``trio.EndOfChannel`` by
                         # ``trio.abc.ReceiveChannel.__anext__()``
                         async for _ in stream:
+                            # trigger failure if we DO NOT
+                            # get an EOC!
                             assert 0
                         else:
 
                             # verify stream is now closed
                             try:
-                                await stream.receive()
+                                with trio.fail_after(0.3):
+                                    await stream.receive()
                             except trio.EndOfChannel:
                                 pass
 
@@ -404,8 +523,7 @@ async def test_callee_closes_ctx_after_stream_open():
 
 @tractor.context
 async def expect_cancelled(
-
+    ctx: Context,
-    ctx: tractor.Context,
 
 ) -> None:
     global _state
@@ -419,12 +537,29 @@ async def expect_cancelled(
                 await stream.send(msg)  # echo server
 
     except trio.Cancelled:
+
+        # on ctx.cancel() the internal RPC scope is cancelled but
+        # never caught until the func exits.
+        assert ctx._scope.cancel_called
+        assert not ctx._scope.cancelled_caught
+
+        # should be the RPC cmd request for `._cancel_task()`
+        assert ctx._cancel_msg
+        # which, has not yet resolved to an error outcome
+        # since this rpc func has not yet exited.
+        assert not ctx.maybe_error
+        assert not ctx._final_result_is_set()
+
+        # debug REPL if needed
+        # with trio.CancelScope(shield=True):
+        #     await tractor.pause()
+
         # expected case
         _state = False
         raise
 
     else:
-        assert 0, "Wasn't cancelled!?"
+        assert 0, "callee wasn't cancelled !?"
 
 
 @pytest.mark.parametrize(
@@ -434,13 +569,17 @@ async def expect_cancelled(
 @tractor_test
 async def test_caller_closes_ctx_after_callee_opens_stream(
     use_ctx_cancel_method: bool,
+    debug_mode: bool,
 ):
-    'caller context closes without using stream'
+    '''
+    caller context closes without using/opening stream
 
-    async with tractor.open_nursery() as an:
+    '''
+    async with tractor.open_nursery(
+        debug_mode=debug_mode,
+    ) as an:
 
         root: Actor = current_actor()
-
         portal = await an.start_actor(
             'ctx_cancelled',
             enable_modules=[__name__],
@@ -453,11 +592,13 @@ async def test_caller_closes_ctx_after_callee_opens_stream(
 
             await portal.run(assert_state, value=True)
 
-            # call cancel explicitly
+            # call `ctx.cancel()` explicitly
             if use_ctx_cancel_method:
-
                 await ctx.cancel()
 
+                # NOTE: means the local side `ctx._scope` will
+                # have been cancelled by an ctxc ack and thus
+                # `._scope.cancelled_caught` should be set.
                 try:
                     async with ctx.open_stream() as stream:
                         async for msg in stream:
@@ -486,7 +627,10 @@ async def test_caller_closes_ctx_after_callee_opens_stream(
                 assert portal.channel.connected()
 
                 # ctx is closed here
-                await portal.run(assert_state, value=False)
+                await portal.run(
+                    assert_state,
+                    value=False,
+                )
 
             else:
                 try:
@@ -497,9 +641,21 @@ async def test_caller_closes_ctx_after_callee_opens_stream(
                     # NO-OP -> since already called above
                     await ctx.cancel()
 
-        # local scope should have absorbed the cancellation
+        # NOTE: local scope should have absorbed the cancellation since
-        assert ctx.cancelled_caught
+        # in this case we call `ctx.cancel()` and the local
-        assert ctx._remote_error is ctx._local_error
+        # `._scope` does not get `.cancel_called` and thus
+        # `.cancelled_caught` neither will ever bet set.
+        if use_ctx_cancel_method:
+            assert not ctx._scope.cancelled_caught
+
+        # rxed ctxc response from far end
+        assert ctx.cancel_acked
+        assert (
+            ctx._remote_error
+            is ctx._local_error
+            is ctx.maybe_error
+            is ctx.outcome
+        )
 
         try:
             async with ctx.open_stream() as stream:
@@ -522,11 +678,13 @@ async def test_caller_closes_ctx_after_callee_opens_stream(
 
 
 @tractor_test
-async def test_multitask_caller_cancels_from_nonroot_task():
+async def test_multitask_caller_cancels_from_nonroot_task(
-
+    debug_mode: bool,
-    async with tractor.open_nursery() as n:
+):
-
+    async with tractor.open_nursery(
-        portal = await n.start_actor(
+        debug_mode=debug_mode,
+    ) as an:
+        portal = await an.start_actor(
             'ctx_cancelled',
             enable_modules=[__name__],
         )
@@ -573,7 +731,7 @@ async def test_multitask_caller_cancels_from_nonroot_task():
 @tractor.context
 async def cancel_self(
 
-    ctx: tractor.Context,
+    ctx: Context,
 
 ) -> None:
     global _state
@@ -610,16 +768,20 @@ async def cancel_self(
 
     raise RuntimeError('Context didnt cancel itself?!')
 
+
 @tractor_test
-async def test_callee_cancels_before_started():
+async def test_callee_cancels_before_started(
+    debug_mode: bool,
+):
     '''
     Callee calls `Context.cancel()` while streaming and caller
     sees stream terminated in `ContextCancelled`.
 
     '''
-    async with tractor.open_nursery() as n:
+    async with tractor.open_nursery(
-
+        debug_mode=debug_mode,
-        portal = await n.start_actor(
+    ) as an:
+        portal = await an.start_actor(
             'cancels_self',
             enable_modules=[__name__],
         )
@@ -645,7 +807,7 @@ async def test_callee_cancels_before_started():
 @tractor.context
 async def never_open_stream(
 
-    ctx:  tractor.Context,
+    ctx:  Context,
 
 ) -> None:
     '''
@@ -659,8 +821,8 @@ async def never_open_stream(
 @tractor.context
 async def keep_sending_from_callee(
 
-    ctx:  tractor.Context,
+    ctx:  Context,
-    msg_buffer_size: Optional[int] = None,
+    msg_buffer_size: int|None = None,
 
 ) -> None:
     '''
@@ -685,7 +847,10 @@ async def keep_sending_from_callee(
     ],
     ids='overrun_condition={}'.format,
 )
-def test_one_end_stream_not_opened(overrun_by):
+def test_one_end_stream_not_opened(
+    overrun_by: tuple[str, int, Callable],
+    debug_mode: bool,
+):
     '''
     This should exemplify the bug from:
     https://github.com/goodboy/tractor/issues/265
@@ -696,8 +861,10 @@ def test_one_end_stream_not_opened(overrun_by):
     buf_size = buf_size_increase + Actor.msg_buffer_size
 
     async def main():
-        async with tractor.open_nursery() as n:
+        async with tractor.open_nursery(
-            portal = await n.start_actor(
+            debug_mode=debug_mode,
+        ) as an:
+            portal = await an.start_actor(
                 entrypoint.__name__,
                 enable_modules=[__name__],
             )
@@ -754,7 +921,7 @@ def test_one_end_stream_not_opened(overrun_by):
 @tractor.context
 async def echo_back_sequence(
 
-    ctx:  tractor.Context,
+    ctx:  Context,
     seq: list[int],
     wait_for_cancel: bool,
     allow_overruns_side: str,
@@ -771,7 +938,10 @@ async def echo_back_sequence(
     # NOTE: ensure that if the caller is expecting to cancel this task
     # that we stay echoing much longer then they are so we don't
     # return early instead of receive the cancel msg.
-    total_batches: int = 1000 if wait_for_cancel else 6
+    total_batches: int = (
+        1000 if wait_for_cancel
+        else 6
+    )
 
     await ctx.started()
     # await tractor.breakpoint()
@@ -790,8 +960,23 @@ async def echo_back_sequence(
         )
 
         seq = list(seq)  # bleh, msgpack sometimes ain't decoded right
-        for _ in range(total_batches):
+        for i in range(total_batches):
+            print(f'starting new stream batch {i} iter in child')
             batch = []
+
+            # EoC case, delay a little instead of hot
+            # iter-stopping (since apparently py3.11+ can do that
+            # faster then a ctxc can be sent) on the async for
+            # loop when child was requested to ctxc.
+            if (
+                stream.closed
+                or
+                ctx.cancel_called
+            ):
+                print('child stream already closed!?!')
+                await trio.sleep(0.05)
+                continue
+
             async for msg in stream:
                 batch.append(msg)
                 if batch == seq:
@@ -802,15 +987,18 @@ async def echo_back_sequence(
 
                 print('callee waiting on next')
 
+            print(f'callee echoing back latest batch\n{batch}')
             for msg in batch:
-                print(f'callee sending {msg}')
+                print(f'callee sending msg\n{msg}')
                 await stream.send(msg)
 
-    print(
+    try:
-        'EXITING CALLEEE:\n'
+        return 'yo'
-        f'{ctx.canceller}'
+    finally:
-    )
+        print(
-    return 'yo'
+            'exiting callee with context:\n'
+            f'{pformat(ctx)}\n'
+        )
 
 
 @pytest.mark.parametrize(
@@ -836,7 +1024,10 @@ def test_maybe_allow_overruns_stream(
     cancel_ctx: bool,
     slow_side: str,
     allow_overruns_side: str,
+
+    # conftest wide
     loglevel: str,
+    debug_mode: bool,
 ):
     '''
     Demonstrate small overruns of each task back and forth
@@ -855,13 +1046,14 @@ def test_maybe_allow_overruns_stream(
 
     '''
     async def main():
-        async with tractor.open_nursery() as n:
+        async with tractor.open_nursery(
-            portal = await n.start_actor(
+            debug_mode=debug_mode,
+        ) as an:
+            portal = await an.start_actor(
                 'callee_sends_forever',
                 enable_modules=[__name__],
                 loglevel=loglevel,
-
+                debug_mode=debug_mode,
-                # debug_mode=True,
             )
             seq = list(range(10))
             async with portal.open_context(
@@ -870,8 +1062,8 @@ def test_maybe_allow_overruns_stream(
                 wait_for_cancel=cancel_ctx,
                 be_slow=(slow_side == 'child'),
                 allow_overruns_side=allow_overruns_side,
-            ) as (ctx, sent):
 
+            ) as (ctx, sent):
                 assert sent is None
 
                 async with ctx.open_stream(
@@ -899,10 +1091,10 @@ def test_maybe_allow_overruns_stream(
 
                 if cancel_ctx:
                     # cancel the remote task
-                    print('sending root side cancel')
+                    print('Requesting `ctx.cancel()` in parent!')
                     await ctx.cancel()
 
-            res = await ctx.result()
+            res: str|ContextCancelled = await ctx.result()
 
             if cancel_ctx:
                 assert isinstance(res, ContextCancelled)
@@ -957,3 +1149,52 @@ def test_maybe_allow_overruns_stream(
         # if this hits the logic blocks from above are not
         # exhaustive..
         pytest.fail('PARAMETRIZED CASE GEN PROBLEM YO')
+
+
+def test_ctx_with_self_actor(
+    loglevel: str,
+    debug_mode: bool,
+):
+    '''
+    NOTE: for now this is an INVALID OP!
+
+    BUT, eventually presuming we add a "side" key to `Actor.get_context()`,
+    we might be able to get this working symmetrically, but should we??
+
+    Open a context back to the same actor and ensure all cancellation
+    and error semantics hold the same.
+
+    '''
+    async def main():
+        async with tractor.open_nursery(
+            debug_mode=debug_mode,
+            enable_modules=[__name__],
+        ) as an:
+            assert an
+            async with (
+                tractor.find_actor('root') as portal,
+                portal.open_context(
+                    expect_cancelled,
+                    # echo_back_sequence,
+                    # seq=seq,
+                    # wait_for_cancel=cancel_ctx,
+                    # be_slow=(slow_side == 'child'),
+                    # allow_overruns_side=allow_overruns_side,
+
+                ) as (ctx, sent),
+                ctx.open_stream() as ipc,
+            ):
+                assert sent is None
+
+                seq = list(range(10))
+                for i in seq:
+                    await ipc.send(i)
+                    rx: int = await ipc.receive()
+                    assert rx == i
+
+                await ctx.cancel()
+
+    with pytest.raises(RuntimeError) as excinfo:
+        trio.run(main)
+
+    assert 'Invalid Operation' in repr(excinfo.value)

tests/test_debugger.py

@@ -11,11 +11,9 @@ TODO:
 
 """
 import itertools
-from os import path
 from typing import Optional
 import platform
 import pathlib
-import sys
 import time
 
 import pytest
@@ -25,8 +23,10 @@ from pexpect.exceptions import (
     EOF,
 )
 
-from conftest import (
+from tractor._testing import (
     examples_dir,
+)
+from conftest import (
     _ci_env,
 )
 

tests/test_discovery.py

@@ -9,10 +9,9 @@ import itertools
 
 import pytest
 import tractor
+from tractor._testing import tractor_test
 import trio
 
-from conftest import tractor_test
-
 
 @tractor_test
 async def test_reg_then_unreg(reg_addr):

tests/test_docs_examples.py

@@ -11,8 +11,7 @@ import platform
 import shutil
 
 import pytest
-
+from tractor._testing import (
-from conftest import (
     examples_dir,
 )
 

tests/test_infected_asyncio.py

@@ -8,7 +8,6 @@ import builtins
 import itertools
 import importlib
 
-from exceptiongroup import BaseExceptionGroup
 import pytest
 import trio
 import tractor
@@ -18,6 +17,7 @@ from tractor import (
     ContextCancelled,
 )
 from tractor.trionics import BroadcastReceiver
+from tractor._testing import expect_ctxc
 
 
 async def sleep_and_err(
@@ -68,7 +68,7 @@ def test_trio_cancels_aio_on_actor_side(reg_addr):
 async def asyncio_actor(
 
     target: str,
-    expect_err: Optional[Exception] = None
+    expect_err: Exception|None = None
 
 ) -> None:
 
@@ -112,10 +112,21 @@ def test_aio_simple_error(reg_addr):
                 infect_asyncio=True,
             )
 
-    with pytest.raises(RemoteActorError) as excinfo:
+    with pytest.raises(
+        expected_exception=(RemoteActorError, ExceptionGroup),
+    ) as excinfo:
         trio.run(main)
 
     err = excinfo.value
+
+    # might get multiple `trio.Cancelled`s as well inside an inception
+    if isinstance(err, ExceptionGroup):
+        err = next(itertools.dropwhile(
+            lambda exc: not isinstance(exc, tractor.RemoteActorError),
+            err.exceptions
+        ))
+        assert err
+
     assert isinstance(err, RemoteActorError)
     assert err.type == AssertionError
 
@@ -190,7 +201,8 @@ async def trio_ctx(
 
 
 @pytest.mark.parametrize(
-    'parent_cancels', [False, True],
+    'parent_cancels',
+    ['context', 'actor', False],
     ids='parent_actor_cancels_child={}'.format
 )
 def test_context_spawns_aio_task_that_errors(
@@ -214,18 +226,36 @@ def test_context_spawns_aio_task_that_errors(
                     # debug_mode=True,
                     loglevel='cancel',
                 )
-                async with p.open_context(
+                async with (
-                    trio_ctx,
+                    expect_ctxc(
-                ) as (ctx, first):
+                        yay=parent_cancels == 'actor',
+                    ),
+                    p.open_context(
+                        trio_ctx,
+                    ) as (ctx, first),
+                ):
 
                     assert first == 'start'
 
-                    if parent_cancels:
+                    if parent_cancels == 'actor':
                         await p.cancel_actor()
 
-                    await trio.sleep_forever()
+                    elif parent_cancels == 'context':
+                        await ctx.cancel()
 
-                return await ctx.result()
+                    else:
+                        await trio.sleep_forever()
+
+                async with expect_ctxc(
+                    yay=parent_cancels == 'actor',
+                ):
+                    await ctx.result()
+
+                if parent_cancels == 'context':
+                    # to tear down sub-acor
+                    await p.cancel_actor()
+
+        return ctx.outcome
 
     if parent_cancels:
         # bc the parent made the cancel request,
@@ -269,11 +299,22 @@ def test_aio_cancelled_from_aio_causes_trio_cancelled(reg_addr):
                 infect_asyncio=True,
             )
 
-    with pytest.raises(RemoteActorError) as excinfo:
+    with pytest.raises(
+        expected_exception=(RemoteActorError, ExceptionGroup),
+    ) as excinfo:
         trio.run(main)
 
+    # might get multiple `trio.Cancelled`s as well inside an inception
+    err = excinfo.value
+    if isinstance(err, ExceptionGroup):
+        err = next(itertools.dropwhile(
+            lambda exc: not isinstance(exc, tractor.RemoteActorError),
+            err.exceptions
+        ))
+        assert err
+
     # ensure boxed error is correct
-    assert excinfo.value.type == to_asyncio.AsyncioCancelled
+    assert err.type == to_asyncio.AsyncioCancelled
 
 
 # TODO: verify open_channel_from will fail on this..

tests/test_inter_peer_cancellation.py

@@ -10,6 +10,9 @@ import pytest
 import trio
 import tractor
 from tractor import (  # typing
+    Actor,
+    current_actor,
+    open_nursery,
     Portal,
     Context,
     ContextCancelled,
@@ -123,7 +126,9 @@ async def error_before_started(
                 await peer_ctx.cancel()
 
 
-def test_do_not_swallow_error_before_started_by_remote_contextcancelled():
+def test_do_not_swallow_error_before_started_by_remote_contextcancelled(
+    debug_mode: bool,
+):
     '''
     Verify that an error raised in a remote context which itself
     opens YET ANOTHER remote context, which it then cancels, does not
@@ -132,7 +137,9 @@ def test_do_not_swallow_error_before_started_by_remote_contextcancelled():
 
     '''
     async def main():
-        async with tractor.open_nursery() as n:
+        async with tractor.open_nursery(
+            debug_mode=debug_mode,
+        ) as n:
             portal = await n.start_actor(
                 'errorer',
                 enable_modules=[__name__],
@@ -213,11 +220,12 @@ async def stream_from_peer(
             # - what about IPC-transport specific errors, should
             #   they bubble from the async for and trigger
             #   other special cases?
+            #
             # NOTE: current ctl flow:
             # - stream raises `trio.EndOfChannel` and
             #   exits the loop
-            # - `.open_context()` will raise the ctxcanc
+            # - `.open_context()` will raise the ctxc received
-            #   received from the sleeper.
+            #   from the sleeper.
             async for msg in stream:
                 assert msg is not None
                 print(msg)
@@ -225,25 +233,37 @@ async def stream_from_peer(
     # NOTE: cancellation of the (sleeper) peer should always
     # cause a `ContextCancelled` raise in this streaming
     # actor.
-    except ContextCancelled as ctxerr:
+    except ContextCancelled as ctxc:
-        err = ctxerr
+        ctxerr = ctxc
+
         assert peer_ctx._remote_error is ctxerr
-        assert peer_ctx.canceller == ctxerr.canceller
+        assert peer_ctx._remote_error.msgdata == ctxerr.msgdata
+
+        # the peer ctx is the canceller even though it's canceller
+        # is the "canceller" XD
+        assert peer_name in peer_ctx.canceller
+
+        assert "canceller" in ctxerr.canceller
 
         # caller peer should not be the cancel requester
         assert not ctx.cancel_called
-        # XXX can never be true since `._invoke` only
+        assert not ctx.cancel_acked
+
+        # XXX can NEVER BE TRUE since `._invoke` only
         # sets this AFTER the nursery block this task
         # was started in, exits.
-        assert not ctx.cancelled_caught
+        assert not ctx._scope.cancelled_caught
 
-        # we never requested cancellation
+        # we never requested cancellation, it was the 'canceller'
+        # peer.
         assert not peer_ctx.cancel_called
+        assert not peer_ctx.cancel_acked
+
         # the `.open_context()` exit definitely caught
         # a cancellation in the internal `Context._scope` since
         # likely the runtime called `_deliver_msg()` after
         # receiving the remote error from the streaming task.
-        assert peer_ctx.cancelled_caught
+        assert not peer_ctx._scope.cancelled_caught
 
         # TODO / NOTE `.canceller` won't have been set yet
         # here because that machinery is inside
@@ -252,10 +272,11 @@ async def stream_from_peer(
         # checkpoint) that cancellation was due to
         # a remote, we COULD assert this here..see,
         # https://github.com/goodboy/tractor/issues/368
+        #
+        # assert 'canceller' in ctx.canceller
 
         # root/parent actor task should NEVER HAVE cancelled us!
         assert not ctx.canceller
-        assert 'canceller' in peer_ctx.canceller
 
         raise
         # TODO: IN THEORY we could have other cases depending on
@@ -269,17 +290,17 @@ async def stream_from_peer(
         # assert ctx.canceller[0] == 'root'
         # assert peer_ctx.canceller[0] == 'sleeper'
 
-    raise RuntimeError(
+    raise RuntimeError('Never triggered local `ContextCancelled` ?!?')
-        'peer never triggered local `ContextCancelled`?'
-    )
 
 
 @pytest.mark.parametrize(
     'error_during_ctxerr_handling',
     [False, True],
+    ids=lambda item: f'rte_during_ctxerr={item}',
 )
 def test_peer_canceller(
     error_during_ctxerr_handling: bool,
+    debug_mode: bool,
 ):
     '''
     Verify that a cancellation triggered by an in-actor-tree peer
@@ -336,7 +357,7 @@ def test_peer_canceller(
     async def main():
         async with tractor.open_nursery(
             # NOTE: to halt the peer tasks on ctxc, uncomment this.
-            # debug_mode=True
+            debug_mode=debug_mode,
         ) as an:
             canceller: Portal = await an.start_actor(
                 'canceller',
@@ -350,8 +371,7 @@ def test_peer_canceller(
                 'just_caller',  # but i just met her?
                 enable_modules=[__name__],
             )
-
+            root: Actor = current_actor()
-            root = tractor.current_actor()
 
             try:
                 async with (
@@ -369,15 +389,16 @@ def test_peer_canceller(
                     ) as (canceller_ctx, sent),
 
                 ):
-                    ctxs: list[Context] = [
+                    ctxs: dict[str, Context] = {
-                        sleeper_ctx,
+                        'sleeper': sleeper_ctx,
-                        caller_ctx,
+                        'caller': caller_ctx,
-                        canceller_ctx,
+                        'canceller': canceller_ctx,
-                    ]
+                    }
 
                     try:
                         print('PRE CONTEXT RESULT')
-                        await sleeper_ctx.result()
+                        res = await sleeper_ctx.result()
+                        assert res
 
                         # should never get here
                         pytest.fail(
@@ -387,13 +408,19 @@ def test_peer_canceller(
                     # should always raise since this root task does
                     # not request the sleeper cancellation ;)
                     except ContextCancelled as ctxerr:
-                        print(f'CAUGHT REMOTE CONTEXT CANCEL {ctxerr}')
+                        print(
+                            'CAUGHT REMOTE CONTEXT CANCEL\n\n'
+                            f'{ctxerr}\n'
+                        )
 
                         # canceller and caller peers should not
                         # have been remotely cancelled.
                         assert canceller_ctx.canceller is None
                         assert caller_ctx.canceller is None
 
+                        # we were not the actor, our peer was
+                        assert not sleeper_ctx.cancel_acked
+
                         assert ctxerr.canceller[0] == 'canceller'
 
                         # XXX NOTE XXX: since THIS `ContextCancelled`
@@ -401,123 +428,273 @@ def test_peer_canceller(
                         # `sleeper.open_context().__aexit__()` this
                         # value is not yet set, however outside this
                         # block it should be.
-                        assert not sleeper_ctx.cancelled_caught
+                        assert not sleeper_ctx._scope.cancelled_caught
 
+                        # CASE_1: error-during-ctxc-handling,
                         if error_during_ctxerr_handling:
                             raise RuntimeError('Simulated error during teardown')
 
+                        # CASE_2: standard teardown inside in `.open_context()` block
                         raise
 
                     # XXX SHOULD NEVER EVER GET HERE XXX
                     except BaseException as berr:
-                        err = berr
+                        raise
-                        pytest.fail('did not rx ctx-cancelled error?')
+
+                        # XXX if needed to debug failure
+                        # _err = berr
+                        # await tractor.pause()
+                        # await trio.sleep_forever()
+
+                        pytest.fail(
+                            'did not rx ctxc ?!?\n\n'
+
+                            f'{berr}\n'
+                        )
+
                     else:
-                        pytest.fail('did not rx ctx-cancelled error?')
+                        pytest.fail(
+                            'did not rx ctxc ?!?\n\n'
+                            f'{ctxs}\n'
+                        )
 
             except (
                 ContextCancelled,
                 RuntimeError,
-            )as ctxerr:
+            )as loc_err:
-                _err = ctxerr
+                _loc_err = loc_err
 
                 # NOTE: the main state to check on `Context` is:
-                # - `.cancelled_caught` (maps to nursery cs)
                 # - `.cancel_called` (bool of whether this side
                 #    requested)
+                # - `.cancel_acked` (bool of whether a ctxc
+                #   response was received due to cancel req).
+                # - `.maybe_error` (highest prio error to raise
+                #    locally)
+                # - `.outcome` (final error or result value)
                 # - `.canceller` (uid of cancel-causing actor-task)
                 # - `._remote_error` (any `RemoteActorError`
                 #    instance from other side of context)
+                # - `._local_error` (any error caught inside the
+                #   `.open_context()` block).
+                #
+                # XXX: Deprecated and internal only
+                # - `.cancelled_caught` (maps to nursery cs)
+                #  - now just use `._scope.cancelled_caught`
+                #    since it maps to the internal (maps to nursery cs)
+                #
                 # TODO: are we really planning to use this tho?
                 # - `._cancel_msg` (any msg that caused the
                 #    cancel)
 
-                # CASE: error raised during handling of
+                # CASE_1: error-during-ctxc-handling,
-                # `ContextCancelled` inside `.open_context()`
+                # - far end cancels due to peer 'canceller',
-                # block
+                # - `ContextCancelled` relayed to this scope,
+                # - inside `.open_context()` ctxc is caught and
+                #   a rte raised instead
+                #
+                # => block should raise the rte but all peers
+                #   should be cancelled by US.
+                #
                 if error_during_ctxerr_handling:
-                    assert isinstance(ctxerr, RuntimeError)
+                    # since we do a rte reraise above, the
+                    # `.open_context()` error handling should have
+                    # raised a local rte, thus the internal
+                    # `.open_context()` enterer task's
+                    # cancel-scope should have raised the RTE, NOT
+                    # a `trio.Cancelled` due to a local
+                    # `._scope.cancel()` call.
+                    assert not sleeper_ctx._scope.cancelled_caught
+
+                    assert isinstance(loc_err, RuntimeError)
+                    print(f'_loc_err: {_loc_err}\n')
+                    # assert sleeper_ctx._local_error is _loc_err
+                    # assert sleeper_ctx._local_error is _loc_err
+                    assert not (
+                        loc_err
+                        is sleeper_ctx.maybe_error
+                        is sleeper_ctx.outcome
+                        is sleeper_ctx._remote_error
+                    )
 
                     # NOTE: this root actor task should have
                     # called `Context.cancel()` on the
                     # `.__aexit__()` to every opened ctx.
-                    for ctx in ctxs:
+                    for name, ctx in ctxs.items():
-                        assert ctx.cancel_called
 
                         # this root actor task should have
                         # cancelled all opened contexts except the
                         # sleeper which is obvi by the "canceller"
                         # peer.
                         re = ctx._remote_error
-                        if (
+                        le = ctx._local_error
-                            ctx is sleeper_ctx
-                            or ctx is caller_ctx
-                        ):
-                            assert (
-                                re.canceller
-                                ==
-                                ctx.canceller
-                                ==
-                                canceller.channel.uid
-                            )
 
-                        else:
+                        assert ctx.cancel_called
+
+                        if ctx is sleeper_ctx:
+                            assert 'canceller' in re.canceller
+                            assert 'sleeper' in ctx.canceller
+
+                        if ctx is canceller_ctx:
                             assert (
                                 re.canceller
                                 ==
-                                ctx.canceller
-                                ==
                                 root.uid
                             )
 
-                # CASE: standard teardown inside in `.open_context()` block
+                        else:  # the other 2 ctxs
-                else:
+                            assert (
-                    assert ctxerr.canceller == sleeper_ctx.canceller
+                                re.canceller
+                                ==
+                                canceller.channel.uid
+                            )
+
+                    # since the sleeper errors while handling a
+                    # peer-cancelled (by ctxc) scenario, we expect
+                    # that the `.open_context()` block DOES call
+                    # `.cancel() (despite in this test case it
+                    # being unecessary).
                     assert (
-                        ctxerr.canceller[0]
+                        sleeper_ctx.cancel_called
-                        ==
+                        and
-                        sleeper_ctx.canceller[0]
+                        not sleeper_ctx.cancel_acked
-                        ==
-                        'canceller'
                     )
 
+                # CASE_2: standard teardown inside in `.open_context()` block
+                # - far end cancels due to peer 'canceller',
+                # - `ContextCancelled` relayed to this scope and
+                #   raised locally without any raise-during-handle,
+                #
+                # => inside `.open_context()` ctxc is raised and
+                #   propagated
+                #
+                else:
+                    # since sleeper_ctx.result() IS called above
+                    # we should have (silently) absorbed the
+                    # corresponding `ContextCancelled` for it and
+                    # `._scope.cancel()` should never have been
+                    # called.
+                    assert not sleeper_ctx._scope.cancelled_caught
+
+                    assert isinstance(loc_err, ContextCancelled)
+
+                    # the received remote error's `.canceller`
+                    # will of course be the "canceller" actor BUT
+                    # the canceller set on the local handle to
+                    # `sleeper_ctx` will be the "sleeper" uid
+                    # since it's the actor that relayed us the
+                    # error which was **caused** by the
+                    # "canceller".
+                    assert 'sleeper' in sleeper_ctx.canceller
+                    assert 'canceller' == loc_err.canceller[0]
+
                     # the sleeper's remote error is the error bubbled
                     # out of the context-stack above!
-                    re = sleeper_ctx._remote_error
+                    final_err = sleeper_ctx.outcome
-                    assert re is ctxerr
+                    assert (
+                        final_err is loc_err
+                        is sleeper_ctx.maybe_error
+                        is sleeper_ctx._remote_error
+                    )
 
-                    for ctx in ctxs:
+                    for name, ctx in ctxs.items():
-                        re: BaseException | None = ctx._remote_error
+
-                        assert re
+                        re: BaseException|None = ctx._remote_error
+                        le: BaseException|None = ctx._local_error
+                        err = ctx.maybe_error
+                        out = ctx.outcome
+
+                        # every ctx should error!
+                        assert out is err
+
+                        # the recorded local erro should always be
+                        # the same as the one raised by the
+                        # `sleeper_ctx.result()` call
+                        assert (
+                            le
+                            and
+                            le is loc_err
+                        )
 
                         # root doesn't cancel sleeper since it's
                         # cancelled by its peer.
                         if ctx is sleeper_ctx:
+                            assert re
+                            assert (
+                                ctx._remote_error
+                                is ctx.maybe_error
+                                is ctx.outcome
+                                is ctx._local_error
+                            )
+
                             assert not ctx.cancel_called
+                            assert not ctx.cancel_acked
+
                             # since sleeper_ctx.result() IS called
                             # above we should have (silently)
                             # absorbed the corresponding
                             # `ContextCancelled` for it and thus
                             # the logic inside `.cancelled_caught`
                             # should trigger!
-                            assert ctx.cancelled_caught
+                            assert not ctx._scope.cancelled_caught
 
-                        elif ctx is caller_ctx:
+                        elif ctx in (
-                            # since its context was remotely
+                            caller_ctx,
-                            # cancelled, we never needed to
+                            canceller_ctx,
-                            # call `Context.cancel()` bc it was
+                        ):
-                            # done by the peer and also we never 
-                            assert ctx.cancel_called
 
-                            # TODO: figure out the details of
+                            assert not ctx._remote_error
-                            # this..
+
+                            # neither of the `caller/canceller_ctx` should
+                            # have called `ctx.cancel()` bc the
+                            # canceller's task internally issues
+                            # a `Portal.cancel_actor()` to the
+                            # sleeper and thus never should call
+                            # `ctx.cancel()` per say UNLESS the
+                            # sleeper's `.result()` call above
+                            # ctxc exception results in the
+                            # canceller's
+                            # `.open_context().__aexit__()` error
+                            # handling to kick in BEFORE a remote
+                            # error is delivered - which since
+                            # we're asserting what we are above,
+                            # that should normally be the case
+                            # right?
+                            #
+                            assert not ctx.cancel_called
+                            #
+                            # assert ctx.cancel_called
+                            # orig ^
+
+                            # TODO: figure out the details of this..?
                             # if you look the `._local_error` here
                             # is a multi of ctxc + 2 Cancelleds?
-                            # assert not ctx.cancelled_caught
+                            # assert not ctx._scope.cancelled_caught
+
+                            assert (
+                                not ctx.cancel_called
+                                and not ctx.cancel_acked
+                            )
+                            assert not ctx._scope.cancelled_caught
+
+                        # elif ctx is canceller_ctx:
+                        #     assert not ctx._remote_error
+
+                            # XXX NOTE XXX: ONLY the canceller
+                            # will get a self-cancelled outcome
+                            # whilst everyone else gets
+                            # a peer-caused cancellation!
+                            #
+                            # TODO: really we should avoid calling
+                            # .cancel() whenever an interpeer
+                            # cancel takes place since each
+                            # reception of a ctxc
 
                         else:
-                            assert ctx.cancel_called
+                            pytest.fail(
-                            assert not ctx.cancelled_caught
+                                'Uhh wut ctx is this?\n'
+                                f'{ctx}\n'
+                            )
 
                         # TODO: do we even need this flag?
                         # -> each context should have received
@@ -533,14 +710,24 @@ def test_peer_canceller(
                     # `Context.cancel()` SHOULD NOT have been
                     # called inside
                     # `Portal.open_context().__aexit__()`.
-                    assert not sleeper_ctx.cancel_called
+                    assert not (
+                        sleeper_ctx.cancel_called
+                        or
+                        sleeper_ctx.cancel_acked
+                    )
 
                 # XXX NOTE XXX: and see matching comment above but,
-                # this flag is set only AFTER the `.open_context()`
+                # the `._scope` is only set by `trio` AFTER the
-                # has exited and should be set in both outcomes
+                # `.open_context()` block has exited and should be
-                # including the case where ctx-cancel handling
+                # set in both outcomes including the case where
-                # itself errors.
+                # ctx-cancel handling itself errors.
-                assert sleeper_ctx.cancelled_caught
+                assert not sleeper_ctx._scope.cancelled_caught
+                assert _loc_err is sleeper_ctx._local_error
+                assert (
+                    sleeper_ctx.outcome
+                    is sleeper_ctx.maybe_error
+                    is sleeper_ctx._remote_error
+                )
 
                 raise  # always to ensure teardown
 
@@ -554,3 +741,317 @@ def test_peer_canceller(
 
         assert excinfo.value.type == ContextCancelled
         assert excinfo.value.canceller[0] == 'canceller'
+
+
+@tractor.context
+async def basic_echo_server(
+    ctx: Context,
+    peer_name: str = 'stepbro',
+
+) -> None:
+    '''
+    Just the simplest `MsgStream` echo server which resays what
+    you told it but with its uid in front ;)
+
+    '''
+    actor: Actor = tractor.current_actor()
+    uid: tuple = actor.uid
+    await ctx.started(uid)
+    async with ctx.open_stream() as ipc:
+        async for msg in ipc:
+
+            # repack msg pair with our uid
+            # as first element.
+            (
+                client_uid,
+                i,
+            ) = msg
+            resp: tuple = (
+                uid,
+                i,
+            )
+            # OOF! looks like my runtime-error is causing a lockup
+            # assert 0
+            await ipc.send(resp)
+
+
+@tractor.context
+async def serve_subactors(
+    ctx: Context,
+    peer_name: str,
+
+) -> None:
+    async with open_nursery() as an:
+        await ctx.started(peer_name)
+        async with ctx.open_stream() as reqs:
+            async for msg in reqs:
+                peer_name: str = msg
+                peer: Portal = await an.start_actor(
+                    name=peer_name,
+                    enable_modules=[__name__],
+                )
+                print(
+                    'Spawning new subactor\n'
+                    f'{peer_name}\n'
+                    f'|_{peer}\n'
+                )
+                await reqs.send((
+                    peer.chan.uid,
+                    peer.chan.raddr,
+                ))
+
+        print('Spawner exiting spawn serve loop!')
+
+
+@tractor.context
+async def client_req_subactor(
+    ctx: Context,
+    peer_name: str,
+
+    # used to simulate a user causing an error to be raised
+    # directly in thread (like a KBI) to better replicate the
+    # case where a `modden` CLI client would hang afer requesting
+    # a `Context.cancel()` to `bigd`'s wks spawner.
+    reraise_on_cancel: str|None = None,
+
+) -> None:
+    # TODO: other cases to do with sub lifetimes:
+    # -[ ] test that we can have the server spawn a sub
+    #   that lives longer then ctx with this client.
+    # -[ ] test that
+
+    # open ctx with peer spawn server and ask it to spawn a little
+    # bro which we'll then connect and stream with.
+    async with (
+        tractor.find_actor(
+            name='spawn_server',
+            raise_on_none=True,
+
+            # TODO: we should be isolating this from other runs!
+            # => ideally so we can eventually use something like
+            # `pytest-xdist` Bo
+            # registry_addrs=bigd._reg_addrs,
+        ) as spawner,
+
+        spawner.open_context(
+            serve_subactors,
+            peer_name=peer_name,
+        ) as (spawner_ctx, first),
+    ):
+        assert first == peer_name
+        await ctx.started(
+            'yup i had brudder',
+        )
+
+        async with spawner_ctx.open_stream() as reqs:
+
+            # send single spawn request to the server
+            await reqs.send(peer_name)
+            with trio.fail_after(3):
+                (
+                    sub_uid,
+                    sub_raddr,
+                ) = await reqs.receive()
+
+
+            await tell_little_bro(
+                actor_name=sub_uid[0],
+                caller='client',
+            )
+
+            # TODO: test different scope-layers of
+            # cancellation?
+            # with trio.CancelScope() as cs:
+            try:
+                await trio.sleep_forever()
+
+            # TODO: would be super nice to have a special injected
+            # cancel type here (maybe just our ctxc) but using
+            # some native mechanism in `trio` :p
+            except (
+                trio.Cancelled
+            ) as err:
+                _err = err
+                if reraise_on_cancel:
+                    errtype = globals()['__builtins__'][reraise_on_cancel]
+                    assert errtype
+                    to_reraise: BaseException = errtype()
+                    print(f'client re-raising on cancel: {repr(to_reraise)}')
+                    raise err
+
+                raise
+
+            # if cs.cancelled_caught:
+            #     print('client handling expected KBI!')
+            #     await ctx.
+            #     await trio.sleep(
+            #     await tractor.pause()
+            #     await spawner_ctx.cancel()
+
+            # cancel spawned sub-actor directly?
+            # await sub_ctx.cancel()
+
+            # maybe cancel runtime?
+            # await sub.cancel_actor()
+
+
+async def tell_little_bro(
+    actor_name: str,
+    caller: str = ''
+):
+    # contact target actor, do a stream dialog.
+    async with (
+        tractor.wait_for_actor(
+            name=actor_name
+        ) as lb,
+        lb.open_context(
+            basic_echo_server,
+        ) as (sub_ctx, first),
+        sub_ctx.open_stream(
+            basic_echo_server,
+        ) as echo_ipc,
+    ):
+        actor: Actor = current_actor()
+        uid: tuple = actor.uid
+        for i in range(100):
+            msg: tuple = (
+                uid,
+                i,
+            )
+            await echo_ipc.send(msg)
+            resp = await echo_ipc.receive()
+            print(
+                f'{caller} => {actor_name}: {msg}\n'
+                f'{caller} <= {actor_name}: {resp}\n'
+            )
+            (
+                sub_uid,
+                _i,
+            ) = resp
+            assert sub_uid != uid
+            assert _i == i
+
+
+@pytest.mark.parametrize(
+    'raise_client_error',
+    [None, 'KeyboardInterrupt'],
+)
+def test_peer_spawns_and_cancels_service_subactor(
+    debug_mode: bool,
+    raise_client_error: str,
+):
+    # NOTE: this tests for the modden `mod wks open piker` bug
+    # discovered as part of implementing workspace ctx
+    # open-.pause()-ctx.cancel() as part of the CLI..
+
+    # -> start actor-tree (server) that offers sub-actor spawns via
+    #   context API
+    # -> start another full actor-tree (client) which requests to the first to
+    #   spawn over its `@context` ep / api.
+    # -> client actor cancels the context and should exit gracefully
+    #   and the server's spawned child should cancel and terminate!
+    peer_name: str = 'little_bro'
+
+    async def main():
+        async with tractor.open_nursery(
+            # NOTE: to halt the peer tasks on ctxc, uncomment this.
+            debug_mode=debug_mode,
+        ) as an:
+            server: Portal = await an.start_actor(
+                (server_name := 'spawn_server'),
+                enable_modules=[__name__],
+            )
+            print(f'Spawned `{server_name}`')
+
+            client: Portal = await an.start_actor(
+                client_name := 'client',
+                enable_modules=[__name__],
+            )
+            print(f'Spawned `{client_name}`')
+
+            try:
+                async with (
+                    server.open_context(
+                        serve_subactors,
+                        peer_name=peer_name,
+                    ) as (spawn_ctx, first),
+
+                    client.open_context(
+                        client_req_subactor,
+                        peer_name=peer_name,
+                        reraise_on_cancel=raise_client_error,
+                    ) as (client_ctx, client_says),
+                ):
+                    print(
+                        f'Server says: {first}\n'
+                        f'Client says: {client_says}\n'
+                    )
+
+                    # attach to client-requested-to-spawn
+                    # (grandchild of this root actor) "little_bro"
+                    # and ensure we can also use it as an echo
+                    # server.
+                    async with tractor.wait_for_actor(
+                        name=peer_name,
+                    ) as sub:
+                        assert sub
+
+                    print(
+                        'Sub-spawn came online\n'
+                        f'portal: {sub}\n'
+                        f'.uid: {sub.actor.uid}\n'
+                        f'chan.raddr: {sub.chan.raddr}\n'
+                    )
+                    await tell_little_bro(
+                        actor_name=peer_name,
+                        caller='root',
+                    )
+
+                    # signal client to raise a KBI
+                    await client_ctx.cancel()
+                    print('root cancelled client, checking that sub-spawn is down')
+
+                    async with tractor.find_actor(
+                        name=peer_name,
+                    ) as sub:
+                        assert not sub
+
+                    print('root cancelling server/client sub-actors')
+
+                    # await tractor.pause()
+                    res = await client_ctx.result(hide_tb=False)
+                    assert isinstance(res, ContextCancelled)
+                    assert client_ctx.cancel_acked
+                    assert res.canceller == current_actor().uid
+
+                    await spawn_ctx.cancel()
+                    # await server.cancel_actor()
+
+            # since we called `.cancel_actor()`, `.cancel_ack`
+            # will not be set on the ctx bc `ctx.cancel()` was not
+            # called directly fot this confext.
+            except ContextCancelled as ctxc:
+                print('caught ctxc from contexts!')
+                assert ctxc.canceller == current_actor().uid
+                assert ctxc is spawn_ctx.outcome
+                assert ctxc is spawn_ctx.maybe_error
+                raise
+
+            # assert spawn_ctx.cancel_acked
+            assert spawn_ctx.cancel_acked
+            assert client_ctx.cancel_acked
+
+            await client.cancel_actor()
+            await server.cancel_actor()
+
+            # WOA WOA WOA! we need this to close..!!!??
+            # that's super bad XD
+
+            # TODO: why isn't this working!?!?
+            # we're now outside the `.open_context()` block so
+            # the internal `Context._scope: CancelScope` should be
+            # gracefully "closed" ;)
+
+            # assert spawn_ctx.cancelled_caught
+
+    trio.run(main)

tests/test_legacy_one_way_streaming.py

@@ -9,7 +9,7 @@ import trio
 import tractor
 import pytest
 
-from conftest import tractor_test
+from tractor._testing import tractor_test
 
 
 def test_must_define_ctx():

tests/test_local.py

@@ -7,7 +7,7 @@ import pytest
 import trio
 import tractor
 
-from conftest import tractor_test
+from tractor._testing import tractor_test
 
 
 @pytest.mark.trio

tests/test_multi_program.py

@@ -7,8 +7,10 @@ import time
 import pytest
 import trio
 import tractor
-from conftest import (
+from tractor._testing import (
     tractor_test,
+)
+from conftest import (
     sig_prog,
     _INT_SIGNAL,
     _INT_RETURN_CODE,

tests/test_pubsub.py

@@ -5,8 +5,7 @@ import pytest
 import trio
 import tractor
 from tractor.experimental import msgpub
-
+from tractor._testing import tractor_test
-from conftest import tractor_test
 
 
 def test_type_checks():

tests/test_rpc.py

@@ -1,6 +1,8 @@
-"""
+'''
-RPC related
+RPC (or maybe better labelled as "RTS: remote task scheduling"?)
-"""
+related API and error checks.
+
+'''
 import itertools
 
 import pytest
@@ -52,8 +54,13 @@ async def short_sleep():
         (['tmp_mod'], 'import doggy', ModuleNotFoundError),
         (['tmp_mod'], '4doggy', SyntaxError),
     ],
-    ids=['no_mods', 'this_mod', 'this_mod_bad_func', 'fail_to_import',
+    ids=[
-         'fail_on_syntax'],
+        'no_mods',
+        'this_mod',
+        'this_mod_bad_func',
+        'fail_to_import',
+        'fail_on_syntax',
+    ],
 )
 def test_rpc_errors(
     reg_addr,
@@ -127,14 +134,16 @@ def test_rpc_errors(
         run()
     else:
         # underlying errors aren't propagated upwards (yet)
-        with pytest.raises(remote_err) as err:
+        with pytest.raises(
+            expected_exception=(remote_err, ExceptionGroup),
+        ) as err:
             run()
 
         # get raw instance from pytest wrapper
         value = err.value
 
         # might get multiple `trio.Cancelled`s as well inside an inception
-        if isinstance(value, trio.MultiError):
+        if isinstance(value, ExceptionGroup):
             value = next(itertools.dropwhile(
                 lambda exc: not isinstance(exc, tractor.RemoteActorError),
                 value.exceptions

tests/test_runtime.py

@@ -8,7 +8,7 @@ import pytest
 import trio
 import tractor
 
-from conftest import tractor_test
+from tractor._testing import tractor_test
 
 
 _file_path: str = ''
@@ -64,7 +64,8 @@ async def test_lifetime_stack_wipes_tmpfile(
 
     except (
         tractor.RemoteActorError,
-        tractor.BaseExceptionGroup,
+        # tractor.BaseExceptionGroup,
+        BaseExceptionGroup,
     ):
         pass
 

tests/test_shm.py

@@ -1,167 +0,0 @@
-"""
-Shared mem primitives and APIs.
-
-"""
-import uuid
-
-# import numpy
-import pytest
-import trio
-import tractor
-from tractor._shm import (
-    open_shm_list,
-    attach_shm_list,
-)
-
-
-@tractor.context
-async def child_attach_shml_alot(
-    ctx: tractor.Context,
-    shm_key: str,
-) -> None:
-
-    await ctx.started(shm_key)
-
-    # now try to attach a boatload of times in a loop..
-    for _ in range(1000):
-        shml = attach_shm_list(
-            key=shm_key,
-            readonly=False,
-        )
-        assert shml.shm.name == shm_key
-        await trio.sleep(0.001)
-
-
-def test_child_attaches_alot():
-    async def main():
-        async with tractor.open_nursery() as an:
-
-            # allocate writeable list in parent
-            key = f'shml_{uuid.uuid4()}'
-            shml = open_shm_list(
-                key=key,
-            )
-
-            portal = await an.start_actor(
-                'shm_attacher',
-                enable_modules=[__name__],
-            )
-
-            async with (
-                portal.open_context(
-                    child_attach_shml_alot,
-                    shm_key=shml.key,
-                ) as (ctx, start_val),
-            ):
-                assert start_val == key
-                await ctx.result()
-
-            await portal.cancel_actor()
-
-    trio.run(main)
-
-
-@tractor.context
-async def child_read_shm_list(
-    ctx: tractor.Context,
-    shm_key: str,
-    use_str: bool,
-    frame_size: int,
-) -> None:
-
-    # attach in child
-    shml = attach_shm_list(
-        key=shm_key,
-        # dtype=str if use_str else float,
-    )
-    await ctx.started(shml.key)
-
-    async with ctx.open_stream() as stream:
-        async for i in stream:
-            print(f'(child): reading shm list index: {i}')
-
-            if use_str:
-                expect = str(float(i))
-            else:
-                expect = float(i)
-
-            if frame_size == 1:
-                val = shml[i]
-                assert expect == val
-                print(f'(child): reading value: {val}')
-            else:
-                frame = shml[i - frame_size:i]
-                print(f'(child): reading frame: {frame}')
-
-
-@pytest.mark.parametrize(
-    'use_str',
-    [False, True],
-    ids=lambda i: f'use_str_values={i}',
-)
-@pytest.mark.parametrize(
-    'frame_size',
-    [1, 2**6, 2**10],
-    ids=lambda i: f'frame_size={i}',
-)
-def test_parent_writer_child_reader(
-    use_str: bool,
-    frame_size: int,
-):
-
-    async def main():
-        async with tractor.open_nursery(
-            # debug_mode=True,
-        ) as an:
-
-            portal = await an.start_actor(
-                'shm_reader',
-                enable_modules=[__name__],
-                debug_mode=True,
-            )
-
-            # allocate writeable list in parent
-            key = 'shm_list'
-            seq_size = int(2 * 2 ** 10)
-            shml = open_shm_list(
-                key=key,
-                size=seq_size,
-                dtype=str if use_str else float,
-                readonly=False,
-            )
-
-            async with (
-                portal.open_context(
-                    child_read_shm_list,
-                    shm_key=key,
-                    use_str=use_str,
-                    frame_size=frame_size,
-                ) as (ctx, sent),
-
-                ctx.open_stream() as stream,
-            ):
-
-                assert sent == key
-
-                for i in range(seq_size):
-
-                    val = float(i)
-                    if use_str:
-                        val = str(val)
-
-                    # print(f'(parent): writing {val}')
-                    shml[i] = val
-
-                    # only on frame fills do we
-                    # signal to the child that a frame's
-                    # worth is ready.
-                    if (i % frame_size) == 0:
-                        print(f'(parent): signalling frame full on {val}')
-                        await stream.send(i)
-                else:
-                    print(f'(parent): signalling final frame on {val}')
-                    await stream.send(i)
-
-            await portal.cancel_actor()
-
-    trio.run(main)

tests/test_spawning.py

@@ -8,7 +8,7 @@ import pytest
 import trio
 import tractor
 
-from conftest import tractor_test
+from tractor._testing import tractor_test
 
 data_to_pass_down = {'doggy': 10, 'kitty': 4}
 

tests/test_trioisms.py

@@ -5,7 +5,7 @@ want to see changed.
 '''
 import pytest
 import trio
-from trio_typing import TaskStatus
+from trio import TaskStatus
 
 
 @pytest.mark.parametrize(

tractor/_debug.py

@@ -27,11 +27,7 @@ from functools import (
     partial,
     cached_property,
 )
-from contextlib import (
+from contextlib import asynccontextmanager as acm
-    asynccontextmanager as acm,
-    contextmanager as cm,
-    nullcontext,
-)
 from typing import (
     Any,
     Callable,
@@ -48,24 +44,22 @@ from trio_typing import (
     # Task,
 )
 
-from ..log import get_logger
+from .log import get_logger
-from .._state import (
+from ._discovery import get_root
+from ._state import (
     is_root_process,
     debug_mode,
 )
-from .._exceptions import (
+from ._exceptions import (
     is_multi_cancelled,
     ContextCancelled,
 )
-from .._ipc import Channel
+from ._ipc import Channel
 
 log = get_logger(__name__)
 
 
-__all__ = [
+__all__ = ['breakpoint', 'post_mortem']
-    'breakpoint',
-    'post_mortem',
-]
 
 
 class Lock:
@@ -330,7 +324,7 @@ async def lock_tty_for_child(
             f'Actor {subactor_uid} is blocked from acquiring debug lock\n'
             f"remote task: {task_name}:{subactor_uid}"
         )
-        ctx._enter_debugger_on_cancel: bool = False
+        ctx._enter_debugger_on_cancel = False
         await ctx.cancel(f'Debug lock blocked for {subactor_uid}')
         return 'pdb_lock_blocked'
 
@@ -381,14 +375,12 @@ async def wait_for_parent_stdin_hijack(
 
     This function is used by any sub-actor to acquire mutex access to
     the ``pdb`` REPL and thus the root's TTY for interactive debugging
-    (see below inside ``pause()``). It can be used to ensure that
+    (see below inside ``_pause()``). It can be used to ensure that
     an intermediate nursery-owning actor does not clobber its children
     if they are in debug (see below inside
     ``maybe_wait_for_debugger()``).
 
     '''
-    from .._discovery import get_root
-
     with trio.CancelScope(shield=True) as cs:
         Lock._debugger_request_cs = cs
 
@@ -398,7 +390,7 @@ async def wait_for_parent_stdin_hijack(
                 # this syncs to child's ``Context.started()`` call.
                 async with portal.open_context(
 
-                    lock_tty_for_child,
+                    tractor._debug.lock_tty_for_child,
                     subactor_uid=actor_uid,
 
                 ) as (ctx, val):
@@ -449,6 +441,171 @@ def mk_mpdb() -> tuple[MultiActorPdb, Callable]:
     return pdb, Lock.unshield_sigint
 
 
+async def _pause(
+
+    debug_func: Callable | None = None,
+    release_lock_signal: trio.Event | None = None,
+
+    # TODO:
+    # shield: bool = False
+    task_status: TaskStatus[trio.Event] = trio.TASK_STATUS_IGNORED
+
+) -> None:
+    '''
+    A pause point (more commonly known as a "breakpoint") interrupt
+    instruction for engaging a blocking debugger instance to
+    conduct manual console-based-REPL-interaction from within
+    `tractor`'s async runtime, normally from some single-threaded
+    and currently executing actor-hosted-`trio`-task in some
+    (remote) process.
+
+    NOTE: we use the semantics "pause" since it better encompasses
+    the entirety of the necessary global-runtime-state-mutation any
+    actor-task must access and lock in order to get full isolated
+    control over the process tree's root TTY:
+    https://en.wikipedia.org/wiki/Breakpoint
+
+    '''
+    __tracebackhide__ = True
+    actor = tractor.current_actor()
+    pdb, undo_sigint = mk_mpdb()
+    task_name = trio.lowlevel.current_task().name
+
+    # TODO: is it possible to debug a trio.Cancelled except block?
+    # right now it seems like we can kinda do with by shielding
+    # around ``tractor.breakpoint()`` but not if we move the shielded
+    # scope here???
+    # with trio.CancelScope(shield=shield):
+    #     await trio.lowlevel.checkpoint()
+
+    if (
+        not Lock.local_pdb_complete
+        or Lock.local_pdb_complete.is_set()
+    ):
+        Lock.local_pdb_complete = trio.Event()
+
+    # TODO: need a more robust check for the "root" actor
+    if (
+        not is_root_process()
+        and actor._parent_chan  # a connected child
+    ):
+
+        if Lock.local_task_in_debug:
+
+            # Recurrence entry case: this task already has the lock and
+            # is likely recurrently entering a breakpoint
+            if Lock.local_task_in_debug == task_name:
+                # noop on recurrent entry case but we want to trigger
+                # a checkpoint to allow other actors error-propagate and
+                # potetially avoid infinite re-entries in some subactor.
+                await trio.lowlevel.checkpoint()
+                return
+
+            # if **this** actor is already in debug mode block here
+            # waiting for the control to be released - this allows
+            # support for recursive entries to `tractor.breakpoint()`
+            log.warning(f"{actor.uid} already has a debug lock, waiting...")
+
+            await Lock.local_pdb_complete.wait()
+            await trio.sleep(0.1)
+
+        # mark local actor as "in debug mode" to avoid recurrent
+        # entries/requests to the root process
+        Lock.local_task_in_debug = task_name
+
+        # this **must** be awaited by the caller and is done using the
+        # root nursery so that the debugger can continue to run without
+        # being restricted by the scope of a new task nursery.
+
+        # TODO: if we want to debug a trio.Cancelled triggered exception
+        # we have to figure out how to avoid having the service nursery
+        # cancel on this task start? I *think* this works below:
+        # ```python
+        #   actor._service_n.cancel_scope.shield = shield
+        # ```
+        # but not entirely sure if that's a sane way to implement it?
+        try:
+            with trio.CancelScope(shield=True):
+                await actor._service_n.start(
+                    wait_for_parent_stdin_hijack,
+                    actor.uid,
+                )
+                Lock.repl = pdb
+        except RuntimeError:
+            Lock.release()
+
+            if actor._cancel_called:
+                # service nursery won't be usable and we
+                # don't want to lock up the root either way since
+                # we're in (the midst of) cancellation.
+                return
+
+            raise
+
+    elif is_root_process():
+
+        # we also wait in the root-parent for any child that
+        # may have the tty locked prior
+        # TODO: wait, what about multiple root tasks acquiring it though?
+        if Lock.global_actor_in_debug == actor.uid:
+            # re-entrant root process already has it: noop.
+            return
+
+        # XXX: since we need to enter pdb synchronously below,
+        # we have to release the lock manually from pdb completion
+        # callbacks. Can't think of a nicer way then this atm.
+        if Lock._debug_lock.locked():
+            log.warning(
+                'Root actor attempting to shield-acquire active tty lock'
+                f' owned by {Lock.global_actor_in_debug}')
+
+            # must shield here to avoid hitting a ``Cancelled`` and
+            # a child getting stuck bc we clobbered the tty
+            with trio.CancelScope(shield=True):
+                await Lock._debug_lock.acquire()
+        else:
+            # may be cancelled
+            await Lock._debug_lock.acquire()
+
+        Lock.global_actor_in_debug = actor.uid
+        Lock.local_task_in_debug = task_name
+        Lock.repl = pdb
+
+    try:
+        # breakpoint()
+        if debug_func is None:
+            # assert release_lock_signal, (
+            #     'Must pass `release_lock_signal: trio.Event` if no '
+            #     'trace func provided!'
+            # )
+            print(f"{actor.uid} ENTERING WAIT")
+            task_status.started()
+
+            # with trio.CancelScope(shield=True):
+            #     await release_lock_signal.wait()
+
+        else:
+            # block here one (at the appropriate frame *up*) where
+            # ``breakpoint()`` was awaited and begin handling stdio.
+            log.debug("Entering the synchronous world of pdb")
+            debug_func(actor, pdb)
+
+    except bdb.BdbQuit:
+        Lock.release()
+        raise
+
+    # XXX: apparently we can't do this without showing this frame
+    # in the backtrace on first entry to the REPL? Seems like an odd
+    # behaviour that should have been fixed by now. This is also why
+    # we scrapped all the @cm approaches that were tried previously.
+    # finally:
+    #     __tracebackhide__ = True
+    #     # frame = sys._getframe()
+    #     # last_f = frame.f_back
+    #     # last_f.f_globals['__tracebackhide__'] = True
+    #     # signal.signal = pdbp.hideframe(signal.signal)
+
+
 def shield_sigint_handler(
     signum: int,
     frame: 'frame',  # type: ignore # noqa
@@ -610,9 +767,8 @@ def shield_sigint_handler(
 def _set_trace(
     actor: tractor.Actor | None = None,
     pdb: MultiActorPdb | None = None,
-    shield: bool = False,
 ):
-    __tracebackhide__: bool = True
+    __tracebackhide__ = True
     actor: tractor.Actor = actor or tractor.current_actor()
 
     # start 2 levels up in user code
@@ -622,20 +778,13 @@ def _set_trace(
 
     if (
         frame
-        and (
+        and pdb
-            pdb
+        and actor is not None
-            and actor is not None
-        ) or shield
     ):
-        # pdbp.set_trace()
         log.pdb(f"\nAttaching pdb to actor: {actor.uid}\n")
         # no f!#$&* idea, but when we're in async land
         # we need 2x frames up?
         frame = frame.f_back
-        # frame = frame.f_back
-
-        # if shield:
-        #     frame = frame.f_back
 
     else:
         pdb, undo_sigint = mk_mpdb()
@@ -648,220 +797,29 @@ def _set_trace(
     # undo_
 
 
-async def pause(
+# TODO: allow pausing from sync code, normally by remapping
-
+# python's builtin breakpoint() hook to this runtime aware version.
-    debug_func: Callable = _set_trace,
-    release_lock_signal: trio.Event | None = None,
-
-    # TODO: allow caller to pause despite task cancellation,
-    # exactly the same as wrapping with:
-    # with CancelScope(shield=True):
-    #     await pause()
-    # => the REMAINING ISSUE is that the scope's .__exit__() frame
-    # is always show in the debugger on entry.. and there seems to
-    # be no way to override it?..
-    # shield: bool = False,
-
-    # TODO:
-    # shield: bool = False
-    task_status: TaskStatus[trio.Event] = trio.TASK_STATUS_IGNORED
-
-) -> None:
-    '''
-    A pause point (more commonly known as a "breakpoint") interrupt
-    instruction for engaging a blocking debugger instance to
-    conduct manual console-based-REPL-interaction from within
-    `tractor`'s async runtime, normally from some single-threaded
-    and currently executing actor-hosted-`trio`-task in some
-    (remote) process.
-
-    NOTE: we use the semantics "pause" since it better encompasses
-    the entirety of the necessary global-runtime-state-mutation any
-    actor-task must access and lock in order to get full isolated
-    control over the process tree's root TTY:
-    https://en.wikipedia.org/wiki/Breakpoint
-
-    '''
-    # __tracebackhide__ = True
-    actor = tractor.current_actor()
-    pdb, undo_sigint = mk_mpdb()
-    task_name = trio.lowlevel.current_task().name
-
-    if (
-        not Lock.local_pdb_complete
-        or Lock.local_pdb_complete.is_set()
-    ):
-        Lock.local_pdb_complete = trio.Event()
-
-    # if shield:
-    debug_func = partial(
-        debug_func,
-        # shield=shield,
-    )
-
-    # def _exit(self, *args, **kwargs):
-    #     __tracebackhide__: bool = True
-    #     super().__exit__(*args, **kwargs)
-
-    # trio.CancelScope.__exit__.__tracebackhide__ = True
-
-    # import types
-    # with trio.CancelScope(shield=shield) as cs:
-        # cs.__exit__ = types.MethodType(_exit, cs)
-        # cs.__exit__.__tracebackhide__ = True
-
-        # TODO: need a more robust check for the "root" actor
-    if (
-        not is_root_process()
-        and actor._parent_chan  # a connected child
-    ):
-
-        if Lock.local_task_in_debug:
-
-            # Recurrence entry case: this task already has the lock and
-            # is likely recurrently entering a breakpoint
-            if Lock.local_task_in_debug == task_name:
-                # noop on recurrent entry case but we want to trigger
-                # a checkpoint to allow other actors error-propagate and
-                # potetially avoid infinite re-entries in some subactor.
-                await trio.lowlevel.checkpoint()
-                return
-
-            # if **this** actor is already in debug mode block here
-            # waiting for the control to be released - this allows
-            # support for recursive entries to `tractor.breakpoint()`
-            log.warning(f"{actor.uid} already has a debug lock, waiting...")
-
-            await Lock.local_pdb_complete.wait()
-            await trio.sleep(0.1)
-
-        # mark local actor as "in debug mode" to avoid recurrent
-        # entries/requests to the root process
-        Lock.local_task_in_debug = task_name
-
-        # this **must** be awaited by the caller and is done using the
-        # root nursery so that the debugger can continue to run without
-        # being restricted by the scope of a new task nursery.
-
-        # TODO: if we want to debug a trio.Cancelled triggered exception
-        # we have to figure out how to avoid having the service nursery
-        # cancel on this task start? I *think* this works below:
-        # ```python
-        #   actor._service_n.cancel_scope.shield = shield
-        # ```
-        # but not entirely sure if that's a sane way to implement it?
-        try:
-            with trio.CancelScope(shield=True):
-                await actor._service_n.start(
-                    wait_for_parent_stdin_hijack,
-                    actor.uid,
-                )
-                Lock.repl = pdb
-        except RuntimeError:
-            Lock.release()
-
-            if actor._cancel_called:
-                # service nursery won't be usable and we
-                # don't want to lock up the root either way since
-                # we're in (the midst of) cancellation.
-                return
-
-            raise
-
-    elif is_root_process():
-
-        # we also wait in the root-parent for any child that
-        # may have the tty locked prior
-        # TODO: wait, what about multiple root tasks acquiring it though?
-        if Lock.global_actor_in_debug == actor.uid:
-            # re-entrant root process already has it: noop.
-            return
-
-        # XXX: since we need to enter pdb synchronously below,
-        # we have to release the lock manually from pdb completion
-        # callbacks. Can't think of a nicer way then this atm.
-        if Lock._debug_lock.locked():
-            log.warning(
-                'Root actor attempting to shield-acquire active tty lock'
-                f' owned by {Lock.global_actor_in_debug}')
-
-            # must shield here to avoid hitting a ``Cancelled`` and
-            # a child getting stuck bc we clobbered the tty
-            with trio.CancelScope(shield=True):
-                await Lock._debug_lock.acquire()
-        else:
-            # may be cancelled
-            await Lock._debug_lock.acquire()
-
-        Lock.global_actor_in_debug = actor.uid
-        Lock.local_task_in_debug = task_name
-        Lock.repl = pdb
-
-    try:
-        if debug_func is None:
-            # assert release_lock_signal, (
-            #     'Must pass `release_lock_signal: trio.Event` if no '
-            #     'trace func provided!'
-            # )
-            print(f"{actor.uid} ENTERING WAIT")
-            task_status.started()
-
-            # with trio.CancelScope(shield=True):
-            #     await release_lock_signal.wait()
-
-        else:
-            # block here one (at the appropriate frame *up*) where
-            # ``breakpoint()`` was awaited and begin handling stdio.
-            log.debug("Entering the synchronous world of pdb")
-            debug_func(actor, pdb)
-
-    except bdb.BdbQuit:
-        Lock.release()
-        raise
-
-    # XXX: apparently we can't do this without showing this frame
-    # in the backtrace on first entry to the REPL? Seems like an odd
-    # behaviour that should have been fixed by now. This is also why
-    # we scrapped all the @cm approaches that were tried previously.
-    # finally:
-    #     __tracebackhide__ = True
-    #     # frame = sys._getframe()
-    #     # last_f = frame.f_back
-    #     # last_f.f_globals['__tracebackhide__'] = True
-    #     # signal.signal = pdbp.hideframe(signal.signal)
-
-
-# TODO: allow pausing from sync code.
-# normally by remapping python's builtin breakpoint() hook to this
-# runtime aware version which takes care of all .
 def pause_from_sync() -> None:
     print("ENTER SYNC PAUSE")
-    actor: tractor.Actor = tractor.current_actor(
+    try:
-        err_on_no_runtime=False,
+        import greenback
-    )
+        __tracebackhide__ = True
-    if actor:
-        try:
-            import greenback
-            # __tracebackhide__ = True
 
+        actor: tractor.Actor = tractor.current_actor()
+        # task_can_release_tty_lock = trio.Event()
 
-            # task_can_release_tty_lock = trio.Event()
+        # spawn bg task which will lock out the TTY, we poll
-
+        # just below until the release event is reporting that task as
-            # spawn bg task which will lock out the TTY, we poll
+        # waiting.. not the most ideal but works for now ;)
-            # just below until the release event is reporting that task as
+        greenback.await_(
-            # waiting.. not the most ideal but works for now ;)
+            actor._service_n.start(partial(
-            greenback.await_(
+                pause,
-                actor._service_n.start(partial(
+                debug_func=None,
-                    pause,
+                # release_lock_signal=task_can_release_tty_lock,
-                    debug_func=None,
+            ))
-                    # release_lock_signal=task_can_release_tty_lock,
+        )
-                ))
+    except ModuleNotFoundError:
-            )
+        log.warning('NO GREENBACK FOUND')
-
-        except ModuleNotFoundError:
-            log.warning('NO GREENBACK FOUND')
-    else:
-        log.warning('Not inside actor-runtime')
 
     db, undo_sigint = mk_mpdb()
     Lock.local_task_in_debug = 'sync'
@@ -896,7 +854,11 @@ def pause_from_sync() -> None:
 # using the "pause" semantics instead since
 # that better covers actually somewhat "pausing the runtime"
 # for this particular paralell task to do debugging B)
-# pp = pause  # short-hand for "pause point"
+pause = partial(
+    _pause,
+    _set_trace,
+)
+pp = pause  # short-hand for "pause point"
 
 
 async def breakpoint(**kwargs):
@@ -929,7 +891,7 @@ def _post_mortem(
 
 
 post_mortem = partial(
-    pause,
+    _pause,
     _post_mortem,
 )
 
@@ -1049,56 +1011,3 @@ async def maybe_wait_for_debugger(
             log.debug(
                     'Root acquired TTY LOCK'
             )
-
-
-# TODO: better naming and what additionals?
-# - [ ] optional runtime plugging?
-# - [ ] detection for sync vs. async code?
-# - [ ] specialized REPL entry when in distributed mode?
-# - [x] allow ignoring kbi Bo
-@cm
-def open_crash_handler(
-    catch: set[BaseException] = {
-        Exception,
-        BaseException,
-    },
-    ignore: set[BaseException] = {
-        KeyboardInterrupt,
-    },
-):
-    '''
-    Generic "post mortem" crash handler using `pdbp` REPL debugger.
-
-    We expose this as a CLI framework addon to both `click` and
-    `typer` users so they can quickly wrap cmd endpoints which get
-    automatically wrapped to use the runtime's `debug_mode: bool`
-    AND `pdbp.pm()` around any code that is PRE-runtime entry
-    - any sync code which runs BEFORE the main call to
-      `trio.run()`.
-
-    '''
-    try:
-        yield
-    except tuple(catch) as err:
-
-        if type(err) not in ignore:
-            pdbp.xpm()
-
-        raise
-
-
-@cm
-def maybe_open_crash_handler(pdb: bool = False):
-    '''
-    Same as `open_crash_handler()` but with bool input flag
-    to allow conditional handling.
-
-    Normally this is used with CLI endpoints such that if the --pdb
-    flag is passed the pdb REPL is engaed on any crashes B)
-    '''
-    rtctx = nullcontext
-    if pdb:
-        rtctx = open_crash_handler
-
-    with rtctx():
-        yield

tractor/_entry.py

@@ -106,25 +106,29 @@ def _trio_main(
     Entry point for a `trio_run_in_process` subactor.
 
     '''
-    log.info(f"Started new trio process for {actor.uid}")
-
-    if actor.loglevel is not None:
-        log.info(
-            f"Setting loglevel for {actor.uid} to {actor.loglevel}")
-        get_console_log(actor.loglevel)
-
-    log.info(
-        f"Started {actor.uid}")
-
     _state._current_actor = actor
-
-    log.debug(f"parent_addr is {parent_addr}")
     trio_main = partial(
         async_main,
         actor,
         parent_addr=parent_addr
     )
 
+    if actor.loglevel is not None:
+        get_console_log(actor.loglevel)
+        import os
+        actor_info: str = (
+            f'|_{actor}\n'
+            f'  uid: {actor.uid}\n'
+            f'  pid: {os.getpid()}\n'
+            f'  parent_addr: {parent_addr}\n'
+            f'  loglevel: {actor.loglevel}\n'
+        )
+        log.info(
+            'Started new trio process:\n'
+            +
+            actor_info
+        )
+
     try:
         if infect_asyncio:
             actor._infected_aio = True
@@ -133,8 +137,14 @@ def _trio_main(
             trio.run(trio_main)
     except KeyboardInterrupt:
         log.cancel(
-            f'Actor@{actor.uid} received KBI'
+            'Actor received KBI\n'
+            +
+            actor_info
         )
 
     finally:
-        log.info(f"Actor {actor.uid} terminated")
+        log.info(
+            'Actor terminated\n'
+            +
+            actor_info
+        )

tractor/_exceptions.py

@@ -27,17 +27,21 @@ from typing import (
     Type,
     TYPE_CHECKING,
 )
+import textwrap
 import traceback
 
-import exceptiongroup as eg
 import trio
 
-from ._state import current_actor
+from tractor._state import current_actor
+from tractor.log import get_logger
 
 if TYPE_CHECKING:
     from ._context import Context
-    from ._stream import MsgStream
     from .log import StackLevelAdapter
+    from ._stream import MsgStream
+    from ._ipc import Channel
+
+log = get_logger('tractor')
 
 _this_mod = importlib.import_module(__name__)
 
@@ -46,6 +50,25 @@ class ActorFailure(Exception):
     "General actor failure"
 
 
+class InternalError(RuntimeError):
+    '''
+    Entirely unexpected internal machinery error indicating
+    a completely invalid state or interface.
+
+    '''
+
+_body_fields: list[str] = [
+    'src_actor_uid',
+    'canceller',
+    'sender',
+]
+
+_msgdata_keys: list[str] = [
+    'type_str',
+] + _body_fields
+
+
+
 # TODO: rename to just `RemoteError`?
 class RemoteActorError(Exception):
     '''
@@ -57,6 +80,10 @@ class RemoteActorError(Exception):
     a special "error" IPC msg sent by some remote actor-runtime.
 
     '''
+    reprol_fields: list[str] = [
+        'src_actor_uid',
+    ]
+
     def __init__(
         self,
         message: str,
@@ -74,23 +101,82 @@ class RemoteActorError(Exception):
         # - .remote_type
         # also pertains to our long long oustanding issue XD
         # https://github.com/goodboy/tractor/issues/5
-        self.type: str = suberror_type
+        self.boxed_type: str = suberror_type
         self.msgdata: dict[str, Any] = msgdata
 
     @property
-    def src_actor_uid(self) -> tuple[str, str] | None:
+    def type(self) -> str:
+        return self.boxed_type
+
+    @property
+    def type_str(self) -> str:
+        return str(type(self.boxed_type).__name__)
+
+    @property
+    def src_actor_uid(self) -> tuple[str, str]|None:
         return self.msgdata.get('src_actor_uid')
 
-    def __repr__(self) -> str:
+    @property
+    def tb_str(
+        self,
+        indent: str = ' '*3,
+    ) -> str:
         if remote_tb := self.msgdata.get('tb_str'):
-            pformat(remote_tb)
+            return textwrap.indent(
-            return (
+                remote_tb,
-                f'{type(self).__name__}(\n'
+                prefix=indent,
-                f'msgdata={pformat(self.msgdata)}\n'
-                ')'
             )
 
-        return super().__repr__()
+        return ''
+
+    def reprol(self) -> str:
+        '''
+        Represent this error for "one line" display, like in
+        a field of our `Context.__repr__()` output.
+
+        '''
+        _repr: str = f'{type(self).__name__}('
+        for key in self.reprol_fields:
+            val: Any|None = self.msgdata.get(key)
+            if val:
+                _repr += f'{key}={repr(val)} '
+
+        return _repr
+
+    def __repr__(self) -> str:
+
+        fields: str = ''
+        for key in _body_fields:
+            val: str|None = self.msgdata.get(key)
+            if val:
+                fields += f'{key}={val}\n'
+
+        fields: str = textwrap.indent(
+            fields,
+            # prefix=' '*2,
+            prefix=' |_',
+        )
+        indent: str = ''*1
+        body: str = (
+            f'{fields}'
+            f'  |\n'
+            f'   ------ - ------\n\n'
+            f'{self.tb_str}\n'
+            f'   ------ - ------\n'
+            f' _|\n'
+        )
+            # f'|\n'
+            # f'         |\n'
+        if indent:
+            body: str = textwrap.indent(
+                body,
+                prefix=indent,
+            )
+        return (
+            f'<{type(self).__name__}(\n'
+            f'{body}'
+            ')>'
+        )
 
     # TODO: local recontruction of remote exception deats
     # def unbox(self) -> BaseException:
@@ -99,8 +185,9 @@ class RemoteActorError(Exception):
 
 class InternalActorError(RemoteActorError):
     '''
-    Remote internal ``tractor`` error indicating
+    (Remote) internal `tractor` error indicating failure of some
-    failure of some primitive or machinery.
+    primitive, machinery state or lowlevel task that should never
+    occur.
 
     '''
 
@@ -111,12 +198,43 @@ class ContextCancelled(RemoteActorError):
     ``Portal.cancel_actor()`` or ``Context.cancel()``.
 
     '''
+    reprol_fields: list[str] = [
+        'canceller',
+    ]
     @property
-    def canceller(self) -> tuple[str, str] | None:
+    def canceller(self) -> tuple[str, str]|None:
+        '''
+        Return the (maybe) `Actor.uid` for the requesting-author
+        of this ctxc.
+
+        Emit a warning msg when `.canceller` has not been set,
+        which usually idicates that a `None` msg-loop setinel was
+        sent before expected in the runtime. This can happen in
+        a few situations:
+
+        - (simulating) an IPC transport network outage
+        - a (malicious) pkt sent specifically to cancel an actor's
+          runtime non-gracefully without ensuring ongoing RPC tasks are 
+          incrementally cancelled as is done with:
+          `Actor`
+          |_`.cancel()`
+          |_`.cancel_soon()`
+          |_`._cancel_task()`
+
+        '''
         value = self.msgdata.get('canceller')
         if value:
             return tuple(value)
 
+        log.warning(
+            'IPC Context cancelled without a requesting actor?\n'
+            'Maybe the IPC transport ended abruptly?\n\n'
+            f'{self}'
+        )
+
+    # to make `.__repr__()` work uniformly
+    # src_actor_uid = canceller
+
 
 class TransportClosed(trio.ClosedResourceError):
     "Underlying channel transport was closed prior to use"
@@ -138,6 +256,9 @@ class StreamOverrun(
     RemoteActorError,
     trio.TooSlowError,
 ):
+    reprol_fields: list[str] = [
+        'sender',
+    ]
     '''
     This stream was overrun by sender
 
@@ -185,6 +306,7 @@ def pack_error(
     ] = {
         'tb_str': tb_str,
         'type_str': type(exc).__name__,
+        'boxed_type': type(exc).__name__,
         'src_actor_uid': current_actor().uid,
     }
 
@@ -199,7 +321,6 @@ def pack_error(
     ):
         error_msg.update(exc.msgdata)
 
-
     pkt: dict = {'error': error_msg}
     if cid:
         pkt['cid'] = cid
@@ -210,8 +331,10 @@ def pack_error(
 def unpack_error(
 
     msg: dict[str, Any],
-    chan=None,
+
-    err_type=RemoteActorError,
+    chan: Channel|None = None,
+    box_type: RemoteActorError = RemoteActorError,
+
     hide_tb: bool = True,
 
 ) -> None|Exception:
@@ -235,18 +358,20 @@ def unpack_error(
     # retrieve the remote error's msg encoded details
     tb_str: str = error_dict.get('tb_str', '')
     message: str = f'{chan.uid}\n' + tb_str
-    type_name: str = error_dict['type_str']
+    type_name: str = (
+        error_dict.get('type_str')
+        or error_dict['boxed_type']
+    )
     suberror_type: Type[BaseException] = Exception
 
     if type_name == 'ContextCancelled':
-        err_type = ContextCancelled
+        box_type = ContextCancelled
-        suberror_type = err_type
+        suberror_type = box_type
 
     else:  # try to lookup a suitable local error type
         for ns in [
             builtins,
             _this_mod,
-            eg,
             trio,
         ]:
             if suberror_type := getattr(
@@ -256,7 +381,7 @@ def unpack_error(
             ):
                 break
 
-    exc = err_type(
+    exc = box_type(
         message,
         suberror_type=suberror_type,
 
@@ -269,12 +394,13 @@ def unpack_error(
 
 def is_multi_cancelled(exc: BaseException) -> bool:
     '''
-    Predicate to determine if a possible ``eg.BaseExceptionGroup`` contains
+    Predicate to determine if a possible ``BaseExceptionGroup`` contains
     only ``trio.Cancelled`` sub-exceptions (and is likely the result of
     cancelling a collection of subtasks.
 
     '''
-    if isinstance(exc, eg.BaseExceptionGroup):
+    # if isinstance(exc, eg.BaseExceptionGroup):
+    if isinstance(exc, BaseExceptionGroup):
         return exc.subgroup(
             lambda exc: isinstance(exc, trio.Cancelled)
         ) is not None
@@ -287,37 +413,63 @@ def _raise_from_no_key_in_msg(
     msg: dict,
     src_err: KeyError,
     log: StackLevelAdapter,  # caller specific `log` obj
+
     expect_key: str = 'yield',
     stream: MsgStream | None = None,
 
+    # allow "deeper" tbs when debugging B^o
+    hide_tb: bool = True,
+
 ) -> bool:
     '''
-    Raise an appopriate local error when a `MsgStream` msg arrives
+    Raise an appopriate local error when a
-    which does not contain the expected (under normal operation)
+    `MsgStream` msg arrives which does not
-    `'yield'` field.
+    contain the expected (at least under normal
+    operation) `'yield'` field.
+
+    `Context` and any embedded `MsgStream` termination,
+    as well as remote task errors are handled in order
+    of priority as:
+
+    - any 'error' msg is re-boxed and raised locally as
+      -> `RemoteActorError`|`ContextCancelled`
+
+    - a `MsgStream` 'stop' msg is constructed, assigned
+      and raised locally as -> `trio.EndOfChannel`
+
+    - All other mis-keyed msgss (like say a "final result"
+      'return' msg, normally delivered from `Context.result()`)
+      are re-boxed inside a `MessagingError` with an explicit
+      exc content describing the missing IPC-msg-key.
 
     '''
-    __tracebackhide__: bool = True
+    __tracebackhide__: bool = hide_tb
 
-    # internal error should never get here
+    # an internal error should never get here
     try:
         cid: str = msg['cid']
     except KeyError as src_err:
         raise MessagingError(
             f'IPC `Context` rx-ed msg without a ctx-id (cid)!?\n'
-            f'cid: {cid}\n'
+            f'cid: {cid}\n\n'
-            'received msg:\n'
+
             f'{pformat(msg)}\n'
         ) from src_err
 
     # TODO: test that shows stream raising an expected error!!!
+
+    # raise the error message in a boxed exception type!
     if msg.get('error'):
-        # raise the error message
         raise unpack_error(
             msg,
             ctx.chan,
+            hide_tb=hide_tb,
+
         ) from None
 
+    # `MsgStream` termination msg.
+    # TODO: does it make more sense to pack 
+    # the stream._eoc outside this in the calleer always?
     elif (
         msg.get('stop')
         or (
@@ -330,29 +482,26 @@ def _raise_from_no_key_in_msg(
             f'cid: {cid}\n'
         )
 
-        # XXX: important to set so that a new ``.receive()``
-        # call (likely by another task using a broadcast receiver)
-        # doesn't accidentally pull the ``return`` message
-        # value out of the underlying feed mem chan!
-        stream._eoc: bool = True
-
         # TODO: if the a local task is already blocking on
         # a `Context.result()` and thus a `.receive()` on the
         # rx-chan, we close the chan and set state ensuring that
         # an eoc is raised!
 
-        # # when the send is closed we assume the stream has
-        # # terminated and signal this local iterator to stop
-        # await stream.aclose()
-
         # XXX: this causes ``ReceiveChannel.__anext__()`` to
         # raise a ``StopAsyncIteration`` **and** in our catch
         # block below it will trigger ``.aclose()``.
-        raise trio.EndOfChannel(
+        eoc = trio.EndOfChannel(
-            f'Context stream ended due to msg:\n'
+            f'Context stream ended due to msg:\n\n'
-            f'{pformat(msg)}'
+            f'{pformat(msg)}\n'
-        ) from src_err
+        )
+        # XXX: important to set so that a new `.receive()`
+        # call (likely by another task using a broadcast receiver)
+        # doesn't accidentally pull the `return` message
+        # value out of the underlying feed mem chan which is
+        # destined for the `Context.result()` call during ctx-exit!
+        stream._eoc: Exception = eoc
 
+        raise eoc from src_err
 
     if (
         stream

tractor/_ipc.py

@@ -19,34 +19,33 @@ Inter-process comms abstractions
 
 """
 from __future__ import annotations
-import platform
-import struct
-import typing
 from collections.abc import (
     AsyncGenerator,
     AsyncIterator,
 )
+from contextlib import asynccontextmanager as acm
+import platform
+from pprint import pformat
+import struct
+import typing
 from typing import (
     Any,
     runtime_checkable,
-    Optional,
     Protocol,
     Type,
     TypeVar,
 )
 
-from tricycle import BufferedReceiveStream
 import msgspec
+from tricycle import BufferedReceiveStream
 import trio
-from async_generator import asynccontextmanager
 
-from .log import get_logger
+from tractor.log import get_logger
-from ._exceptions import TransportClosed
+from tractor._exceptions import TransportClosed
+
 log = get_logger(__name__)
 
-
 _is_windows = platform.system() == 'Windows'
-log = get_logger(__name__)
 
 
 def get_stream_addrs(stream: trio.SocketStream) -> tuple:
@@ -112,6 +111,13 @@ class MsgpackTCPStream(MsgTransport):
     using the ``msgspec`` codec lib.
 
     '''
+    layer_key: int = 4
+    name_key: str = 'tcp'
+
+    # TODO: better naming for this?
+    # -[ ] check how libp2p does naming for such things?
+    codec_key: str = 'msgpack'
+
     def __init__(
         self,
         stream: trio.SocketStream,
@@ -199,7 +205,17 @@ class MsgpackTCPStream(MsgTransport):
                 else:
                     raise
 
-    async def send(self, msg: Any) -> None:
+    async def send(
+        self,
+        msg: Any,
+
+        # hide_tb: bool = False,
+    ) -> None:
+        '''
+        Send a msgpack coded blob-as-msg over TCP.
+
+        '''
+        # __tracebackhide__: bool = hide_tb
         async with self._send_lock:
 
             bytes_data: bytes = self.encode(msg)
@@ -267,7 +283,7 @@ class Channel:
     def __init__(
 
         self,
-        destaddr: Optional[tuple[str, int]],
+        destaddr: tuple[str, int]|None,
 
         msg_transport_type_key: tuple[str, str] = ('msgpack', 'tcp'),
 
@@ -285,14 +301,14 @@ class Channel:
 
         # Either created in ``.connect()`` or passed in by
         # user in ``.from_stream()``.
-        self._stream: Optional[trio.SocketStream] = None
+        self._stream: trio.SocketStream|None = None
-        self.msgstream: Optional[MsgTransport] = None
+        self._transport: MsgTransport|None = None
 
         # set after handshake - always uid of far end
-        self.uid: Optional[tuple[str, str]] = None
+        self.uid: tuple[str, str]|None = None
 
         self._agen = self._aiter_recv()
-        self._exc: Optional[Exception] = None  # set if far end actor errors
+        self._exc: Exception|None = None  # set if far end actor errors
         self._closed: bool = False
 
         # flag set by ``Portal.cancel_actor()`` indicating remote
@@ -300,6 +316,15 @@ class Channel:
         # runtime.
         self._cancel_called: bool = False
 
+    @property
+    def msgstream(self) -> MsgTransport:
+        log.info('`Channel.msgstream` is an old name, use `._transport`')
+        return self._transport
+
+    @property
+    def transport(self) -> MsgTransport:
+        return self._transport
+
     @classmethod
     def from_stream(
         cls,
@@ -309,40 +334,44 @@ class Channel:
     ) -> Channel:
 
         src, dst = get_stream_addrs(stream)
-        chan = Channel(destaddr=dst, **kwargs)
+        chan = Channel(
+            destaddr=dst,
+            **kwargs,
+        )
 
         # set immediately here from provided instance
-        chan._stream = stream
+        chan._stream: trio.SocketStream = stream
         chan.set_msg_transport(stream)
         return chan
 
     def set_msg_transport(
         self,
         stream: trio.SocketStream,
-        type_key: Optional[tuple[str, str]] = None,
+        type_key: tuple[str, str]|None = None,
 
     ) -> MsgTransport:
         type_key = type_key or self._transport_key
-        self.msgstream = get_msg_transport(type_key)(stream)
+        self._transport = get_msg_transport(type_key)(stream)
-        return self.msgstream
+        return self._transport
 
     def __repr__(self) -> str:
-        if self.msgstream:
+        if not self._transport:
-            return repr(
+            return '<Channel with inactive transport?>'
-                self.msgstream.stream.socket._sock
+
-            ).replace(  # type: ignore
+        return repr(
-                "socket.socket",
+            self._transport.stream.socket._sock
-                "Channel",
+        ).replace(  # type: ignore
-            )
+            "socket.socket",
-        return object.__repr__(self)
+            "Channel",
+        )
 
     @property
-    def laddr(self) -> Optional[tuple[str, int]]:
+    def laddr(self) -> tuple[str, int]|None:
-        return self.msgstream.laddr if self.msgstream else None
+        return self._transport.laddr if self._transport else None
 
     @property
-    def raddr(self) -> Optional[tuple[str, int]]:
+    def raddr(self) -> tuple[str, int]|None:
-        return self.msgstream.raddr if self.msgstream else None
+        return self._transport.raddr if self._transport else None
 
     async def connect(
         self,
@@ -361,26 +390,42 @@ class Channel:
             *destaddr,
             **kwargs
         )
-        msgstream = self.set_msg_transport(stream)
+        transport = self.set_msg_transport(stream)
 
         log.transport(
-            f'Opened channel[{type(msgstream)}]: {self.laddr} -> {self.raddr}'
+            f'Opened channel[{type(transport)}]: {self.laddr} -> {self.raddr}'
         )
-        return msgstream
+        return transport
 
-    async def send(self, item: Any) -> None:
+    async def send(
+        self,
+        payload: Any,
 
-        log.transport(f"send `{item}`")  # type: ignore
+        # hide_tb: bool = False,
-        assert self.msgstream
 
-        await self.msgstream.send(item)
+    ) -> None:
+        '''
+        Send a coded msg-blob over the transport.
+
+        '''
+        # __tracebackhide__: bool = hide_tb
+        log.transport(
+            '=> send IPC msg:\n\n'
+            f'{pformat(payload)}\n'
+        )  # type: ignore
+        assert self._transport
+
+        await self._transport.send(
+            payload,
+            # hide_tb=hide_tb,
+        )
 
     async def recv(self) -> Any:
-        assert self.msgstream
+        assert self._transport
-        return await self.msgstream.recv()
+        return await self._transport.recv()
 
         # try:
-        #     return await self.msgstream.recv()
+        #     return await self._transport.recv()
         # except trio.BrokenResourceError:
         #     if self._autorecon:
         #         await self._reconnect()
@@ -393,8 +438,8 @@ class Channel:
             f'Closing channel to {self.uid} '
             f'{self.laddr} -> {self.raddr}'
         )
-        assert self.msgstream
+        assert self._transport
-        await self.msgstream.stream.aclose()
+        await self._transport.stream.aclose()
         self._closed = True
 
     async def __aenter__(self):
@@ -445,16 +490,16 @@ class Channel:
         Async iterate items from underlying stream.
 
         '''
-        assert self.msgstream
+        assert self._transport
         while True:
             try:
-                async for item in self.msgstream:
+                async for item in self._transport:
                     yield item
                     # sent = yield item
                     # if sent is not None:
                     #     # optimization, passing None through all the
                     #     # time is pointless
-                    #     await self.msgstream.send(sent)
+                    #     await self._transport.send(sent)
             except trio.BrokenResourceError:
 
                 # if not self._autorecon:
@@ -467,10 +512,10 @@ class Channel:
             #     continue
 
     def connected(self) -> bool:
-        return self.msgstream.connected() if self.msgstream else False
+        return self._transport.connected() if self._transport else False
 
 
-@asynccontextmanager
+@acm
 async def _connect_chan(
     host: str,
     port: int

tractor/_portal.py

@@ -24,55 +24,73 @@ OS processes, possibly on different (hardware) hosts.
 
 '''
 from __future__ import annotations
+from contextlib import asynccontextmanager as acm
 import importlib
 import inspect
 from typing import (
-    Any, Optional,
+    Any,
-    Callable, AsyncGenerator,
+    Callable,
-    Type,
+    AsyncGenerator,
+    # Type,
 )
 from functools import partial
 from dataclasses import dataclass
 import warnings
 
 import trio
-from async_generator import asynccontextmanager
 
 from .trionics import maybe_open_nursery
-from ._state import current_actor
+from ._state import (
+    current_actor,
+)
 from ._ipc import Channel
 from .log import get_logger
 from .msg import NamespacePath
 from ._exceptions import (
-    _raise_from_no_key_in_msg,
     unpack_error,
     NoResult,
-    ContextCancelled,
 )
 from ._context import (
     Context,
+    open_context_from_portal,
 )
 from ._streaming import (
     MsgStream,
 )
-from .devx._debug import maybe_wait_for_debugger
 
 
 log = get_logger(__name__)
 
 
+# TODO: rename to `unwrap_result()` and use
+# `._raise_from_no_key_in_msg()` (after tweak to
+# accept a `chan: Channel` arg) in key block!
 def _unwrap_msg(
     msg: dict[str, Any],
-    channel: Channel
+    channel: Channel,
+
+    hide_tb: bool = True,
 
 ) -> Any:
-    __tracebackhide__ = True
+    '''
+    Unwrap a final result from a `{return: <Any>}` IPC msg.
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
     try:
         return msg['return']
     except KeyError as ke:
+
         # internal error should never get here
-        assert msg.get('cid'), "Received internal error at portal?"
+        assert msg.get('cid'), (
-        raise unpack_error(msg, channel) from ke
+            "Received internal error at portal?"
+        )
+
+        raise unpack_error(
+            msg,
+            channel
+        ) from ke
 
 
 class Portal:
@@ -99,9 +117,9 @@ class Portal:
     cancel_timeout: float = 0.5
 
     def __init__(self, channel: Channel) -> None:
-        self.channel = channel
+        self.chan = channel
         # during the portal's lifetime
-        self._result_msg: Optional[dict] = None
+        self._result_msg: dict|None = None
 
         # When set to a ``Context`` (when _submit_for_result is called)
         # it is expected that ``result()`` will be awaited at some
@@ -110,6 +128,18 @@ class Portal:
         self._streams: set[MsgStream] = set()
         self.actor = current_actor()
 
+    @property
+    def channel(self) -> Channel:
+        '''
+        Proxy to legacy attr name..
+
+        Consider the shorter `Portal.chan` instead of `.channel` ;)
+        '''
+        log.debug(
+            'Consider the shorter `Portal.chan` instead of `.channel` ;)'
+        )
+        return self.chan
+
     async def _submit_for_result(
         self,
         ns: str,
@@ -117,14 +147,14 @@ class Portal:
         **kwargs
     ) -> None:
 
-        assert self._expect_result is None, \
+        assert self._expect_result is None, (
-                "A pending main result has already been submitted"
+            "A pending main result has already been submitted"
+        )
 
         self._expect_result = await self.actor.start_remote_task(
             self.channel,
-            ns,
+            nsf=NamespacePath(f'{ns}:{func}'),
-            func,
+            kwargs=kwargs
-            kwargs
         )
 
     async def _return_once(
@@ -134,7 +164,7 @@ class Portal:
     ) -> dict[str, Any]:
 
         assert ctx._remote_func_type == 'asyncfunc'  # single response
-        msg = await ctx._recv_chan.receive()
+        msg: dict = await ctx._recv_chan.receive()
         return msg
 
     async def result(self) -> Any:
@@ -165,7 +195,10 @@ class Portal:
                 self._expect_result
             )
 
-        return _unwrap_msg(self._result_msg, self.channel)
+        return _unwrap_msg(
+            self._result_msg,
+            self.channel,
+        )
 
     async def _cancel_streams(self):
         # terminate all locally running async generator
@@ -207,26 +240,33 @@ class Portal:
         purpose.
 
         '''
-        if not self.channel.connected():
+        chan: Channel = self.channel
-            log.cancel("This channel is already closed can't cancel")
+        if not chan.connected():
+            log.runtime(
+                'This channel is already closed, skipping cancel request..'
+            )
             return False
 
+        reminfo: str = (
+            f'`Portal.cancel_actor()` => {self.channel.uid}\n'
+            f' |_{chan}\n'
+        )
         log.cancel(
-            f"Sending actor cancel request to {self.channel.uid} on "
+            f'Sending runtime `.cancel()` request to peer\n\n'
-            f"{self.channel}")
+            f'{reminfo}'
-
+        )
-        self.channel._cancel_called = True
 
+        self.channel._cancel_called: bool = True
         try:
             # send cancel cmd - might not get response
             # XXX: sure would be nice to make this work with
             # a proper shield
             with trio.move_on_after(
                 timeout
-                or self.cancel_timeout
+                or
+                self.cancel_timeout
             ) as cs:
-                cs.shield = True
+                cs.shield: bool = True
-
                 await self.run_from_ns(
                     'self',
                     'cancel',
@@ -234,7 +274,12 @@ class Portal:
                 return True
 
             if cs.cancelled_caught:
-                log.cancel(f"May have failed to cancel {self.channel.uid}")
+                # may timeout and we never get an ack (obvi racy)
+                # but that doesn't mean it wasn't cancelled.
+                log.debug(
+                    'May have failed to cancel peer?\n'
+                    f'{reminfo}'
+                )
 
             # if we get here some weird cancellation case happened
             return False
@@ -243,9 +288,11 @@ class Portal:
             trio.ClosedResourceError,
             trio.BrokenResourceError,
         ):
-            log.cancel(
+            log.debug(
-                f"{self.channel} for {self.channel.uid} was already "
+                'IPC chan for actor already closed or broken?\n\n'
-                "closed or broken?")
+                f'{self.channel.uid}\n'
+                f' |_{self.channel}\n'
+            )
             return False
 
     async def run_from_ns(
@@ -264,27 +311,33 @@ class Portal:
 
         Note::
 
-            A special namespace `self` can be used to invoke `Actor`
+          A special namespace `self` can be used to invoke `Actor`
-            instance methods in the remote runtime. Currently this
+          instance methods in the remote runtime. Currently this
-            should only be used solely for ``tractor`` runtime
+          should only ever be used for `Actor` (method) runtime
-            internals.
+          internals!
 
         '''
+        nsf = NamespacePath(
+            f'{namespace_path}:{function_name}'
+        )
         ctx = await self.actor.start_remote_task(
-            self.channel,
+            chan=self.channel,
-            namespace_path,
+            nsf=nsf,
-            function_name,
+            kwargs=kwargs,
-            kwargs,
         )
         ctx._portal = self
         msg = await self._return_once(ctx)
-        return _unwrap_msg(msg, self.channel)
+        return _unwrap_msg(
+            msg,
+            self.channel,
+        )
 
     async def run(
         self,
         func: str,
-        fn_name: Optional[str] = None,
+        fn_name: str|None = None,
         **kwargs
+
     ) -> Any:
         '''
         Submit a remote function to be scheduled and run by actor, in
@@ -303,8 +356,9 @@ class Portal:
                 DeprecationWarning,
                 stacklevel=2,
             )
-            fn_mod_path = func
+            fn_mod_path: str = func
             assert isinstance(fn_name, str)
+            nsf = NamespacePath(f'{fn_mod_path}:{fn_name}')
 
         else:  # function reference was passed directly
             if (
@@ -317,13 +371,12 @@ class Portal:
                 raise TypeError(
                     f'{func} must be a non-streaming async function!')
 
-            fn_mod_path, fn_name = NamespacePath.from_ref(func).to_tuple()
+            nsf = NamespacePath.from_ref(func)
 
         ctx = await self.actor.start_remote_task(
             self.channel,
-            fn_mod_path,
+            nsf=nsf,
-            fn_name,
+            kwargs=kwargs,
-            kwargs,
         )
         ctx._portal = self
         return _unwrap_msg(
@@ -331,7 +384,7 @@ class Portal:
             self.channel,
         )
 
-    @asynccontextmanager
+    @acm
     async def open_stream_from(
         self,
         async_gen_func: Callable,  # typing: ignore
@@ -347,15 +400,10 @@ class Portal:
                 raise TypeError(
                     f'{async_gen_func} must be an async generator function!')
 
-        fn_mod_path, fn_name = NamespacePath.from_ref(
+        ctx: Context = await self.actor.start_remote_task(
-            async_gen_func
-        ).to_tuple()
-
-        ctx = await self.actor.start_remote_task(
             self.channel,
-            fn_mod_path,
+            nsf=NamespacePath.from_ref(async_gen_func),
-            fn_name,
+            kwargs=kwargs,
-            kwargs
         )
         ctx._portal = self
 
@@ -365,7 +413,8 @@ class Portal:
         try:
             # deliver receive only stream
             async with MsgStream(
-                ctx, ctx._recv_chan,
+                ctx=ctx,
+                rx_chan=ctx._recv_chan,
             ) as rchan:
                 self._streams.add(rchan)
                 yield rchan
@@ -392,367 +441,12 @@ class Portal:
             # await recv_chan.aclose()
             self._streams.remove(rchan)
 
-    @asynccontextmanager
+    # NOTE: impl is found in `._context`` mod to make
-    async def open_context(
+    # reading/groking the details simpler code-org-wise. This
-
+    # method does not have to be used over that `@acm` module func
-        self,
+    # directly, it is for conventience and from the original API
-        func: Callable,
+    # design.
-        allow_overruns: bool = False,
+    open_context = open_context_from_portal
-        **kwargs,
-
-    ) -> AsyncGenerator[tuple[Context, Any], None]:
-        '''
-        Open an inter-actor "task context"; a remote task is
-        scheduled and cancel-scope-state-linked to a `trio.run()` across
-        memory boundaries in another actor's runtime.
-
-        This is an `@acm` API which allows for deterministic setup
-        and teardown of a remotely scheduled task in another remote
-        actor. Once opened, the 2 now "linked" tasks run completely
-        in parallel in each actor's runtime with their enclosing
-        `trio.CancelScope`s kept in a synced state wherein if
-        either side errors or cancels an equivalent error is
-        relayed to the other side via an SC-compat IPC protocol.
-
-        The yielded `tuple` is a pair delivering a `tractor.Context`
-        and any first value "sent" by the "callee" task via a call
-        to `Context.started(<value: Any>)`; this side of the
-        context does not unblock until the "callee" task calls
-        `.started()` in similar style to `trio.Nursery.start()`.
-        When the "callee" (side that is "called"/started by a call
-        to *this* method) returns, the caller side (this) unblocks
-        and any final value delivered from the other end can be
-        retrieved using the `Contex.result()` api.
-
-        The yielded ``Context`` instance further allows for opening
-        bidirectional streams, explicit cancellation and
-        structurred-concurrency-synchronized final result-msg
-        collection. See ``tractor.Context`` for more details.
-
-        '''
-        # conduct target func method structural checks
-        if not inspect.iscoroutinefunction(func) and (
-            getattr(func, '_tractor_contex_function', False)
-        ):
-            raise TypeError(
-                f'{func} must be an async generator function!')
-
-        # TODO: i think from here onward should probably
-        # just be factored into an `@acm` inside a new
-        # a new `_context.py` mod.
-        fn_mod_path, fn_name = NamespacePath.from_ref(func).to_tuple()
-
-        ctx = await self.actor.start_remote_task(
-            self.channel,
-            fn_mod_path,
-            fn_name,
-            kwargs,
-
-            # NOTE: it's imporant to expose this since you might
-            # get the case where the parent who opened the context does
-            # not open a stream until after some slow startup/init
-            # period, in which case when the first msg is read from
-            # the feeder mem chan, say when first calling
-            # `Context.open_stream(allow_overruns=True)`, the overrun condition will be
-            # raised before any ignoring of overflow msgs can take
-            # place..
-            allow_overruns=allow_overruns,
-        )
-
-        assert ctx._remote_func_type == 'context'
-        msg: dict = await ctx._recv_chan.receive()
-
-        try:
-            # the "first" value here is delivered by the callee's
-            # ``Context.started()`` call.
-            first: Any = msg['started']
-            ctx._started_called: bool = True
-
-        except KeyError as src_error:
-
-            _raise_from_no_key_in_msg(
-                ctx=ctx,
-                msg=msg,
-                src_err=src_error,
-                log=log,
-                expect_key='started',
-            )
-
-        ctx._portal: Portal = self
-        uid: tuple = self.channel.uid
-        cid: str = ctx.cid
-
-        # placeholder for any exception raised in the runtime
-        # or by user tasks which cause this context's closure.
-        scope_err: BaseException | None = None
-        try:
-            async with trio.open_nursery() as nurse:
-                ctx._scope_nursery: trio.Nursery = nurse
-                ctx._scope: trio.CancelScope = nurse.cancel_scope
-
-                # deliver context instance and .started() msg value
-                # in enter tuple.
-                yield ctx, first
-
-                # when in allow_overruns mode there may be
-                # lingering overflow sender tasks remaining?
-                if nurse.child_tasks:
-                    # XXX: ensure we are in overrun state
-                    # with ``._allow_overruns=True`` bc otherwise
-                    # there should be no tasks in this nursery!
-                    if (
-                        not ctx._allow_overruns
-                        or len(nurse.child_tasks) > 1
-                    ):
-                        raise RuntimeError(
-                            'Context has sub-tasks but is '
-                            'not in `allow_overruns=True` mode!?'
-                        )
-
-                    # ensure cancel of all overflow sender tasks
-                    # started in the ctx nursery.
-                    ctx._scope.cancel()
-
-        # XXX NOTE XXX: maybe shield against
-        # self-context-cancellation (which raises a local
-        # `ContextCancelled`) when requested (via
-        # `Context.cancel()`) by the same task (tree) which entered
-        # THIS `.open_context()`.
-        #
-        # NOTE: There are 2 operating cases for a "graceful cancel"
-        # of a `Context`. In both cases any `ContextCancelled`
-        # raised in this scope-block came from a transport msg
-        # relayed from some remote-actor-task which our runtime set
-        # as a `Context._remote_error`
-        #
-        # the CASES:
-        #
-        # - if that context IS THE SAME ONE that called
-        #   `Context.cancel()`, we want to absorb the error
-        #   silently and let this `.open_context()` block to exit
-        #   without raising.
-        #
-        # - if it is from some OTHER context (we did NOT call
-        #   `.cancel()`), we want to re-RAISE IT whilst also
-        #   setting our own ctx's "reason for cancel" to be that
-        #   other context's cancellation condition; we set our
-        #   `.canceller: tuple[str, str]` to be same value as
-        #   caught here in a `ContextCancelled.canceller`.
-        #
-        # Again, there are 2 cases:
-        #
-        # 1-some other context opened in this `.open_context()`
-        #   block cancelled due to a self or peer cancellation
-        #   request in which case we DO let the error bubble to the
-        #   opener.
-        #
-        # 2-THIS "caller" task somewhere invoked `Context.cancel()`
-        #   and received a `ContextCanclled` from the "callee"
-        #   task, in which case we mask the `ContextCancelled` from
-        #   bubbling to this "caller" (much like how `trio.Nursery`
-        #   swallows any `trio.Cancelled` bubbled by a call to
-        #   `Nursery.cancel_scope.cancel()`)
-        except ContextCancelled as ctxc:
-            scope_err = ctxc
-
-            # CASE 2: context was cancelled by local task calling
-            # `.cancel()`, we don't raise and the exit block should
-            # exit silently.
-            if (
-                ctx._cancel_called
-                and (
-                    ctxc is ctx._remote_error
-                    or
-                    ctxc.canceller is self.canceller
-                )
-            ):
-                log.debug(
-                    f'Context {ctx} cancelled gracefully with:\n'
-                    f'{ctxc}'
-                )
-            # CASE 1: this context was never cancelled via a local
-            # task (tree) having called `Context.cancel()`, raise
-            # the error since it was caused by someone else!
-            else:
-                raise
-
-        # the above `._scope` can be cancelled due to:
-        # 1. an explicit self cancel via `Context.cancel()` or
-        #    `Actor.cancel()`,
-        # 2. any "callee"-side remote error, possibly also a cancellation
-        #    request by some peer,
-        # 3. any "caller" (aka THIS scope's) local error raised in the above `yield`
-        except (
-            # CASE 3: standard local error in this caller/yieldee
-            Exception,
-
-            # CASES 1 & 2: normally manifested as
-            # a `Context._scope_nursery` raised
-            # exception-group of,
-            # 1.-`trio.Cancelled`s, since
-            #   `._scope.cancel()` will have been called and any
-            #   `ContextCancelled` absorbed and thus NOT RAISED in
-            #   any `Context._maybe_raise_remote_err()`,
-            # 2.-`BaseExceptionGroup[ContextCancelled | RemoteActorError]`
-            #    from any error raised in the "callee" side with
-            #    a group only raised if there was any more then one
-            #    task started here in the "caller" in the
-            #    `yield`-ed to task.
-            BaseExceptionGroup,  # since overrun handler tasks may have been spawned
-            trio.Cancelled,  # NOTE: NOT from inside the ctx._scope
-            KeyboardInterrupt,
-
-        ) as err:
-            scope_err = err
-
-            # XXX: ALWAYS request the context to CANCEL ON any ERROR.
-            # NOTE: `Context.cancel()` is conversely NEVER CALLED in
-            # the `ContextCancelled` "self cancellation absorbed" case
-            # handled in the block above!
-            log.cancel(
-                'Context cancelled for task due to\n'
-                f'{err}\n'
-                'Sending cancel request..\n'
-                f'task:{cid}\n'
-                f'actor:{uid}'
-            )
-            try:
-                await ctx.cancel()
-            except trio.BrokenResourceError:
-                log.warning(
-                    'IPC connection for context is broken?\n'
-                    f'task:{cid}\n'
-                    f'actor:{uid}'
-                )
-
-            raise  # duh
-
-        # no local scope error, the "clean exit with a result" case.
-        else:
-            if ctx.chan.connected():
-                log.info(
-                    'Waiting on final context-task result for\n'
-                    f'task: {cid}\n'
-                    f'actor: {uid}'
-                )
-                # XXX NOTE XXX: the below call to
-                # `Context.result()` will ALWAYS raise
-                # a `ContextCancelled` (via an embedded call to
-                # `Context._maybe_raise_remote_err()`) IFF
-                # a `Context._remote_error` was set by the runtime
-                # via a call to
-                # `Context._maybe_cancel_and_set_remote_error()`.
-                # As per `Context._deliver_msg()`, that error IS
-                # ALWAYS SET any time "callee" side fails and causes "caller
-                # side" cancellation via a `ContextCancelled` here.
-                # result = await ctx.result()
-                try:
-                    result = await ctx.result()
-                    log.runtime(
-                        f'Context {fn_name} returned value from callee:\n'
-                        f'`{result}`'
-                    )
-                except BaseException as berr:
-                    # on normal teardown, if we get some error
-                    # raised in `Context.result()` we still want to
-                    # save that error on the ctx's state to
-                    # determine things like `.cancelled_caught` for
-                    # cases where there was remote cancellation but
-                    # this task didn't know until final teardown
-                    # / value collection.
-                    scope_err = berr
-                    raise
-
-        finally:
-            # though it should be impossible for any tasks
-            # operating *in* this scope to have survived
-            # we tear down the runtime feeder chan last
-            # to avoid premature stream clobbers.
-            rxchan: trio.ReceiveChannel = ctx._recv_chan
-            if (
-                 rxchan
-
-                # maybe TODO: yes i know the below check is
-                # touching `trio` memchan internals..BUT, there are
-                # only a couple ways to avoid a `trio.Cancelled`
-                # bubbling from the `.aclose()` call below:
-                #
-                # - catch and mask it via the cancel-scope-shielded call
-                #   as we are rn (manual and frowned upon) OR,
-                # - specially handle the case where `scope_err` is
-                #   one of {`BaseExceptionGroup`, `trio.Cancelled`}
-                #   and then presume that the `.aclose()` call will
-                #   raise a `trio.Cancelled` and just don't call it
-                #   in those cases..
-                #
-                # that latter approach is more logic, LOC, and more
-                # convoluted so for now stick with the first
-                # psuedo-hack-workaround where we just try to avoid
-                # the shielded call as much as we can detect from
-                # the memchan's `._closed` state..
-                #
-                # XXX MOTIVATION XXX-> we generally want to raise
-                # any underlying actor-runtime/internals error that
-                # surfaces from a bug in tractor itself so it can
-                # be easily detected/fixed AND, we also want to
-                # minimize noisy runtime tracebacks (normally due
-                # to the cross-actor linked task scope machinery
-                # teardown) displayed to user-code and instead only
-                # displaying `ContextCancelled` traces where the
-                # cause of crash/exit IS due to something in
-                # user/app code on either end of the context.
-                and not rxchan._closed
-            ):
-                # XXX NOTE XXX: and again as per above, we mask any
-                # `trio.Cancelled` raised here so as to NOT mask
-                # out any exception group or legit (remote) ctx
-                # error that sourced from the remote task or its
-                # runtime.
-                with trio.CancelScope(shield=True):
-                    await ctx._recv_chan.aclose()
-
-            # XXX: we always raise remote errors locally and
-            # generally speaking mask runtime-machinery related
-            # multi-`trio.Cancelled`s. As such, any `scope_error`
-            # which was the underlying cause of this context's exit
-            # should be stored as the `Context._local_error` and
-            # used in determining `Context.cancelled_caught: bool`.
-            if scope_err is not None:
-                ctx._local_error: BaseException = scope_err
-                etype: Type[BaseException] = type(scope_err)
-
-                # CASE 2
-                if ctx._cancel_called:
-                    log.cancel(
-                        f'Context {fn_name} cancelled by caller with\n'
-                        f'{etype}'
-                    )
-
-                # CASE 1
-                else:
-                    log.cancel(
-                        f'Context cancelled by callee with {etype}\n'
-                        f'target: `{fn_name}`\n'
-                        f'task:{cid}\n'
-                        f'actor:{uid}'
-                    )
-
-            # XXX: (MEGA IMPORTANT) if this is a root opened process we
-            # wait for any immediate child in debug before popping the
-            # context from the runtime msg loop otherwise inside
-            # ``Actor._push_result()`` the msg will be discarded and in
-            # the case where that msg is global debugger unlock (via
-            # a "stop" msg for a stream), this can result in a deadlock
-            # where the root is waiting on the lock to clear but the
-            # child has already cleared it and clobbered IPC.
-            await maybe_wait_for_debugger()
-
-            # FINALLY, remove the context from runtime tracking and
-            # exit!
-            self.actor._contexts.pop(
-                (self.channel.uid, ctx.cid),
-                None,
-            )
 
 
 @dataclass
@@ -783,11 +477,11 @@ class LocalPortal:
         return await func(**kwargs)
 
 
-@asynccontextmanager
+@acm
 async def open_portal(
 
     channel: Channel,
-    nursery: Optional[trio.Nursery] = None,
+    nursery: trio.Nursery|None = None,
     start_msg_loop: bool = True,
     shield: bool = False,
 
@@ -812,7 +506,7 @@ async def open_portal(
         if channel.uid is None:
             await actor._do_handshake(channel)
 
-        msg_loop_cs: Optional[trio.CancelScope] = None
+        msg_loop_cs: trio.CancelScope|None = None
         if start_msg_loop:
             from ._runtime import process_messages
             msg_loop_cs = await nursery.start(

tractor/_root.py

@@ -28,15 +28,16 @@ import os
 import warnings
 
 
-from exceptiongroup import BaseExceptionGroup
 import trio
 
 from ._runtime import (
     Actor,
     Arbiter,
+    # TODO: rename and make a non-actor subtype?
+    # Arbiter as Registry,
     async_main,
 )
-from .devx import _debug
+from . import _debug
 from . import _spawn
 from . import _state
 from . import log
@@ -98,7 +99,7 @@ async def open_root_actor(
     # https://github.com/python-trio/trio/issues/1155#issuecomment-742964018
     builtin_bp_handler = sys.breakpointhook
     orig_bp_path: str | None = os.environ.get('PYTHONBREAKPOINT', None)
-    os.environ['PYTHONBREAKPOINT'] = 'tractor.devx._debug.pause_from_sync'
+    os.environ['PYTHONBREAKPOINT'] = 'tractor._debug.pause_from_sync'
 
     # attempt to retreive ``trio``'s sigint handler and stash it
     # on our debugger lock state.
@@ -145,7 +146,7 @@ async def open_root_actor(
 
         # expose internal debug module to every actor allowing
         # for use of ``await tractor.breakpoint()``
-        enable_modules.append('tractor.devx._debug')
+        enable_modules.append('tractor._debug')
 
         # if debug mode get's enabled *at least* use that level of
         # logging for some informative console prompts.
@@ -302,12 +303,12 @@ async def open_root_actor(
             ) as err:
 
                 entered: bool = await _debug._maybe_enter_pm(err)
-
                 if (
                     not entered
-                    and not is_multi_cancelled(err)
+                    and
+                    not is_multi_cancelled(err)
                 ):
-                    logger.exception("Root actor crashed:")
+                    logger.exception('Root actor crashed:\n')
 
                 # ALWAYS re-raise any error bubbled up from the
                 # runtime!
@@ -323,12 +324,13 @@ async def open_root_actor(
                 #     for an in nurseries:
                 #         tempn.start_soon(an.exited.wait)
 
-                logger.cancel("Shutting down root actor")
+                logger.info(
-                await actor.cancel(
+                    'Closing down root actor'
-                    requesting_uid=actor.uid,
                 )
+                await actor.cancel(None)  # self cancel
     finally:
         _state._current_actor = None
+        _state._last_actor_terminated = actor
 
         # restore built-in `breakpoint()` hook state
         sys.breakpointhook = builtin_bp_handler

tractor/_shm.py

@@ -1,833 +0,0 @@
-# tractor: structured concurrent "actors".
-# Copyright 2018-eternity Tyler Goodlet.
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-"""
-SC friendly shared memory management geared at real-time
-processing.
-
-Support for ``numpy`` compatible array-buffers is provided but is
-considered optional within the context of this runtime-library.
-
-"""
-from __future__ import annotations
-from sys import byteorder
-import time
-from typing import Optional
-from multiprocessing import shared_memory as shm
-from multiprocessing.shared_memory import (
-    SharedMemory,
-    ShareableList,
-)
-
-from msgspec import Struct
-import tractor
-
-from .log import get_logger
-
-
-_USE_POSIX = getattr(shm, '_USE_POSIX', False)
-if _USE_POSIX:
-    from _posixshmem import shm_unlink
-
-
-try:
-    import numpy as np
-    from numpy.lib import recfunctions as rfn
-    import nptyping
-except ImportError:
-    pass
-
-
-log = get_logger(__name__)
-
-
-def disable_mantracker():
-    '''
-    Disable all ``multiprocessing``` "resource tracking" machinery since
-    it's an absolute multi-threaded mess of non-SC madness.
-
-    '''
-    from multiprocessing import resource_tracker as mantracker
-
-    # Tell the "resource tracker" thing to fuck off.
-    class ManTracker(mantracker.ResourceTracker):
-        def register(self, name, rtype):
-            pass
-
-        def unregister(self, name, rtype):
-            pass
-
-        def ensure_running(self):
-            pass
-
-    # "know your land and know your prey"
-    # https://www.dailymotion.com/video/x6ozzco
-    mantracker._resource_tracker = ManTracker()
-    mantracker.register = mantracker._resource_tracker.register
-    mantracker.ensure_running = mantracker._resource_tracker.ensure_running
-    mantracker.unregister = mantracker._resource_tracker.unregister
-    mantracker.getfd = mantracker._resource_tracker.getfd
-
-
-disable_mantracker()
-
-
-class SharedInt:
-    '''
-    Wrapper around a single entry shared memory array which
-    holds an ``int`` value used as an index counter.
-
-    '''
-    def __init__(
-        self,
-        shm: SharedMemory,
-    ) -> None:
-        self._shm = shm
-
-    @property
-    def value(self) -> int:
-        return int.from_bytes(self._shm.buf, byteorder)
-
-    @value.setter
-    def value(self, value) -> None:
-        self._shm.buf[:] = value.to_bytes(self._shm.size, byteorder)
-
-    def destroy(self) -> None:
-        if _USE_POSIX:
-            # We manually unlink to bypass all the "resource tracker"
-            # nonsense meant for non-SC systems.
-            name = self._shm.name
-            try:
-                shm_unlink(name)
-            except FileNotFoundError:
-                # might be a teardown race here?
-                log.warning(f'Shm for {name} already unlinked?')
-
-
-class NDToken(Struct, frozen=True):
-    '''
-    Internal represenation of a shared memory ``numpy`` array "token"
-    which can be used to key and load a system (OS) wide shm entry
-    and correctly read the array by type signature.
-
-    This type is msg safe.
-
-    '''
-    shm_name: str  # this servers as a "key" value
-    shm_first_index_name: str
-    shm_last_index_name: str
-    dtype_descr: tuple
-    size: int  # in struct-array index / row terms
-
-    # TODO: use nptyping here on dtypes
-    @property
-    def dtype(self) -> list[tuple[str, str, tuple[int, ...]]]:
-        return np.dtype(
-            list(
-                map(tuple, self.dtype_descr)
-            )
-        ).descr
-
-    def as_msg(self):
-        return self.to_dict()
-
-    @classmethod
-    def from_msg(cls, msg: dict) -> NDToken:
-        if isinstance(msg, NDToken):
-            return msg
-
-        # TODO: native struct decoding
-        # return _token_dec.decode(msg)
-
-        msg['dtype_descr'] = tuple(map(tuple, msg['dtype_descr']))
-        return NDToken(**msg)
-
-
-# _token_dec = msgspec.msgpack.Decoder(NDToken)
-
-# TODO: this api?
-# _known_tokens = tractor.ActorVar('_shm_tokens', {})
-# _known_tokens = tractor.ContextStack('_known_tokens', )
-# _known_tokens = trio.RunVar('shms', {})
-
-# TODO: this should maybe be provided via
-# a `.trionics.maybe_open_context()` wrapper factory?
-# process-local store of keys to tokens
-_known_tokens: dict[str, NDToken] = {}
-
-
-def get_shm_token(key: str) -> NDToken | None:
-    '''
-    Convenience func to check if a token
-    for the provided key is known by this process.
-
-    Returns either the ``numpy`` token or a string for a shared list.
-
-    '''
-    return _known_tokens.get(key)
-
-
-def _make_token(
-    key: str,
-    size: int,
-    dtype: np.dtype,
-
-) -> NDToken:
-    '''
-    Create a serializable token that can be used
-    to access a shared array.
-
-    '''
-    return NDToken(
-        shm_name=key,
-        shm_first_index_name=key + "_first",
-        shm_last_index_name=key + "_last",
-        dtype_descr=tuple(np.dtype(dtype).descr),
-        size=size,
-    )
-
-
-class ShmArray:
-    '''
-    A shared memory ``numpy.ndarray`` API.
-
-    An underlying shared memory buffer is allocated based on
-    a user specified ``numpy.ndarray``. This fixed size array
-    can be read and written to by pushing data both onto the "front"
-    or "back" of a set index range. The indexes for the "first" and
-    "last" index are themselves stored in shared memory (accessed via
-    ``SharedInt`` interfaces) values such that multiple processes can
-    interact with the same array using a synchronized-index.
-
-    '''
-    def __init__(
-        self,
-        shmarr: np.ndarray,
-        first: SharedInt,
-        last: SharedInt,
-        shm: SharedMemory,
-        # readonly: bool = True,
-    ) -> None:
-        self._array = shmarr
-
-        # indexes for first and last indices corresponding
-        # to fille data
-        self._first = first
-        self._last = last
-
-        self._len = len(shmarr)
-        self._shm = shm
-        self._post_init: bool = False
-
-        # pushing data does not write the index (aka primary key)
-        self._write_fields: list[str] | None = None
-        dtype = shmarr.dtype
-        if dtype.fields:
-            self._write_fields = list(shmarr.dtype.fields.keys())[1:]
-
-    # TODO: ringbuf api?
-
-    @property
-    def _token(self) -> NDToken:
-        return NDToken(
-            shm_name=self._shm.name,
-            shm_first_index_name=self._first._shm.name,
-            shm_last_index_name=self._last._shm.name,
-            dtype_descr=tuple(self._array.dtype.descr),
-            size=self._len,
-        )
-
-    @property
-    def token(self) -> dict:
-        """Shared memory token that can be serialized and used by
-        another process to attach to this array.
-        """
-        return self._token.as_msg()
-
-    @property
-    def index(self) -> int:
-        return self._last.value % self._len
-
-    @property
-    def array(self) -> np.ndarray:
-        '''
-        Return an up-to-date ``np.ndarray`` view of the
-        so-far-written data to the underlying shm buffer.
-
-        '''
-        a = self._array[self._first.value:self._last.value]
-
-        # first, last = self._first.value, self._last.value
-        # a = self._array[first:last]
-
-        # TODO: eventually comment this once we've not seen it in the
-        # wild in a long time..
-        # XXX: race where first/last indexes cause a reader
-        # to load an empty array..
-        if len(a) == 0 and self._post_init:
-            raise RuntimeError('Empty array race condition hit!?')
-            # breakpoint()
-
-        return a
-
-    def ustruct(
-        self,
-        fields: Optional[list[str]] = None,
-
-        # type that all field values will be cast to
-        # in the returned view.
-        common_dtype: np.dtype = float,
-
-    ) -> np.ndarray:
-
-        array = self._array
-
-        if fields:
-            selection = array[fields]
-            # fcount = len(fields)
-        else:
-            selection = array
-            # fcount = len(array.dtype.fields)
-
-        # XXX: manual ``.view()`` attempt that also doesn't work.
-        # uview = selection.view(
-        #     dtype='<f16',
-        # ).reshape(-1, 4, order='A')
-
-        # assert len(selection) == len(uview)
-
-        u = rfn.structured_to_unstructured(
-            selection,
-            # dtype=float,
-            copy=True,
-        )
-
-        # unstruct = np.ndarray(u.shape, dtype=a.dtype, buffer=shm.buf)
-        # array[:] = a[:]
-        return u
-        # return ShmArray(
-        #     shmarr=u,
-        #     first=self._first,
-        #     last=self._last,
-        #     shm=self._shm
-        # )
-
-    def last(
-        self,
-        length: int = 1,
-
-    ) -> np.ndarray:
-        '''
-        Return the last ``length``'s worth of ("row") entries from the
-        array.
-
-        '''
-        return self.array[-length:]
-
-    def push(
-        self,
-        data: np.ndarray,
-
-        field_map: Optional[dict[str, str]] = None,
-        prepend: bool = False,
-        update_first: bool = True,
-        start: int | None = None,
-
-    ) -> int:
-        '''
-        Ring buffer like "push" to append data
-        into the buffer and return updated "last" index.
-
-        NB: no actual ring logic yet to give a "loop around" on overflow
-        condition, lel.
-
-        '''
-        length = len(data)
-
-        if prepend:
-            index = (start or self._first.value) - length
-
-            if index < 0:
-                raise ValueError(
-                    f'Array size of {self._len} was overrun during prepend.\n'
-                    f'You have passed {abs(index)} too many datums.'
-                )
-
-        else:
-            index = start if start is not None else self._last.value
-
-        end = index + length
-
-        if field_map:
-            src_names, dst_names = zip(*field_map.items())
-        else:
-            dst_names = src_names = self._write_fields
-
-        try:
-            self._array[
-                list(dst_names)
-            ][index:end] = data[list(src_names)][:]
-
-            # NOTE: there was a race here between updating
-            # the first and last indices and when the next reader
-            # tries to access ``.array`` (which due to the index
-            # overlap will be empty). Pretty sure we've fixed it now
-            # but leaving this here as a reminder.
-            if (
-                prepend
-                and update_first
-                and length
-            ):
-                assert index < self._first.value
-
-            if (
-                index < self._first.value
-                and update_first
-            ):
-                assert prepend, 'prepend=True not passed but index decreased?'
-                self._first.value = index
-
-            elif not prepend:
-                self._last.value = end
-
-            self._post_init = True
-            return end
-
-        except ValueError as err:
-            if field_map:
-                raise
-
-            # should raise if diff detected
-            self.diff_err_fields(data)
-            raise err
-
-    def diff_err_fields(
-        self,
-        data: np.ndarray,
-    ) -> None:
-        # reraise with any field discrepancy
-        our_fields, their_fields = (
-            set(self._array.dtype.fields),
-            set(data.dtype.fields),
-        )
-
-        only_in_ours = our_fields - their_fields
-        only_in_theirs = their_fields - our_fields
-
-        if only_in_ours:
-            raise TypeError(
-                f"Input array is missing field(s): {only_in_ours}"
-            )
-        elif only_in_theirs:
-            raise TypeError(
-                f"Input array has unknown field(s): {only_in_theirs}"
-            )
-
-    # TODO: support "silent" prepends that don't update ._first.value?
-    def prepend(
-        self,
-        data: np.ndarray,
-    ) -> int:
-        end = self.push(data, prepend=True)
-        assert end
-
-    def close(self) -> None:
-        self._first._shm.close()
-        self._last._shm.close()
-        self._shm.close()
-
-    def destroy(self) -> None:
-        if _USE_POSIX:
-            # We manually unlink to bypass all the "resource tracker"
-            # nonsense meant for non-SC systems.
-            shm_unlink(self._shm.name)
-
-        self._first.destroy()
-        self._last.destroy()
-
-    def flush(self) -> None:
-        # TODO: flush to storage backend like markestore?
-        ...
-
-
-def open_shm_ndarray(
-    size: int,
-    key: str | None = None,
-    dtype: np.dtype | None = None,
-    append_start_index: int | None = None,
-    readonly: bool = False,
-
-) -> ShmArray:
-    '''
-    Open a memory shared ``numpy`` using the standard library.
-
-    This call unlinks (aka permanently destroys) the buffer on teardown
-    and thus should be used from the parent-most accessor (process).
-
-    '''
-    # create new shared mem segment for which we
-    # have write permission
-    a = np.zeros(size, dtype=dtype)
-    a['index'] = np.arange(len(a))
-
-    shm = SharedMemory(
-        name=key,
-        create=True,
-        size=a.nbytes
-    )
-    array = np.ndarray(
-        a.shape,
-        dtype=a.dtype,
-        buffer=shm.buf
-    )
-    array[:] = a[:]
-    array.setflags(write=int(not readonly))
-
-    token = _make_token(
-        key=key,
-        size=size,
-        dtype=dtype,
-    )
-
-    # create single entry arrays for storing an first and last indices
-    first = SharedInt(
-        shm=SharedMemory(
-            name=token.shm_first_index_name,
-            create=True,
-            size=4,  # std int
-        )
-    )
-
-    last = SharedInt(
-        shm=SharedMemory(
-            name=token.shm_last_index_name,
-            create=True,
-            size=4,  # std int
-        )
-    )
-
-    # Start the "real-time" append-updated (or "pushed-to") section
-    # after some start index: ``append_start_index``. This allows appending
-    # from a start point in the array which isn't the 0 index and looks
-    # something like,
-    # -------------------------
-    # |              |        i
-    # _________________________
-    # <-------------> <------->
-    #  history         real-time
-    #
-    # Once fully "prepended", the history section will leave the
-    # ``ShmArray._start.value: int = 0`` and the yet-to-be written
-    # real-time section will start at ``ShmArray.index: int``.
-
-    # this sets the index to nearly 2/3rds into the the length of
-    # the buffer leaving at least a "days worth of second samples"
-    # for the real-time section.
-    if append_start_index is None:
-        append_start_index = round(size * 0.616)
-
-    last.value = first.value = append_start_index
-
-    shmarr = ShmArray(
-        array,
-        first,
-        last,
-        shm,
-    )
-
-    assert shmarr._token == token
-    _known_tokens[key] = shmarr.token
-
-    # "unlink" created shm on process teardown by
-    # pushing teardown calls onto actor context stack
-    stack = tractor.current_actor().lifetime_stack
-    stack.callback(shmarr.close)
-    stack.callback(shmarr.destroy)
-
-    return shmarr
-
-
-def attach_shm_ndarray(
-    token: tuple[str, str, tuple[str, str]],
-    readonly: bool = True,
-
-) -> ShmArray:
-    '''
-    Attach to an existing shared memory array previously
-    created by another process using ``open_shared_array``.
-
-    No new shared mem is allocated but wrapper types for read/write
-    access are constructed.
-
-    '''
-    token = NDToken.from_msg(token)
-    key = token.shm_name
-
-    if key in _known_tokens:
-        assert NDToken.from_msg(_known_tokens[key]) == token, "WTF"
-
-    # XXX: ugh, looks like due to the ``shm_open()`` C api we can't
-    # actually place files in a subdir, see discussion here:
-    # https://stackoverflow.com/a/11103289
-
-    # attach to array buffer and view as per dtype
-    _err: Optional[Exception] = None
-    for _ in range(3):
-        try:
-            shm = SharedMemory(
-                name=key,
-                create=False,
-            )
-            break
-        except OSError as oserr:
-            _err = oserr
-            time.sleep(0.1)
-    else:
-        if _err:
-            raise _err
-
-    shmarr = np.ndarray(
-        (token.size,),
-        dtype=token.dtype,
-        buffer=shm.buf
-    )
-    shmarr.setflags(write=int(not readonly))
-
-    first = SharedInt(
-        shm=SharedMemory(
-            name=token.shm_first_index_name,
-            create=False,
-            size=4,  # std int
-        ),
-    )
-    last = SharedInt(
-        shm=SharedMemory(
-            name=token.shm_last_index_name,
-            create=False,
-            size=4,  # std int
-        ),
-    )
-
-    # make sure we can read
-    first.value
-
-    sha = ShmArray(
-        shmarr,
-        first,
-        last,
-        shm,
-    )
-    # read test
-    sha.array
-
-    # Stash key -> token knowledge for future queries
-    # via `maybe_opepn_shm_array()` but only after we know
-    # we can attach.
-    if key not in _known_tokens:
-        _known_tokens[key] = token
-
-    # "close" attached shm on actor teardown
-    tractor.current_actor().lifetime_stack.callback(sha.close)
-
-    return sha
-
-
-def maybe_open_shm_ndarray(
-    key: str,  # unique identifier for segment
-    size: int,
-    dtype: np.dtype | None = None,
-    append_start_index: int = 0,
-    readonly: bool = True,
-
-) -> tuple[ShmArray, bool]:
-    '''
-    Attempt to attach to a shared memory block using a "key" lookup
-    to registered blocks in the users overall "system" registry
-    (presumes you don't have the block's explicit token).
-
-    This function is meant to solve the problem of discovering whether
-    a shared array token has been allocated or discovered by the actor
-    running in **this** process. Systems where multiple actors may seek
-    to access a common block can use this function to attempt to acquire
-    a token as discovered by the actors who have previously stored
-    a "key" -> ``NDToken`` map in an actor local (aka python global)
-    variable.
-
-    If you know the explicit ``NDToken`` for your memory segment instead
-    use ``attach_shm_array``.
-
-    '''
-    try:
-        # see if we already know this key
-        token = _known_tokens[key]
-        return (
-            attach_shm_ndarray(
-                token=token,
-                readonly=readonly,
-            ),
-            False,  # not newly opened
-        )
-    except KeyError:
-        log.warning(f"Could not find {key} in shms cache")
-        if dtype:
-            token = _make_token(
-                key,
-                size=size,
-                dtype=dtype,
-            )
-        else:
-
-            try:
-                return (
-                    attach_shm_ndarray(
-                        token=token,
-                        readonly=readonly,
-                    ),
-                    False,
-                )
-            except FileNotFoundError:
-                log.warning(f"Could not attach to shm with token {token}")
-
-        # This actor does not know about memory
-        # associated with the provided "key".
-        # Attempt to open a block and expect
-        # to fail if a block has been allocated
-        # on the OS by someone else.
-        return (
-            open_shm_ndarray(
-                key=key,
-                size=size,
-                dtype=dtype,
-                append_start_index=append_start_index,
-                readonly=readonly,
-            ),
-            True,
-        )
-
-
-class ShmList(ShareableList):
-    '''
-    Carbon copy of ``.shared_memory.ShareableList`` with a few
-    enhancements:
-
-    - readonly mode via instance var flag  `._readonly: bool`
-    - ``.__getitem__()`` accepts ``slice`` inputs
-    - exposes the underlying buffer "name" as a ``.key: str``
-
-    '''
-    def __init__(
-        self,
-        sequence: list | None = None,
-        *,
-        name: str | None = None,
-        readonly: bool = True
-
-    ) -> None:
-        self._readonly = readonly
-        self._key = name
-        return super().__init__(
-            sequence=sequence,
-            name=name,
-        )
-
-    @property
-    def key(self) -> str:
-        return self._key
-
-    @property
-    def readonly(self) -> bool:
-        return self._readonly
-
-    def __setitem__(
-        self,
-        position,
-        value,
-
-    ) -> None:
-
-        # mimick ``numpy`` error
-        if self._readonly:
-            raise ValueError('assignment destination is read-only')
-
-        return super().__setitem__(position, value)
-
-    def __getitem__(
-        self,
-        indexish,
-    ) -> list:
-
-        # NOTE: this is a non-writeable view (copy?) of the buffer
-        # in a new list instance.
-        if isinstance(indexish, slice):
-            return list(self)[indexish]
-
-        return super().__getitem__(indexish)
-
-    # TODO: should we offer a `.array` and `.push()` equivalent
-    # to the `ShmArray`?
-    # currently we have the following limitations:
-    # - can't write slices of input using traditional slice-assign
-    #   syntax due to the ``ShareableList.__setitem__()`` implementation.
-    # - ``list(shmlist)`` returns a non-mutable copy instead of
-    #   a writeable view which would be handier numpy-style ops.
-
-
-def open_shm_list(
-    key: str,
-    sequence: list | None = None,
-    size: int = int(2 ** 10),
-    dtype: float | int | bool | str | bytes | None = float,
-    readonly: bool = True,
-
-) -> ShmList:
-
-    if sequence is None:
-        default = {
-            float: 0.,
-            int: 0,
-            bool: True,
-            str: 'doggy',
-            None: None,
-        }[dtype]
-        sequence = [default] * size
-
-    shml = ShmList(
-        sequence=sequence,
-        name=key,
-        readonly=readonly,
-    )
-
-    # "close" attached shm on actor teardown
-    try:
-        actor = tractor.current_actor()
-        actor.lifetime_stack.callback(shml.shm.close)
-        actor.lifetime_stack.callback(shml.shm.unlink)
-    except RuntimeError:
-        log.warning('tractor runtime not active, skipping teardown steps')
-
-    return shml
-
-
-def attach_shm_list(
-    key: str,
-    readonly: bool = False,
-
-) -> ShmList:
-
-    return ShmList(
-        name=key,
-        readonly=readonly,
-    )

tractor/_spawn.py

@@ -31,25 +31,24 @@ from typing import (
     TYPE_CHECKING,
 )
 
-from exceptiongroup import BaseExceptionGroup
 import trio
-from trio_typing import TaskStatus
+from trio import TaskStatus
 
-from .devx._debug import (
+from ._debug import (
     maybe_wait_for_debugger,
     acquire_debug_lock,
 )
-from ._state import (
+from tractor._state import (
     current_actor,
     is_main_process,
     is_root_process,
     debug_mode,
 )
-from .log import get_logger
+from tractor.log import get_logger
-from ._portal import Portal
+from tractor._portal import Portal
-from ._runtime import Actor
+from tractor._runtime import Actor
-from ._entry import _mp_main
+from tractor._entry import _mp_main
-from ._exceptions import ActorFailure
+from tractor._exceptions import ActorFailure
 
 
 if TYPE_CHECKING:
@@ -144,7 +143,7 @@ async def exhaust_portal(
 
         # XXX: streams should never be reaped here since they should
         # always be established and shutdown using a context manager api
-        final = await portal.result()
+        final: Any = await portal.result()
 
     except (
         Exception,
@@ -152,13 +151,23 @@ async def exhaust_portal(
     ) as err:
         # we reraise in the parent task via a ``BaseExceptionGroup``
         return err
+
     except trio.Cancelled as err:
         # lol, of course we need this too ;P
         # TODO: merge with above?
-        log.warning(f"Cancelled result waiter for {portal.actor.uid}")
+        log.warning(
+            'Cancelled portal result waiter task:\n'
+            f'uid: {portal.channel.uid}\n'
+            f'error: {err}\n'
+        )
         return err
+
     else:
-        log.debug(f"Returning final result: {final}")
+        log.debug(
+            f'Returning final result from portal:\n'
+            f'uid: {portal.channel.uid}\n'
+            f'result: {final}\n'
+        )
         return final
 
 
@@ -170,34 +179,46 @@ async def cancel_on_completion(
 
 ) -> None:
     '''
-    Cancel actor gracefully once it's "main" portal's
+    Cancel actor gracefully once its "main" portal's
     result arrives.
 
-    Should only be called for actors spawned with `run_in_actor()`.
+    Should only be called for actors spawned via the
+    `Portal.run_in_actor()` API.
+
+    => and really this API will be deprecated and should be
+    re-implemented as a `.hilevel.one_shot_task_nursery()`..)
 
     '''
     # if this call errors we store the exception for later
     # in ``errors`` which will be reraised inside
     # an exception group and we still send out a cancel request
-    result = await exhaust_portal(portal, actor)
+    result: Any|Exception = await exhaust_portal(portal, actor)
     if isinstance(result, Exception):
-        errors[actor.uid] = result
+        errors[actor.uid]: Exception = result
-        log.warning(
+        log.cancel(
-            f"Cancelling {portal.channel.uid} after error {result}"
+            'Cancelling subactor runtime due to error:\n\n'
+            f'Portal.cancel_actor() => {portal.channel.uid}\n\n'
+            f'error: {result}\n'
         )
 
     else:
         log.runtime(
-            f"Cancelling {portal.channel.uid} gracefully "
+            'Cancelling subactor gracefully:\n\n'
-            f"after result {result}")
+            f'Portal.cancel_actor() => {portal.channel.uid}\n\n'
+            f'result: {result}\n'
+        )
 
     # cancel the process now that we have a final result
     await portal.cancel_actor()
 
 
-async def do_hard_kill(
+async def hard_kill(
     proc: trio.Process,
-    terminate_after: int = 3,
+    terminate_after: int = 1.6,
+
+    # NOTE: for mucking with `.pause()`-ing inside the runtime
+    # whilst also hacking on it XD
+    # terminate_after: int = 99999,
 
     # NOTE: for mucking with `.pause()`-ing inside the runtime
     # whilst also hacking on it XD
@@ -219,11 +240,14 @@ async def do_hard_kill(
       to be handled.
 
     '''
+    log.cancel(
+        'Terminating sub-proc:\n'
+        f'|_{proc}\n'
+    )
     # NOTE: this timeout used to do nothing since we were shielding
     # the ``.wait()`` inside ``new_proc()`` which will pretty much
     # never release until the process exits, now it acts as
     # a hard-kill time ultimatum.
-    log.debug(f"Terminating {proc}")
     with trio.move_on_after(terminate_after) as cs:
 
         # NOTE: code below was copied verbatim from the now deprecated
@@ -260,11 +284,17 @@ async def do_hard_kill(
     # zombies (as a feature) we ask the OS to do send in the
     # removal swad as the last resort.
     if cs.cancelled_caught:
-        log.critical(f"#ZOMBIE_LORD_IS_HERE: {proc}")
+        # TODO: toss in the skynet-logo face as ascii art?
+        log.critical(
+            # 'Well, the #ZOMBIE_LORD_IS_HERE# to collect\n'
+            '#T-800 deployed to collect zombie B0\n'
+            f'|\n'
+            f'|_{proc}\n'
+        )
         proc.kill()
 
 
-async def soft_wait(
+async def soft_kill(
 
     proc: ProcessType,
     wait_func: Callable[
@@ -275,16 +305,25 @@ async def soft_wait(
 
 ) -> None:
     '''
-    Wait for proc termination but **dont' yet** teardown
+    Wait for proc termination but **don't yet** teardown
-    std-streams (since it will clobber any ongoing pdb REPL
+    std-streams since it will clobber any ongoing pdb REPL
-    session). This is our "soft" (and thus itself cancellable)
+    session.
-    join/reap on an actor-runtime-in-process.
+
+    This is our "soft"/graceful, and thus itself also cancellable,
+    join/reap on an actor-runtime-in-process shutdown; it is
+    **not** the same as a "hard kill" via an OS signal (for that
+    see `.hard_kill()`).
 
     '''
-    uid = portal.channel.uid
+    uid: tuple[str, str] = portal.channel.uid
     try:
-        log.cancel(f'Soft waiting on actor:\n{uid}')
+        log.cancel(
+            'Soft killing sub-actor via `Portal.cancel_actor()`\n'
+            f'|_{proc}\n'
+        )
+        # wait on sub-proc to signal termination
         await wait_func(proc)
+
     except trio.Cancelled:
         # if cancelled during a soft wait, cancel the child
         # actor before entering the hard reap sequence
@@ -296,8 +335,9 @@ async def soft_wait(
 
             async def cancel_on_proc_deth():
                 '''
-                Cancel the actor cancel request if we detect that
+                "Cancel-the-cancel" request: if we detect that the
-                that the process terminated.
+                underlying sub-process exited prior to
+                a `Portal.cancel_actor()` call completing .
 
                 '''
                 await wait_func(proc)
@@ -314,10 +354,10 @@ async def soft_wait(
 
             if proc.poll() is None:  # type: ignore
                 log.warning(
-                    'Actor still alive after cancel request:\n'
+                    'Subactor still alive after cancel request?\n\n'
-                    f'{uid}'
+                    f'uid: {uid}\n'
+                    f'|_{proc}\n'
                 )
-
                 n.cancel_scope.cancel()
         raise
 
@@ -341,7 +381,7 @@ async def new_proc(
 ) -> None:
 
     # lookup backend spawning target
-    target = _methods[_spawn_method]
+    target: Callable = _methods[_spawn_method]
 
     # mark the new actor with the global spawn method
     subactor._spawn_method = _spawn_method
@@ -409,19 +449,22 @@ async def trio_proc(
         spawn_cmd.append("--asyncio")
 
     cancelled_during_spawn: bool = False
-    proc: trio.Process | None = None
+    proc: trio.Process|None = None
     try:
         try:
             # TODO: needs ``trio_typing`` patch?
             proc = await trio.lowlevel.open_process(spawn_cmd)
-
+            log.runtime(
-            log.runtime(f"Started {proc}")
+                'Started new sub-proc\n'
+                f'|_{proc}\n'
+            )
 
             # wait for actor to spawn and connect back to us
             # channel should have handshake completed by the
             # local actor by the time we get a ref to it
             event, chan = await actor_nursery._actor.wait_for_peer(
-                subactor.uid)
+                subactor.uid
+            )
 
         except trio.Cancelled:
             cancelled_during_spawn = True
@@ -482,7 +525,7 @@ async def trio_proc(
             # This is a "soft" (cancellable) join/reap which
             # will remote cancel the actor on a ``trio.Cancelled``
             # condition.
-            await soft_wait(
+            await soft_kill(
                 proc,
                 trio.Process.wait,
                 portal
@@ -491,8 +534,9 @@ async def trio_proc(
             # cancel result waiter that may have been spawned in
             # tandem if not done already
             log.cancel(
-                "Cancelling existing result waiter task for "
+                'Cancelling existing result waiter task for '
-                f"{subactor.uid}")
+                f'{subactor.uid}'
+            )
             nursery.cancel_scope.cancel()
 
     finally:
@@ -510,22 +554,39 @@ async def trio_proc(
                         with trio.move_on_after(0.5):
                             await proc.wait()
 
-                if is_root_process():
+                log.pdb(
-                    # TODO: solve the following issue where we need
+                    'Delaying subproc reaper while debugger locked..'
-                    # to do a similar wait like this but in an
+                )
-                    # "intermediary" parent actor that itself isn't
+                await maybe_wait_for_debugger(
-                    # in debug but has a child that is, and we need
+                    child_in_debug=_runtime_vars.get(
-                    # to hold off on relaying SIGINT until that child
+                        '_debug_mode', False
-                    # is complete.
+                    ),
-                    # https://github.com/goodboy/tractor/issues/320
+                    # TODO: need a diff value then default?
-                    await maybe_wait_for_debugger(
+                    # poll_steps=9999999,
-                        child_in_debug=_runtime_vars.get(
+                )
-                            '_debug_mode', False),
+                # TODO: solve the following issue where we need
-                    )
+                # to do a similar wait like this but in an
+                # "intermediary" parent actor that itself isn't
+                # in debug but has a child that is, and we need
+                # to hold off on relaying SIGINT until that child
+                # is complete.
+                # https://github.com/goodboy/tractor/issues/320
+                # -[ ] we need to handle non-root parent-actors specially
+                # by somehow determining if a child is in debug and then
+                # avoiding cancel/kill of said child by this
+                # (intermediary) parent until such a time as the root says
+                # the pdb lock is released and we are good to tear down
+                # (our children)..
+                #
+                # -[ ] so maybe something like this where we try to
+                #     acquire the lock and get notified of who has it,
+                #     check that uid against our known children?
+                # this_uid: tuple[str, str] = current_actor().uid
+                # await acquire_debug_lock(this_uid)
 
                 if proc.poll() is None:
                     log.cancel(f"Attempting to hard kill {proc}")
-                    await do_hard_kill(proc)
+                    await hard_kill(proc)
 
                 log.debug(f"Joined {proc}")
         else:
@@ -669,7 +730,7 @@ async def mp_proc(
             # This is a "soft" (cancellable) join/reap which
             # will remote cancel the actor on a ``trio.Cancelled``
             # condition.
-            await soft_wait(
+            await soft_kill(
                 proc,
                 proc_waiter,
                 portal

tractor/_state.py

@@ -18,12 +18,18 @@
 Per process state
 
 """
+from __future__ import annotations
 from typing import (
-    Optional,
     Any,
+    TYPE_CHECKING,
 )
 
-_current_actor: Optional['Actor'] = None  # type: ignore # noqa
+if TYPE_CHECKING:
+    from ._runtime import Actor
+
+
+_current_actor: Actor|None = None  # type: ignore # noqa
+_last_actor_terminated: Actor|None = None
 _runtime_vars: dict[str, Any] = {
     '_debug_mode': False,
     '_is_root': False,
@@ -31,14 +37,49 @@ _runtime_vars: dict[str, Any] = {
 }
 
 
-def current_actor(err_on_no_runtime: bool = True) -> 'Actor':  # type: ignore # noqa
+def last_actor() -> Actor|None:
+    '''
+    Try to return last active `Actor` singleton
+    for this process.
+
+    For case where runtime already exited but someone is asking
+    about the "last" actor probably to get its `.uid: tuple`.
+
+    '''
+    return _last_actor_terminated
+
+
+def current_actor(
+    err_on_no_runtime: bool = True,
+) -> Actor:
     '''
     Get the process-local actor instance.
 
     '''
-    from ._exceptions import NoRuntime
+    if (
-    if _current_actor is None and err_on_no_runtime:
+        err_on_no_runtime
-        raise NoRuntime("No local actor has been initialized yet")
+        and _current_actor is None
+    ):
+        msg: str = 'No local actor has been initialized yet'
+        from ._exceptions import NoRuntime
+
+        if last := last_actor():
+            msg += (
+                f'Apparently the lact active actor was\n'
+                f'|_{last}\n'
+                f'|_{last.uid}\n'
+            )
+        # no actor runtime has (as of yet) ever been started for
+        # this process.
+        else:
+            msg += (
+                'No last actor found?\n'
+                'Did you forget to open one of:\n\n'
+                '- `tractor.open_root_actor()`\n'
+                '- `tractor.open_nursery()`\n'
+            )
+
+        raise NoRuntime(msg)
 
     return _current_actor
 

tractor/_streaming.py

@@ -21,8 +21,9 @@ The machinery and types behind ``Context.open_stream()``
 
 '''
 from __future__ import annotations
-import inspect
 from contextlib import asynccontextmanager as acm
+import inspect
+from pprint import pformat
 from typing import (
     Any,
     Callable,
@@ -35,6 +36,7 @@ import trio
 
 from ._exceptions import (
     _raise_from_no_key_in_msg,
+    ContextCancelled,
 )
 from .log import get_logger
 from .trionics import (
@@ -84,31 +86,47 @@ class MsgStream(trio.abc.Channel):
         self._broadcaster = _broadcaster
 
         # flag to denote end of stream
-        self._eoc: bool = False
+        self._eoc: bool|trio.EndOfChannel = False
-        self._closed: bool = False
+        self._closed: bool|trio.ClosedResourceError = False
 
     # delegate directly to underlying mem channel
-    def receive_nowait(self):
+    def receive_nowait(
-        msg = self._rx_chan.receive_nowait()
+        self,
-        try:
+        allow_msg_keys: list[str] = ['yield'],
-            return msg['yield']
+    ):
-        except KeyError as kerr:
+        msg: dict = self._rx_chan.receive_nowait()
-            _raise_from_no_key_in_msg(
+        for (
-                ctx=self._ctx,
+            i,
-                msg=msg,
+            key,
-                src_err=kerr,
+        ) in enumerate(allow_msg_keys):
-                log=log,
+            try:
-                expect_key='yield',
+                return msg[key]
-                stream=self,
+            except KeyError as kerr:
-            )
+                if i < (len(allow_msg_keys) - 1):
+                    continue
 
-    async def receive(self):
+                _raise_from_no_key_in_msg(
+                    ctx=self._ctx,
+                    msg=msg,
+                    src_err=kerr,
+                    log=log,
+                    expect_key=key,
+                    stream=self,
+                )
+
+    async def receive(
+        self,
+
+        hide_tb: bool = True,
+    ):
         '''
         Receive a single msg from the IPC transport, the next in
         sequence sent by the far end task (possibly in order as
         determined by the underlying protocol).
 
         '''
+        __tracebackhide__: bool = hide_tb
+
         # NOTE: `trio.ReceiveChannel` implements
         # EOC handling as follows (aka uses it
         # to gracefully exit async for loops):
@@ -118,82 +136,198 @@ class MsgStream(trio.abc.Channel):
         #         return await self.receive()
         #     except trio.EndOfChannel:
         #         raise StopAsyncIteration
-
+        #
         # see ``.aclose()`` for notes on the old behaviour prior to
         # introducing this
         if self._eoc:
-            raise trio.EndOfChannel
+            raise self._eoc
 
         if self._closed:
-            raise trio.ClosedResourceError('This stream was closed')
+            raise self._closed
 
+        src_err: Exception|None = None  # orig tb
         try:
-            msg = await self._rx_chan.receive()
+            try:
-            return msg['yield']
+                msg = await self._rx_chan.receive()
+                return msg['yield']
 
-        except KeyError as kerr:
+            except KeyError as kerr:
-            _raise_from_no_key_in_msg(
+                src_err = kerr
-                ctx=self._ctx,
-                msg=msg,
-                src_err=kerr,
-                log=log,
-                expect_key='yield',
-                stream=self,
-            )
 
+                # NOTE: may raise any of the below error types
+                # includg EoC when a 'stop' msg is found.
+                _raise_from_no_key_in_msg(
+                    ctx=self._ctx,
+                    msg=msg,
+                    src_err=kerr,
+                    log=log,
+                    expect_key='yield',
+                    stream=self,
+                )
+
+        # XXX: the stream terminates on either of:
+        # - via `self._rx_chan.receive()` raising  after manual closure
+        #   by the rpc-runtime OR,
+        # - via a received `{'stop': ...}` msg from remote side.
+        #   |_ NOTE: previously this was triggered by calling
+        #   ``._rx_chan.aclose()`` on the send side of the channel inside
+        #   `Actor._push_result()`, but now the 'stop' message handling
+        #   has been put just above inside `_raise_from_no_key_in_msg()`.
         except (
-            trio.ClosedResourceError,  # by self._rx_chan
+            trio.EndOfChannel,
-            trio.EndOfChannel,  # by self._rx_chan or `stop` msg from far end
+        ) as eoc:
-        ):
+            src_err = eoc
-            # XXX: we close the stream on any of these error conditions:
+            self._eoc = eoc
-
-            # a ``ClosedResourceError`` indicates that the internal
-            # feeder memory receive channel was closed likely by the
-            # runtime after the associated transport-channel
-            # disconnected or broke.
-
-            # an ``EndOfChannel`` indicates either the internal recv
-            # memchan exhausted **or** we raisesd it just above after
-            # receiving a `stop` message from the far end of the stream.
-
-            # Previously this was triggered by calling ``.aclose()`` on
-            # the send side of the channel inside
-            # ``Actor._push_result()`` (should still be commented code
-            # there - which should eventually get removed), but now the
-            # 'stop' message handling has been put just above.
 
             # TODO: Locally, we want to close this stream gracefully, by
             # terminating any local consumers tasks deterministically.
-            # One we have broadcast support, we **don't** want to be
+            # Once we have broadcast support, we **don't** want to be
             # closing this stream and not flushing a final value to
             # remaining (clone) consumers who may not have been
             # scheduled to receive it yet.
+            # try:
+            #     maybe_err_msg_or_res: dict = self._rx_chan.receive_nowait()
+            #     if maybe_err_msg_or_res:
+            #         log.warning(
+            #             'Discarding un-processed msg:\n'
+            #             f'{maybe_err_msg_or_res}'
+            #         )
+            # except trio.WouldBlock:
+            #     # no queued msgs that might be another remote
+            #     # error, so just raise the original EoC
+            #     pass
 
-            # when the send is closed we assume the stream has
+            # raise eoc
-            # terminated and signal this local iterator to stop
-            await self.aclose()
 
-            raise  # propagate
+        # a ``ClosedResourceError`` indicates that the internal
+        # feeder memory receive channel was closed likely by the
+        # runtime after the associated transport-channel
+        # disconnected or broke.
+        except trio.ClosedResourceError as cre:  # by self._rx_chan.receive()
+            src_err = cre
+            log.warning(
+                '`Context._rx_chan` was already closed?'
+            )
+            self._closed = cre
 
-    async def aclose(self):
+        # when the send is closed we assume the stream has
+        # terminated and signal this local iterator to stop
+        drained: list[Exception|dict] = await self.aclose()
+        if drained:
+            # from .devx import pause
+            # await pause()
+            log.warning(
+                'Drained context msgs during closure:\n'
+                f'{drained}'
+            )
+        # TODO: pass these to the `._ctx._drained_msgs: deque`
+        # and then iterate them as part of any `.result()` call?
+
+        # NOTE XXX: if the context was cancelled or remote-errored
+        # but we received the stream close msg first, we
+        # probably want to instead raise the remote error
+        # over the end-of-stream connection error since likely
+        # the remote error was the source cause?
+        ctx: Context = self._ctx
+        ctx.maybe_raise(
+            raise_ctxc_from_self_call=True,
+        )
+
+        # propagate any error but hide low-level frame details
+        # from the caller by default for debug noise reduction.
+        if (
+            hide_tb
+
+            # XXX NOTE XXX don't reraise on certain
+            # stream-specific internal error types like,
+            #
+            # - `trio.EoC` since we want to use the exact instance
+            #   to ensure that it is the error that bubbles upward
+            #   for silent absorption by `Context.open_stream()`.
+            and not self._eoc
+
+            # - `RemoteActorError` (or `ContextCancelled`) if it gets
+            #   raised from `_raise_from_no_key_in_msg()` since we
+            #   want the same (as the above bullet) for any
+            #   `.open_context()` block bubbled error raised by
+            #   any nearby ctx API remote-failures.
+            # and not isinstance(src_err, RemoteActorError)
+        ):
+            raise type(src_err)(*src_err.args) from src_err
+        else:
+            raise src_err
+
+    async def aclose(self) -> list[Exception|dict]:
         '''
         Cancel associated remote actor task and local memory channel on
         close.
 
+        Notes: 
+         - REMEMBER that this is also called by `.__aexit__()` so
+           careful consideration must be made to handle whatever
+           internal stsate is mutated, particuarly in terms of
+           draining IPC msgs!
+
+         - more or less we try to maintain adherance to trio's `.aclose()` semantics:
+           https://trio.readthedocs.io/en/stable/reference-io.html#trio.abc.AsyncResource.aclose
         '''
-        # XXX: keep proper adherance to trio's `.aclose()` semantics:
-        # https://trio.readthedocs.io/en/stable/reference-io.html#trio.abc.AsyncResource.aclose
-        rx_chan = self._rx_chan
 
-        if rx_chan._closed:
+        # rx_chan = self._rx_chan
-            log.cancel(f"{self} is already closed")
 
+        # XXX NOTE XXX
+        # it's SUPER IMPORTANT that we ensure we don't DOUBLE
+        # DRAIN msgs on closure so avoid getting stuck handing on
+        # the `._rx_chan` since we call this method on
+        # `.__aexit__()` as well!!!
+        # => SO ENSURE WE CATCH ALL TERMINATION STATES in this
+        # block including the EoC..
+        if self.closed:
             # this stream has already been closed so silently succeed as
             # per ``trio.AsyncResource`` semantics.
             # https://trio.readthedocs.io/en/stable/reference-io.html#trio.abc.AsyncResource.aclose
-            return
+            return []
 
-        self._eoc = True
+        ctx: Context = self._ctx
+        drained: list[Exception|dict] = []
+        while not drained:
+            try:
+                maybe_final_msg = self.receive_nowait(
+                    allow_msg_keys=['yield', 'return'],
+                )
+                if maybe_final_msg:
+                    log.debug(
+                        'Drained un-processed stream msg:\n'
+                        f'{pformat(maybe_final_msg)}'
+                    )
+                    # TODO: inject into parent `Context` buf?
+                    drained.append(maybe_final_msg)
+
+            # NOTE: we only need these handlers due to the
+            # `.receive_nowait()` call above which may re-raise
+            # one of these errors on a msg key error!
+
+            except trio.WouldBlock as be:
+                drained.append(be)
+                break
+
+            except trio.EndOfChannel as eoc:
+                self._eoc: Exception = eoc
+                drained.append(eoc)
+                break
+
+            except trio.ClosedResourceError as cre:
+                self._closed = cre
+                drained.append(cre)
+                break
+
+            except ContextCancelled as ctxc:
+                # log.exception('GOT CTXC')
+                log.cancel(
+                    'Context was cancelled during stream closure:\n'
+                    f'canceller: {ctxc.canceller}\n'
+                    f'{pformat(ctxc.msgdata)}'
+                )
+                break
 
         # NOTE: this is super subtle IPC messaging stuff:
         # Relay stop iteration to far end **iff** we're
@@ -224,26 +358,40 @@ class MsgStream(trio.abc.Channel):
         except (
             trio.BrokenResourceError,
             trio.ClosedResourceError
-        ):
+        ) as re:
             # the underlying channel may already have been pulled
             # in which case our stop message is meaningless since
             # it can't traverse the transport.
-            ctx = self._ctx
             log.warning(
                 f'Stream was already destroyed?\n'
                 f'actor: {ctx.chan.uid}\n'
                 f'ctx id: {ctx.cid}'
             )
+            drained.append(re)
+            self._closed = re
 
-        self._closed = True
+        # if caught_eoc:
+        #     # from .devx import _debug
+        #     # await _debug.pause()
+        #     with trio.CancelScope(shield=True):
+        #         await rx_chan.aclose()
 
-        # Do we close the local mem chan ``self._rx_chan`` ??!?
+        if not self._eoc:
-
+            log.cancel(
-        # NO, DEFINITELY NOT if we're a bi-dir ``MsgStream``!
+                'Stream closed before it received an EoC?\n'
-        # BECAUSE this same core-msg-loop mem recv-chan is used to deliver
+                'Setting eoc manually..\n..'
-        # the potential final result from the surrounding inter-actor
+            )
-        # `Context` so we don't want to close it until that context has
+            self._eoc: bool = trio.EndOfChannel(
-        # run to completion.
+                f'Context stream closed by {self._ctx.side}\n'
+                f'|_{self}\n'
+            )
+        # ?XXX WAIT, why do we not close the local mem chan `._rx_chan` XXX?
+        # => NO, DEFINITELY NOT! <=
+        # if we're a bi-dir ``MsgStream`` BECAUSE this same
+        # core-msg-loop mem recv-chan is used to deliver the
+        # potential final result from the surrounding inter-actor
+        # `Context` so we don't want to close it until that
+        # context has run to completion.
 
         # XXX: Notes on old behaviour:
         # await rx_chan.aclose()
@@ -272,6 +420,26 @@ class MsgStream(trio.abc.Channel):
         # runtime's closure of ``rx_chan`` in the case where we may
         # still need to consume msgs that are "in transit" from the far
         # end (eg. for ``Context.result()``).
+        # self._closed = True
+        return drained
+
+    @property
+    def closed(self) -> bool:
+
+        rxc: bool = self._rx_chan._closed
+        _closed: bool|Exception = self._closed
+        _eoc: bool|trio.EndOfChannel = self._eoc
+        if rxc or _closed or _eoc:
+            log.runtime(
+                f'`MsgStream` is already closed\n'
+                f'{self}\n'
+                f' |_cid: {self._ctx.cid}\n'
+                f' |_rx_chan._closed: {type(rxc)} = {rxc}\n'
+                f' |_closed: {type(_closed)} = {_closed}\n'
+                f' |_eoc: {type(_eoc)} = {_eoc}'
+            )
+            return True
+        return False
 
     @acm
     async def subscribe(
@@ -327,19 +495,43 @@ class MsgStream(trio.abc.Channel):
 
     async def send(
         self,
-        data: Any
+        data: Any,
+
+        hide_tb: bool = True,
     ) -> None:
         '''
         Send a message over this stream to the far end.
 
         '''
-        if self._ctx._remote_error:
+        __tracebackhide__: bool = hide_tb
-            raise self._ctx._remote_error  # from None
+
+        # raise any alreay known error immediately
+        self._ctx.maybe_raise()
+        if self._eoc:
+            raise self._eoc
 
         if self._closed:
-            raise trio.ClosedResourceError('This stream was already closed')
+            raise self._closed
 
-        await self._ctx.chan.send({'yield': data, 'cid': self._ctx.cid})
+        try:
+            await self._ctx.chan.send(
+                payload={
+                    'yield': data,
+                    'cid': self._ctx.cid,
+                },
+                # hide_tb=hide_tb,
+            )
+        except (
+            trio.ClosedResourceError,
+            trio.BrokenResourceError,
+            BrokenPipeError,
+        ) as trans_err:
+            if hide_tb:
+                raise type(trans_err)(
+                    *trans_err.args
+                ) from trans_err
+            else:
+                raise
 
 
 def stream(func: Callable) -> Callable:

tractor/_supervise.py

@@ -21,19 +21,22 @@
 from contextlib import asynccontextmanager as acm
 from functools import partial
 import inspect
+from pprint import pformat
 from typing import TYPE_CHECKING
 import typing
 import warnings
 
-from exceptiongroup import BaseExceptionGroup
 import trio
 
-from .devx._debug import maybe_wait_for_debugger
+from ._debug import maybe_wait_for_debugger
 from ._state import current_actor, is_main_process
 from .log import get_logger, get_loglevel
 from ._runtime import Actor
 from ._portal import Portal
-from ._exceptions import is_multi_cancelled
+from ._exceptions import (
+    is_multi_cancelled,
+    ContextCancelled,
+)
 from ._root import open_root_actor
 from . import _state
 from . import _spawn
@@ -103,6 +106,14 @@ class ActorNursery:
         self.errors = errors
         self.exited = trio.Event()
 
+        # NOTE: when no explicit call is made to
+        # `.open_root_actor()` by application code,
+        # `.open_nursery()` will implicitly call it to start the
+        # actor-tree runtime. In this case we mark ourselves as
+        # such so that runtime components can be aware for logging
+        # and syncing purposes to any actor opened nurseries.
+        self._implicit_runtime_started: bool = False
+
     async def start_actor(
         self,
         name: str,
@@ -156,7 +167,7 @@ class ActorNursery:
 
         # start a task to spawn a process
         # blocks until process has been started and a portal setup
-        nursery = nursery or self._da_nursery
+        nursery: trio.Nursery = nursery or self._da_nursery
 
         # XXX: the type ignore is actually due to a `mypy` bug
         return await nursery.start(  # type: ignore
@@ -189,14 +200,16 @@ class ActorNursery:
         **kwargs,  # explicit args to ``fn``
 
     ) -> Portal:
-        """Spawn a new actor, run a lone task, then terminate the actor and
+        '''
+        Spawn a new actor, run a lone task, then terminate the actor and
         return its result.
 
         Actors spawned using this method are kept alive at nursery teardown
         until the task spawned by executing ``fn`` completes at which point
         the actor is terminated.
-        """
+
-        mod_path = fn.__module__
+        '''
+        mod_path: str = fn.__module__
 
         if name is None:
             # use the explicit function name if not provided
@@ -231,21 +244,37 @@ class ActorNursery:
         )
         return portal
 
-    async def cancel(self, hard_kill: bool = False) -> None:
+    async def cancel(
-        """Cancel this nursery by instructing each subactor to cancel
+        self,
+        hard_kill: bool = False,
+
+    ) -> None:
+        '''
+        Cancel this nursery by instructing each subactor to cancel
         itself and wait for all subactors to terminate.
 
         If ``hard_killl`` is set to ``True`` then kill the processes
         directly without any far end graceful ``trio`` cancellation.
-        """
+
+        '''
         self.cancelled = True
 
-        log.cancel(f"Cancelling nursery in {self._actor.uid}")
+        # TODO: impl a repr for spawn more compact
+        # then `._children`..
+        children: dict = self._children
+        child_count: int = len(children)
+        msg: str = f'Cancelling actor nursery with {child_count} children\n'
         with trio.move_on_after(3) as cs:
+            async with trio.open_nursery() as tn:
 
-            async with trio.open_nursery() as nursery:
+                subactor: Actor
-
+                proc: trio.Process
-                for subactor, proc, portal in self._children.values():
+                portal: Portal
+                for (
+                    subactor,
+                    proc,
+                    portal,
+                ) in children.values():
 
                     # TODO: are we ever even going to use this or
                     # is the spawning backend responsible for such
@@ -257,12 +286,13 @@ class ActorNursery:
                         if portal is None:  # actor hasn't fully spawned yet
                             event = self._actor._peer_connected[subactor.uid]
                             log.warning(
-                                f"{subactor.uid} wasn't finished spawning?")
+                                f"{subactor.uid} never 't finished spawning?"
+                            )
 
                             await event.wait()
 
                             # channel/portal should now be up
-                            _, _, portal = self._children[subactor.uid]
+                            _, _, portal = children[subactor.uid]
 
                             # XXX should be impossible to get here
                             # unless method was called from within
@@ -279,14 +309,24 @@ class ActorNursery:
                         # spawn cancel tasks for each sub-actor
                         assert portal
                         if portal.channel.connected():
-                            nursery.start_soon(portal.cancel_actor)
+                            tn.start_soon(portal.cancel_actor)
 
+                log.cancel(msg)
         # if we cancelled the cancel (we hung cancelling remote actors)
         # then hard kill all sub-processes
         if cs.cancelled_caught:
             log.error(
-                f"Failed to cancel {self}\nHard killing process tree!")
+                f'Failed to cancel {self}?\n'
-            for subactor, proc, portal in self._children.values():
+                'Hard killing underlying subprocess tree!\n'
+            )
+            subactor: Actor
+            proc: trio.Process
+            portal: Portal
+            for (
+                subactor,
+                proc,
+                portal,
+            ) in children.values():
                 log.warning(f"Hard killing process {proc}")
                 proc.terminate()
 
@@ -326,7 +366,7 @@ async def _open_and_supervise_one_cancels_all_nursery(
             # the above "daemon actor" nursery will be notified.
             async with trio.open_nursery() as ria_nursery:
 
-                anursery = ActorNursery(
+                an = ActorNursery(
                     actor,
                     ria_nursery,
                     da_nursery,
@@ -335,16 +375,16 @@ async def _open_and_supervise_one_cancels_all_nursery(
                 try:
                     # spawning of actors happens in the caller's scope
                     # after we yield upwards
-                    yield anursery
+                    yield an
 
                     # When we didn't error in the caller's scope,
                     # signal all process-monitor-tasks to conduct
                     # the "hard join phase".
                     log.runtime(
-                        f"Waiting on subactors {anursery._children} "
+                        'Waiting on subactors to complete:\n'
-                        "to complete"
+                        f'{pformat(an._children)}\n'
                     )
-                    anursery._join_procs.set()
+                    an._join_procs.set()
 
                 except BaseException as inner_err:
                     errors[actor.uid] = inner_err
@@ -356,37 +396,60 @@ async def _open_and_supervise_one_cancels_all_nursery(
                     # Instead try to wait for pdb to be released before
                     # tearing down.
                     await maybe_wait_for_debugger(
-                        child_in_debug=anursery._at_least_one_child_in_debug
+                        child_in_debug=an._at_least_one_child_in_debug
                     )
 
                     # if the caller's scope errored then we activate our
                     # one-cancels-all supervisor strategy (don't
                     # worry more are coming).
-                    anursery._join_procs.set()
+                    an._join_procs.set()
 
-                    # XXX: hypothetically an error could be
+                    # XXX NOTE XXX: hypothetically an error could
-                    # raised and then a cancel signal shows up
+                    # be raised and then a cancel signal shows up
                     # slightly after in which case the `else:`
                     # block here might not complete?  For now,
                     # shield both.
                     with trio.CancelScope(shield=True):
-                        etype = type(inner_err)
+                        etype: type = type(inner_err)
                         if etype in (
                             trio.Cancelled,
-                            KeyboardInterrupt
+                            KeyboardInterrupt,
                         ) or (
                             is_multi_cancelled(inner_err)
                         ):
                             log.cancel(
-                                f"Nursery for {current_actor().uid} "
+                                f'Actor-nursery cancelled by {etype}\n\n'
-                                f"was cancelled with {etype}")
+
+                                f'{current_actor().uid}\n'
+                                f' |_{an}\n\n'
+
+                                # TODO: show tb str?
+                                # f'{tb_str}'
+                            )
+                        elif etype in {
+                            ContextCancelled,
+                        }:
+                            log.cancel(
+                                'Actor-nursery caught remote cancellation\n\n'
+
+                                f'{inner_err.tb_str}'
+                            )
                         else:
                             log.exception(
-                                f"Nursery for {current_actor().uid} "
+                                'Nursery errored with:\n'
-                                f"errored with")
+
+                                # TODO: same thing as in
+                                # `._invoke()` to compute how to
+                                # place this div-line in the
+                                # middle of the above msg
+                                # content..
+                                # -[ ] prolly helper-func it too
+                                #   in our `.log` module..
+                                # '------ - ------'
+                            )
 
                         # cancel all subactors
-                        await anursery.cancel()
+                        await an.cancel()
 
             # ria_nursery scope end
 
@@ -407,18 +470,22 @@ async def _open_and_supervise_one_cancels_all_nursery(
             # XXX: yet another guard before allowing the cancel
             # sequence in case a (single) child is in debug.
             await maybe_wait_for_debugger(
-                child_in_debug=anursery._at_least_one_child_in_debug
+                child_in_debug=an._at_least_one_child_in_debug
             )
 
             # If actor-local error was raised while waiting on
             # ".run_in_actor()" actors then we also want to cancel all
             # remaining sub-actors (due to our lone strategy:
             # one-cancels-all).
-            log.cancel(f"Nursery cancelling due to {err}")
+            if an._children:
-            if anursery._children:
+                log.cancel(
+                    'Actor-nursery cancelling due error type:\n'
+                    f'{err}\n'
+                )
                 with trio.CancelScope(shield=True):
-                    await anursery.cancel()
+                    await an.cancel()
             raise
+
         finally:
             # No errors were raised while awaiting ".run_in_actor()"
             # actors but those actors may have returned remote errors as
@@ -427,9 +494,9 @@ async def _open_and_supervise_one_cancels_all_nursery(
             # collected in ``errors`` so cancel all actors, summarize
             # all errors and re-raise.
             if errors:
-                if anursery._children:
+                if an._children:
                     with trio.CancelScope(shield=True):
-                        await anursery.cancel()
+                        await an.cancel()
 
                 # use `BaseExceptionGroup` as needed
                 if len(errors) > 1:
@@ -464,19 +531,20 @@ async def open_nursery(
     which cancellation scopes correspond to each spawned subactor set.
 
     '''
-    implicit_runtime = False
+    implicit_runtime: bool = False
-
+    actor: Actor = current_actor(err_on_no_runtime=False)
-    actor = current_actor(err_on_no_runtime=False)
+    an: ActorNursery|None = None
-
     try:
-        if actor is None and is_main_process():
+        if (
-
+            actor is None
+            and is_main_process()
+        ):
             # if we are the parent process start the
             # actor runtime implicitly
             log.info("Starting actor runtime!")
 
             # mark us for teardown on exit
-            implicit_runtime = True
+            implicit_runtime: bool = True
 
             async with open_root_actor(**kwargs) as actor:
                 assert actor is current_actor()
@@ -484,24 +552,42 @@ async def open_nursery(
                 try:
                     async with _open_and_supervise_one_cancels_all_nursery(
                         actor
-                    ) as anursery:
+                    ) as an:
-                        yield anursery
+
+                        # NOTE: mark this nursery as having
+                        # implicitly started the root actor so
+                        # that `._runtime` machinery can avoid
+                        # certain teardown synchronization
+                        # blocking/waits and any associated (warn)
+                        # logging when it's known that this
+                        # nursery shouldn't be exited before the
+                        # root actor is.
+                        an._implicit_runtime_started = True
+                        yield an
                 finally:
-                    anursery.exited.set()
+                    # XXX: this event will be set after the root actor
+                    # runtime is already torn down, so we want to
+                    # avoid any blocking on it.
+                    an.exited.set()
 
         else:  # sub-nursery case
 
             try:
                 async with _open_and_supervise_one_cancels_all_nursery(
                     actor
-                ) as anursery:
+                ) as an:
-                    yield anursery
+                    yield an
             finally:
-                anursery.exited.set()
+                an.exited.set()
 
     finally:
-        log.debug("Nursery teardown complete")
+        msg: str = (
+            'Actor-nursery exited\n'
+            f'|_{an}\n\n'
+        )
 
         # shutdown runtime if it was started
         if implicit_runtime:
-            log.info("Shutting down actor tree")
+            msg += '=> Shutting down actor runtime <=\n'
+
+        log.info(msg)

tractor/_testing/__init__.py

@@ -0,0 +1,74 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Various helpers/utils for auditing your `tractor` app and/or the
+core runtime.
+
+'''
+from contextlib import asynccontextmanager as acm
+import pathlib
+
+import tractor
+from .pytest import (
+    tractor_test as tractor_test
+)
+
+
+def repodir() -> pathlib.Path:
+    '''
+    Return the abspath to the repo directory.
+
+    '''
+    # 2 parents up to step up through tests/<repo_dir>
+    return pathlib.Path(
+        __file__
+
+    # 3 .parents bc:
+    # <._testing-pkg>.<tractor-pkg>.<git-repo-dir>
+    # /$HOME/../<tractor-repo-dir>/tractor/_testing/__init__.py
+    ).parent.parent.parent.absolute()
+
+
+def examples_dir() -> pathlib.Path:
+    '''
+    Return the abspath to the examples directory as `pathlib.Path`.
+
+    '''
+    return repodir() / 'examples'
+
+
+@acm
+async def expect_ctxc(
+    yay: bool,
+    reraise: bool = False,
+) -> None:
+    '''
+    Small acm to catch `ContextCancelled` errors when expected
+    below it in a `async with ()` block.
+
+    '''
+    if yay:
+        try:
+            yield
+            raise RuntimeError('Never raised ctxc?')
+        except tractor.ContextCancelled:
+            if reraise:
+                raise
+            else:
+                return
+    else:
+        yield

tractor/_testing/pytest.py

@@ -0,0 +1,113 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+`pytest` utils helpers and plugins for testing `tractor`'s runtime
+and applications.
+
+'''
+from functools import (
+    partial,
+    wraps,
+)
+import inspect
+import platform
+
+import tractor
+import trio
+
+
+def tractor_test(fn):
+    '''
+    Decorator for async test funcs to present them as "native"
+    looking sync funcs runnable by `pytest` using `trio.run()`.
+
+    Use:
+
+    @tractor_test
+    async def test_whatever():
+        await ...
+
+    If fixtures:
+
+        - ``reg_addr`` (a socket addr tuple where arbiter is listening)
+        - ``loglevel`` (logging level passed to tractor internals)
+        - ``start_method`` (subprocess spawning backend)
+
+    are defined in the `pytest` fixture space they will be automatically
+    injected to tests declaring these funcargs.
+    '''
+    @wraps(fn)
+    def wrapper(
+        *args,
+        loglevel=None,
+        reg_addr=None,
+        start_method: str|None = None,
+        debug_mode: bool = False,
+        **kwargs
+    ):
+        # __tracebackhide__ = True
+
+        # NOTE: inject ant test func declared fixture
+        # names by manually checking!
+        if 'reg_addr' in inspect.signature(fn).parameters:
+            # injects test suite fixture value to test as well
+            # as `run()`
+            kwargs['reg_addr'] = reg_addr
+
+        if 'loglevel' in inspect.signature(fn).parameters:
+            # allows test suites to define a 'loglevel' fixture
+            # that activates the internal logging
+            kwargs['loglevel'] = loglevel
+
+        if start_method is None:
+            if platform.system() == "Windows":
+                start_method = 'trio'
+
+        if 'start_method' in inspect.signature(fn).parameters:
+            # set of subprocess spawning backends
+            kwargs['start_method'] = start_method
+
+        if 'debug_mode' in inspect.signature(fn).parameters:
+            # set of subprocess spawning backends
+            kwargs['debug_mode'] = debug_mode
+
+
+        if kwargs:
+
+            # use explicit root actor start
+            async def _main():
+                async with tractor.open_root_actor(
+                    # **kwargs,
+                    registry_addrs=[reg_addr] if reg_addr else None,
+                    loglevel=loglevel,
+                    start_method=start_method,
+
+                    # TODO: only enable when pytest is passed --pdb
+                    debug_mode=debug_mode,
+
+                ):
+                    await fn(*args, **kwargs)
+
+            main = _main
+
+        else:
+            # use implicit root actor start
+            main = partial(fn, *args, **kwargs)
+
+        return trio.run(main)
+
+    return wrapper

tractor/devx/__init__.py

@@ -1,47 +0,0 @@
-# tractor: structured concurrent "actors".
-# Copyright 2018-eternity Tyler Goodlet.
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-"""
-Runtime "developer experience" utils and addons to aid our
-(advanced) users and core devs in building distributed applications
-and working with/on the actor runtime.
-
-"""
-from ._debug import (
-    maybe_wait_for_debugger,
-    acquire_debug_lock,
-    breakpoint,
-    pause,
-    pause_from_sync,
-    shield_sigint_handler,
-    MultiActorPdb,
-    open_crash_handler,
-    maybe_open_crash_handler,
-    post_mortem,
-)
-
-__all__ = [
-    'maybe_wait_for_debugger',
-    'acquire_debug_lock',
-    'breakpoint',
-    'pause',
-    'pause_from_sync',
-    'shield_sigint_handler',
-    'MultiActorPdb',
-    'open_crash_handler',
-    'maybe_open_crash_handler',
-    'post_mortem',
-]

tractor/devx/cli.py

@@ -1,136 +0,0 @@
-# tractor: structured concurrent "actors".
-# Copyright 2018-eternity Tyler Goodlet.
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-"""
-CLI framework extensions for hacking on the actor runtime.
-
-Currently popular frameworks supported are:
-
-  - `typer` via the `@callback` API
-
-"""
-from __future__ import annotations
-from contextlib import (
-    # asynccontextmanager as acm,
-    contextmanager as cm,
-)
-from typing import (
-    Any,
-    Callable,
-)
-from typing_extensions import Annotated
-
-import typer
-
-
-from ._debug import open_crash_handler
-
-
-_runtime_vars: dict[str, Any] = {}
-
-
-def load_runtime_vars(
-    ctx: typer.Context,
-    callback: Callable,
-    pdb: bool = False,  # --pdb
-    ll: Annotated[
-        str,
-        typer.Option(
-            '--loglevel',
-            '-l',
-            help='BigD logging level',
-        ),
-    ] = 'cancel',  # -l info
-):
-    '''
-    Maybe engage crash handling with `pdbp` when code inside
-    a `typer` CLI endpoint cmd raises.
-
-    To use this callback simply take your `app = typer.Typer()` instance
-    and decorate this function with it like so:
-
-    .. code:: python
-
-        from tractor.devx import cli
-
-        app = typer.Typer()
-
-        # manual decoration to hook into `click`'s context system!
-        cli.load_runtime_vars = app.callback(
-            invoke_without_command=True,
-        )
-
-    And then you can use the now augmented `click` CLI context as so,
-
-    .. code:: python
-
-        @app.command(
-            context_settings={
-                "allow_extra_args": True,
-                "ignore_unknown_options": True,
-            }
-        )
-        def my_cli_cmd(
-            ctx: typer.Context,
-        ):
-            rtvars: dict = ctx.runtime_vars
-            pdb: bool = rtvars['pdb']
-
-            with tractor.devx.cli.maybe_open_crash_handler(pdb=pdb):
-                trio.run(
-                    partial(
-                        my_tractor_main_task_func,
-                        debug_mode=pdb,
-                        loglevel=rtvars['ll'],
-                    )
-                )
-
-    which will enable log level and debug mode globally for the entire
-    `tractor` + `trio` runtime thereafter!
-
-    Bo
-
-    '''
-    global _runtime_vars
-    _runtime_vars |= {
-        'pdb': pdb,
-        'll': ll,
-    }
-
-    ctx.runtime_vars: dict[str, Any] = _runtime_vars
-    print(
-        f'`typer` sub-cmd: {ctx.invoked_subcommand}\n'
-        f'`tractor` runtime vars: {_runtime_vars}'
-    )
-
-    # XXX NOTE XXX: hackzone.. if no sub-cmd is specified (the
-    # default if the user just invokes `bigd`) then we simply
-    # invoke the sole `_bigd()` cmd passing in the "parent"
-    # typer.Context directly to that call since we're treating it
-    # as a "non sub-command" or wtv..
-    # TODO: ideally typer would have some kinda built-in way to get
-    # this behaviour without having to construct and manually
-    # invoke our own cmd..
-    if (
-        ctx.invoked_subcommand is None
-        or ctx.invoked_subcommand == callback.__name__
-    ):
-        cmd: typer.core.TyperCommand = typer.core.TyperCommand(
-            name='bigd',
-            callback=callback,
-        )
-        ctx.params = {'ctx': ctx}
-        cmd.invoke(ctx)

tractor/experimental/_pubsub.py

@@ -31,7 +31,7 @@ from typing import (
     Callable,
 )
 from functools import partial
-from async_generator import aclosing
+from contextlib import aclosing
 
 import trio
 import wrapt

tractor/log.py

@@ -289,11 +289,19 @@ def get_console_log(
     if not level:
         return log
 
-    log.setLevel(level.upper() if not isinstance(level, int) else level)
+    log.setLevel(
+        level.upper()
+        if not isinstance(level, int)
+        else level
+    )
 
     if not any(
         handler.stream == sys.stderr  # type: ignore
-        for handler in logger.handlers if getattr(handler, 'stream', None)
+        for handler in logger.handlers if getattr(
+            handler,
+            'stream',
+            None,
+        )
     ):
         handler = logging.StreamHandler()
         formatter = colorlog.ColoredFormatter(

tractor/msg/ptr.py

@@ -43,21 +43,33 @@ IPC-compat cross-mem-boundary object pointer.
 # - https://github.com/msgpack/msgpack-python#packingunpacking-of-custom-data-type
 
 from __future__ import annotations
-from inspect import isfunction
+from inspect import (
+    isfunction,
+    ismethod,
+)
 from pkgutil import resolve_name
 
 
 class NamespacePath(str):
     '''
-    A serializeable description of a (function) Python object
+    A serializeable `str`-subtype implementing a "namespace
-    location described by the target's module path and namespace
+    pointer" to any Python object reference (like a function)
-    key meant as a message-native "packet" to allows actors to
+    using the same format as the built-in `pkgutil.resolve_name()`
-    point-and-load objects by an absolute ``str`` (and thus
+    system.
-    serializable) reference.
+
+    A value describes a target's module-path and namespace-key
+    separated by a ':' and thus can be easily used as
+    a IPC-message-native reference-type allowing memory isolated
+    actors to point-and-load objects via a minimal `str` value.
 
     '''
     _ref: object | type | None = None
 
+    # TODO: support providing the ns instance in
+    # order to support 'self.<meth>` style to make
+    # `Portal.run_from_ns()` work!
+    # _ns: ModuleType|type|None = None
+
     def load_ref(self) -> object | type:
         if self._ref is None:
             self._ref = resolve_name(self)
@@ -76,13 +88,23 @@ class NamespacePath(str):
 
         '''
         if (
-            isinstance(ref, object)
+            isfunction(ref)
-            and not isfunction(ref)
         ):
-            name: str = type(ref).__name__
-        else:
             name: str = getattr(ref, '__name__')
 
+        elif ismethod(ref):
+            # build out the path manually i guess..?
+            # TODO: better way?
+            name: str = '.'.join([
+                type(ref.__self__).__name__,
+                ref.__func__.__name__,
+            ])
+
+        else:  # object or other?
+            # isinstance(ref, object)
+            # and not isfunction(ref)
+            name: str = type(ref).__name__
+
         # fully qualified namespace path, tuple.
         fqnp: tuple[str, str] = (
             ref.__module__,
@@ -100,5 +122,13 @@ class NamespacePath(str):
         fqnp: tuple[str, str] = cls._mk_fqnp(ref)
         return cls(':'.join(fqnp))
 
-    def to_tuple(self) -> tuple[str, str]:
+    def to_tuple(
-        return self._mk_fqnp(self.load_ref())
+        self,
+
+        # TODO: could this work re `self:<meth>` case from above?
+        # load_ref: bool = True,
+
+    ) -> tuple[str, str]:
+        return self._mk_fqnp(
+            self.load_ref()
+        )

tractor/msg/types.py

@@ -35,6 +35,24 @@ from msgspec import (
     structs,
 )
 
+# TODO: auto-gen type sig for input func both for
+# type-msgs and logging of RPC tasks?
+# taken and modified from:
+# https://stackoverflow.com/a/57110117
+# import inspect
+# from typing import List
+
+# def my_function(input_1: str, input_2: int) -> list[int]:
+#     pass
+
+# def types_of(func):
+#     specs = inspect.getfullargspec(func)
+#     return_type = specs.annotations['return']
+#     input_types = [t.__name__ for s, t in specs.annotations.items() if s != 'return']
+#     return f'{func.__name__}({": ".join(input_types)}) -> {return_type}'
+
+# types_of(my_function)
+
 
 class DiffDump(UserList):
     '''
@@ -161,6 +179,7 @@ class Struct(
                 # https://docs.python.org/3.11/library/pprint.html#pprint.saferepr
                 val_str: str = saferepr(v)
 
+            # TODO: LOLOL use `textwrap.indent()` instead dawwwwwg!
             obj_str += (field_ws + f'{k}: {typ_name} = {val_str},\n')
 
         return (

tractor/to_asyncio.py

@@ -216,7 +216,14 @@ def _run_asyncio_task(
         try:
             result = await coro
         except BaseException as aio_err:
-            log.exception('asyncio task errored')
+            if isinstance(aio_err, CancelledError):
+                log.runtime(
+                    '`asyncio` task was cancelled..\n'
+                )
+            else:
+                log.exception(
+                    '`asyncio` task errored\n'
+                )
             chan._aio_err = aio_err
             raise
 
@@ -271,12 +278,22 @@ def _run_asyncio_task(
         except BaseException as terr:
             task_err = terr
 
+            msg: str = (
+                'Infected `asyncio` task {etype_str}\n'
+                f'|_{task}\n'
+            )
             if isinstance(terr, CancelledError):
-                log.cancel(f'`asyncio` task cancelled: {task.get_name()}')
+                log.cancel(
+                    msg.format(etype_str='cancelled')
+                )
             else:
-                log.exception(f'`asyncio` task: {task.get_name()} errored')
+                log.exception(
+                    msg.format(etype_str='cancelled')
+                )
 
-            assert type(terr) is type(aio_err), 'Asyncio task error mismatch?'
+            assert type(terr) is type(aio_err), (
+                '`asyncio` task error mismatch?!?'
+            )
 
         if aio_err is not None:
             # XXX: uhh is this true?
@@ -289,18 +306,22 @@ def _run_asyncio_task(
             # We might want to change this in the future though.
             from_aio.close()
 
-            if type(aio_err) is CancelledError:
+            if task_err is None:
-                log.cancel("infected task was cancelled")
-
-                # TODO: show that the cancellation originated
-                # from the ``trio`` side? right?
-                # if cancel_scope.cancelled:
-                #     raise aio_err from err
-
-            elif task_err is None:
                 assert aio_err
                 aio_err.with_traceback(aio_err.__traceback__)
-                log.error('infected task errorred')
+                # log.error(
+                #     'infected task errorred'
+                # )
+
+            # TODO: show that the cancellation originated
+            # from the ``trio`` side? right?
+            # elif type(aio_err) is CancelledError:
+            #     log.cancel(
+            #         'infected task was cancelled'
+            #     )
+
+                # if cancel_scope.cancelled:
+                #     raise aio_err from err
 
             # XXX: alway cancel the scope on error
             # in case the trio task is blocking

tractor/trionics/_broadcast.py

@@ -26,7 +26,6 @@ from contextlib import asynccontextmanager
 from functools import partial
 from operator import ne
 from typing import (
-    Optional,
     Callable,
     Awaitable,
     Any,
@@ -45,6 +44,11 @@ from tractor.log import get_logger
 
 log = get_logger(__name__)
 
+# TODO: use new type-vars syntax from 3.12
+# https://realpython.com/python312-new-features/#dedicated-type-variable-syntax
+# https://docs.python.org/3/whatsnew/3.12.html#whatsnew312-pep695
+# https://docs.python.org/3/reference/simple_stmts.html#type
+#
 # A regular invariant generic type
 T = TypeVar("T")
 
@@ -110,7 +114,7 @@ class BroadcastState(Struct):
 
     # broadcast event to wake up all sleeping consumer tasks
     # on a newly produced value from the sender.
-    recv_ready: Optional[tuple[int, trio.Event]] = None
+    recv_ready: tuple[int, trio.Event]|None = None
 
     # if a ``trio.EndOfChannel`` is received on any
     # consumer all consumers should be placed in this state
@@ -164,7 +168,7 @@ class BroadcastReceiver(ReceiveChannel):
 
         rx_chan: AsyncReceiver,
         state: BroadcastState,
-        receive_afunc: Optional[Callable[[], Awaitable[Any]]] = None,
+        receive_afunc: Callable[[], Awaitable[Any]]|None = None,
         raise_on_lag: bool = True,
 
     ) -> None:
@@ -452,7 +456,7 @@ def broadcast_receiver(
 
     recv_chan: AsyncReceiver,
     max_buffer_size: int,
-    receive_afunc: Optional[Callable[[], Awaitable[Any]]] = None,
+    receive_afunc: Callable[[], Awaitable[Any]]|None = None,
     raise_on_lag: bool = True,
 
 ) -> BroadcastReceiver:

tractor/trionics/_mngrs.py

@@ -33,10 +33,9 @@ from typing import (
 )
 
 import trio
-from trio_typing import TaskStatus
 
-from .._state import current_actor
+from tractor._state import current_actor
-from ..log import get_logger
+from tractor.log import get_logger
 
 
 log = get_logger(__name__)
@@ -184,7 +183,7 @@ class _Cache:
         cls,
         mng,
         ctx_key: tuple,
-        task_status: TaskStatus[T] = trio.TASK_STATUS_IGNORED,
+        task_status: trio.TaskStatus[T] = trio.TASK_STATUS_IGNORED,
 
     ) -> None:
         async with mng as value: