Add `debug_mode: bool` control to task mngr

Allows dynamically importing `pdbp` when enabled and a way for
eventually linking with `tractor`'s own debug mode flag.

examples/advanced_faults/ipc_failure_during_stream.py

@@ -62,7 +62,9 @@ async def recv_and_spawn_net_killers(
     await ctx.started()
     async with (
         ctx.open_stream() as stream,
-        trio.open_nursery() as n,
+        trio.open_nursery(
+            strict_exception_groups=False,
+        ) as tn,
     ):
         async for i in stream:
             print(f'child echoing {i}')
@@ -77,11 +79,11 @@ async def recv_and_spawn_net_killers(
                 i >= break_ipc_after
             ):
                 broke_ipc = True
-                n.start_soon(
+                tn.start_soon(
                     iter_ipc_stream,
                     stream,
                 )
-                n.start_soon(
+                tn.start_soon(
                     partial(
                         break_ipc_then_error,
                         stream=stream,

examples/debugging/asyncio_bp.py

@@ -1,8 +1,16 @@
+'''
+Examples of using the builtin `breakpoint()` from an `asyncio.Task`
+running in a subactor spawned with `infect_asyncio=True`.
+
+'''
 import asyncio
 
 import trio
 import tractor
-from tractor import to_asyncio
+from tractor import (
+    to_asyncio,
+    Portal,
+)
 
 
 async def aio_sleep_forever():
@@ -17,21 +25,21 @@ async def bp_then_error(
 
 ) -> None:
 
-    # sync with ``trio``-side (caller) task
+    # sync with `trio`-side (caller) task
     to_trio.send_nowait('start')
 
     # NOTE: what happens here inside the hook needs some refinement..
     # => seems like it's still `._debug._set_trace()` but
     #    we set `Lock.local_task_in_debug = 'sync'`, we probably want
-    #    some further, at least, meta-data about the task/actoq in debug
-    #    in terms of making it clear it's asyncio mucking about.
-    breakpoint()
+    #    some further, at least, meta-data about the task/actor in debug
+    #    in terms of making it clear it's `asyncio` mucking about.
+    breakpoint()  # asyncio-side
 
     # short checkpoint / delay
-    await asyncio.sleep(0.5)
+    await asyncio.sleep(0.5)  # asyncio-side
 
     if raise_after_bp:
-        raise ValueError('blah')
+        raise ValueError('asyncio side error!')
 
     # TODO: test case with this so that it gets cancelled?
     else:
@@ -49,23 +57,21 @@ async def trio_ctx(
     # this will block until the ``asyncio`` task sends a "first"
     # message, see first line in above func.
     async with (
-
         to_asyncio.open_channel_from(
             bp_then_error,
-            raise_after_bp=not bp_before_started,
+            # raise_after_bp=not bp_before_started,
         ) as (first, chan),
 
-        trio.open_nursery() as n,
+        trio.open_nursery() as tn,
     ):
-
         assert first == 'start'
 
         if bp_before_started:
-            await tractor.breakpoint()
+            await tractor.pause()  # trio-side
 
-        await ctx.started(first)
+        await ctx.started(first)  # trio-side
 
-        n.start_soon(
+        tn.start_soon(
             to_asyncio.run_task,
             aio_sleep_forever,
         )
@@ -73,39 +79,50 @@ async def trio_ctx(
 
 
 async def main(
-    bps_all_over: bool = False,
+    bps_all_over: bool = True,
+
+    # TODO, WHICH OF THESE HAZ BUGZ?
+    cancel_from_root: bool = False,
+    err_from_root: bool = False,
 
 ) -> None:
 
     async with tractor.open_nursery(
-        # debug_mode=True,
-    ) as n:
-
-        p = await n.start_actor(
+        debug_mode=True,
+        maybe_enable_greenback=True,
+        # loglevel='devx',
+    ) as an:
+        ptl: Portal = await an.start_actor(
             'aio_daemon',
             enable_modules=[__name__],
             infect_asyncio=True,
             debug_mode=True,
-            loglevel='cancel',
+            # loglevel='cancel',
         )
 
-        async with p.open_context(
+        async with ptl.open_context(
             trio_ctx,
             bp_before_started=bps_all_over,
         ) as (ctx, first):
 
             assert first == 'start'
 
-            if bps_all_over:
-                await tractor.breakpoint()
+            # pause in parent to ensure no cross-actor
+            # locking problems exist!
+            await tractor.pause()  # trio-root
+
+            if cancel_from_root:
+                await ctx.cancel()
+
+            if err_from_root:
+                assert 0
+            else:
+                await trio.sleep_forever()
 
-            # await trio.sleep_forever()
-            await ctx.cancel()
-            assert 0
 
         # TODO: case where we cancel from trio-side while asyncio task
         # has debugger lock?
-        # await p.cancel_actor()
+        # await ptl.cancel_actor()
 
 
 if __name__ == '__main__':

examples/debugging/fast_error_in_root_after_spawn.py

@@ -1,5 +1,5 @@
 '''
-Fast fail test with a context.
+Fast fail test with a `Context`.
 
 Ensure the partially initialized sub-actor process
 doesn't cause a hang on error/cancel of the parent

examples/debugging/multi_daemon_subactors.py

@@ -7,7 +7,7 @@ async def breakpoint_forever():
     try:
         while True:
             yield 'yo'
-            await tractor.breakpoint()
+            await tractor.pause()
     except BaseException:
         tractor.log.get_console_log().exception(
             'Cancelled while trying to enter pause point!'
@@ -21,11 +21,14 @@ async def name_error():
 
 
 async def main():
-    """Test breakpoint in a streaming actor.
-    """
+    '''
+    Test breakpoint in a streaming actor.
+
+    '''
     async with tractor.open_nursery(
         debug_mode=True,
         loglevel='cancel',
+        # loglevel='devx',
     ) as n:
 
         p0 = await n.start_actor('bp_forever', enable_modules=[__name__])

examples/debugging/multi_nested_subactors_error_up_through_nurseries.py

@@ -10,7 +10,7 @@ async def name_error():
 async def breakpoint_forever():
     "Indefinitely re-enter debugger in child actor."
     while True:
-        await tractor.breakpoint()
+        await tractor.pause()
 
         # NOTE: if the test never sent 'q'/'quit' commands
         # on the pdb repl, without this checkpoint line the

examples/debugging/multi_subactor_root_errors.py

@@ -40,7 +40,7 @@ async def main():
     """
     async with tractor.open_nursery(
         debug_mode=True,
-        # loglevel='cancel',
+        loglevel='devx',
     ) as n:
 
         # spawn both actors

examples/debugging/multi_subactors.py

@@ -6,7 +6,7 @@ async def breakpoint_forever():
     "Indefinitely re-enter debugger in child actor."
     while True:
         await trio.sleep(0.1)
-        await tractor.breakpoint()
+        await tractor.pause()
 
 
 async def name_error():

examples/debugging/restore_builtin_breakpoint.py

@@ -6,19 +6,44 @@ import tractor
 
 
 async def main() -> None:
-    async with tractor.open_nursery(debug_mode=True) as an:
 
-        assert os.environ['PYTHONBREAKPOINT'] == 'tractor._debug._set_trace'
+    # intially unset, no entry.
+    orig_pybp_var: int = os.environ.get('PYTHONBREAKPOINT')
+    assert orig_pybp_var in {None, "0"}
+
+    async with tractor.open_nursery(
+        debug_mode=True,
+    ) as an:
+        assert an
+        assert (
+            (pybp_var := os.environ['PYTHONBREAKPOINT'])
+            ==
+            'tractor.devx._debug._sync_pause_from_builtin'
+        )
 
         # TODO: an assert that verifies the hook has indeed been, hooked
         # XD
-        assert sys.breakpointhook is not tractor._debug._set_trace
+        assert (
+            (pybp_hook := sys.breakpointhook)
+            is not tractor.devx._debug._set_trace
+        )
 
-        breakpoint()
+        print(
+            f'$PYTHONOBREAKPOINT: {pybp_var!r}\n'
+            f'`sys.breakpointhook`: {pybp_hook!r}\n'
+        )
+        breakpoint()  # first bp, tractor hook set.
 
-    # TODO: an assert that verifies the hook is unhooked..
+    # XXX AFTER EXIT (of actor-runtime) verify the hook is unset..
+    #
+    # YES, this is weird but it's how stdlib docs say to do it..
+    # https://docs.python.org/3/library/sys.html#sys.breakpointhook
+    assert os.environ.get('PYTHONBREAKPOINT') is orig_pybp_var
     assert sys.breakpointhook
-    breakpoint()
+
+    # now ensure a regular builtin pause still works
+    breakpoint()  # last bp, stdlib hook restored
+
 
 if __name__ == '__main__':
     trio.run(main)

examples/debugging/root_actor_breakpoint.py

@@ -10,7 +10,7 @@ async def main():
 
         await trio.sleep(0.1)
 
-        await tractor.breakpoint()
+        await tractor.pause()
 
         await trio.sleep(0.1)
 

examples/debugging/root_actor_breakpoint_forever.py

@@ -11,7 +11,7 @@ async def main(
         # loglevel='runtime',
     ):
         while True:
-            await tractor.breakpoint()
+            await tractor.pause()
 
 
 if __name__ == '__main__':

examples/debugging/shield_hang_in_sub.py

@@ -0,0 +1,83 @@
+'''
+Verify we can dump a `stackscope` tree on a hang.
+
+'''
+import os
+import signal
+
+import trio
+import tractor
+
+@tractor.context
+async def start_n_shield_hang(
+    ctx: tractor.Context,
+):
+    # actor: tractor.Actor = tractor.current_actor()
+
+    # sync to parent-side task
+    await ctx.started(os.getpid())
+
+    print('Entering shield sleep..')
+    with trio.CancelScope(shield=True):
+        await trio.sleep_forever()  # in subactor
+
+    # XXX NOTE ^^^ since this shields, we expect
+    # the zombie reaper (aka T800) to engage on
+    # SIGINT from the user and eventually hard-kill
+    # this subprocess!
+
+
+async def main(
+    from_test: bool = False,
+) -> None:
+
+    async with (
+        tractor.open_nursery(
+            debug_mode=True,
+            enable_stack_on_sig=True,
+            # maybe_enable_greenback=False,
+            loglevel='devx',
+        ) as an,
+    ):
+        ptl: tractor.Portal  = await an.start_actor(
+            'hanger',
+            enable_modules=[__name__],
+            debug_mode=True,
+        )
+        async with ptl.open_context(
+            start_n_shield_hang,
+        ) as (ctx, cpid):
+
+            _, proc, _ = an._children[ptl.chan.uid]
+            assert cpid == proc.pid
+
+            print(
+                'Yo my child hanging..?\n'
+                # "i'm a user who wants to see a `stackscope` tree!\n"
+            )
+
+            # XXX simulate the wrapping test's "user actions"
+            # (i.e. if a human didn't run this manually but wants to
+            # know what they should do to reproduce test behaviour)
+            if from_test:
+                print(
+                    f'Sending SIGUSR1 to {cpid!r}!\n'
+                )
+                os.kill(
+                    cpid,
+                    signal.SIGUSR1,
+                )
+
+                # simulate user cancelling program
+                await trio.sleep(0.5)
+                os.kill(
+                    os.getpid(),
+                    signal.SIGINT,
+                )
+            else:
+                # actually let user send the ctl-c
+                await trio.sleep_forever()  # in root
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/subactor_bp_in_ctx.py

@@ -4,9 +4,9 @@ import trio
 
 async def gen():
     yield 'yo'
-    await tractor.breakpoint()
+    await tractor.pause()
     yield 'yo'
-    await tractor.breakpoint()
+    await tractor.pause()
 
 
 @tractor.context
@@ -15,7 +15,7 @@ async def just_bp(
 ) -> None:
 
     await ctx.started()
-    await tractor.breakpoint()
+    await tractor.pause()
 
     # TODO: bps and errors in this call..
     async for val in gen():

examples/debugging/sync_bp.py

@@ -4,6 +4,13 @@ import time
 import trio
 import tractor
 
+# TODO: only import these when not running from test harness?
+# can we detect `pexpect` usage maybe?
+# from tractor.devx._debug import (
+#     get_lock,
+#     get_debug_req,
+# )
+
 
 def sync_pause(
     use_builtin: bool = False,
@@ -18,7 +25,13 @@ def sync_pause(
         breakpoint(hide_tb=hide_tb)
 
     else:
+        # TODO: maybe for testing some kind of cm style interface
+        # where the `._set_trace()` call doesn't happen until block
+        # exit?
+        # assert get_lock().ctx_in_debug is None
+        # assert get_debug_req().repl is None
         tractor.pause_from_sync()
+        # assert get_debug_req().repl is None
 
     if error:
         raise RuntimeError('yoyo sync code error')
@@ -41,10 +54,11 @@ async def start_n_sync_pause(
 async def main() -> None:
     async with (
         tractor.open_nursery(
-            # NOTE: required for pausing from sync funcs
-            maybe_enable_greenback=True,
             debug_mode=True,
-            # loglevel='cancel',
+            maybe_enable_greenback=True,
+            enable_stack_on_sig=True,
+            # loglevel='warning',
+            # loglevel='devx',
         ) as an,
         trio.open_nursery() as tn,
     ):
@@ -138,7 +152,9 @@ async def main() -> None:
                     # the case 2. from above still exists!
                     use_builtin=True,
                 ),
-                abandon_on_cancel=False,
+                # TODO: with this `False` we can hang!??!
+                # abandon_on_cancel=False,
+                abandon_on_cancel=True,
                 thread_name='inline_root_bg_thread',
             )
 

examples/full_fledged_streaming_service.py

@@ -91,7 +91,7 @@ async def main() -> list[int]:
     an: ActorNursery
     async with tractor.open_nursery(
         loglevel='cancel',
-        debug_mode=True,
+        # debug_mode=True,
     ) as an:
 
         seed = int(1e3)

examples/quick_cluster.py

@@ -3,20 +3,18 @@ import trio
 import tractor
 
 
-async def sleepy_jane():
-    uid = tractor.current_actor().uid
+async def sleepy_jane() -> None:
+    uid: tuple = tractor.current_actor().uid
     print(f'Yo i am actor {uid}')
     await trio.sleep_forever()
 
 
 async def main():
     '''
-    Spawn a flat actor cluster, with one process per
-    detected core.
+    Spawn a flat actor cluster, with one process per detected core.
 
     '''
     portal_map: dict[str, tractor.Portal]
-    results: dict[str, str]
 
     # look at this hip new syntax!
     async with (
@@ -25,11 +23,16 @@ async def main():
             modules=[__name__]
         ) as portal_map,
 
-        trio.open_nursery() as n,
+        trio.open_nursery(
+            strict_exception_groups=False,
+        ) as tn,
     ):
 
         for (name, portal) in portal_map.items():
-            n.start_soon(portal.run, sleepy_jane)
+            tn.start_soon(
+                portal.run,
+                sleepy_jane,
+            )
 
         await trio.sleep(0.5)
 
@@ -41,4 +44,4 @@ if __name__ == '__main__':
     try:
         trio.run(main)
     except KeyboardInterrupt:
-        pass
+        print('trio cancelled by KBI')

notes_to_self/howtorelease.md

@@ -0,0 +1,18 @@
+First generate a built disti:
+
+```
+python -m pip install --upgrade build
+python -m build --sdist --outdir dist/alpha5/
+```
+
+Then try a test ``pypi`` upload:
+
+```
+python -m twine upload --repository testpypi dist/alpha5/*
+```
+
+The push to `pypi` for realz.
+
+```
+python -m twine upload --repository testpypi dist/alpha5/*
+```

pyproject.toml

@@ -37,16 +37,14 @@ dependencies = [
   # https://packaging.python.org/en/latest/discussions/install-requires-vs-requirements/#id5
   # TODO, for 3.13 we must go go `0.27` which means we have to
   # disable strict egs or port to handling them internally!
-  # trio='^0.27'
-  "trio>=0.24,<0.25",
+  "trio>0.27",
   "tricycle>=0.4.1,<0.5",
   "wrapt>=1.16.0,<2",
   "colorlog>=6.8.2,<7",
   # built-in multi-actor `pdb` REPL
-  "pdbp>=1.5.0,<2",
+  "pdbp>=1.6,<2", # windows only (from `pdbp`)
   # typed IPC msging
-  # TODO, get back on release once 3.13 support is out!
-  "msgspec",
+  "msgspec>=0.19.0",
 ]
 
 # ------ project ------
@@ -56,18 +54,14 @@ dev = [
   # test suite
   # TODO: maybe some of these layout choices?
   # https://docs.pytest.org/en/8.0.x/explanation/goodpractices.html#choosing-a-test-layout-import-rules
-  "pytest>=8.2.0,<9",
+  "pytest>=8.3.5",
   "pexpect>=4.9.0,<5",
   # `tractor.devx` tooling
   "greenback>=1.2.1,<2",
   "stackscope>=0.2.2,<0.3",
-
-  # xonsh usage/integration (namely as @goodboy's sh of choice Bp)
-  "xonsh>=0.19.1",
-  "xontrib-vox>=0.0.1,<0.0.2",
-  "prompt-toolkit>=3.0.43,<4",
-  "xonsh-vox-tabcomplete>=0.5,<0.6",
   "pyperclip>=1.9.0",
+  "prompt-toolkit>=3.0.50",
+  "xonsh>=0.19.2",
 ]
 # TODO, add these with sane versions; were originally in
 # `requirements-docs.txt`..
@@ -78,21 +72,39 @@ dev = [
 
 # ------ dependency-groups ------
 
+# ------ dependency-groups ------
+
 [tool.uv.sources]
-msgspec = { git = "https://github.com/jcrist/msgspec.git" }
+# XXX NOTE, only for @goodboy's hacking on `pprint(sort_dicts=False)`
+# for the `pp` alias..
+# pdbp = { path = "../pdbp", editable = true }
 
 # ------ tool.uv.sources ------
 # TODO, distributed (multi-host) extensions
 # linux kernel networking
 # 'pyroute2
 
+# ------ tool.uv.sources ------
+
+[tool.uv]
+# XXX NOTE, prefer the sys python bc apparently the distis from
+# `astral` are built in a way that breaks `pdbp`+`tabcompleter`'s
+# likely due to linking against `libedit` over `readline`..
+# |_https://docs.astral.sh/uv/concepts/python-versions/#managed-python-distributions
+# |_https://gregoryszorc.com/docs/python-build-standalone/main/quirks.html#use-of-libedit-on-linux
+#
+# https://docs.astral.sh/uv/reference/settings/#python-preference
+python-preference = 'system'
+
+# ------ tool.uv ------
+
 [tool.hatch.build.targets.sdist]
 include = ["tractor"]
 
 [tool.hatch.build.targets.wheel]
 include = ["tractor"]
 
-# ------ dependency-groups ------
+# ------ tool.hatch ------
 
 [tool.towncrier]
 package = "tractor"
@@ -142,3 +154,5 @@ log_cli = false
 # TODO: maybe some of these layout choices?
 # https://docs.pytest.org/en/8.0.x/explanation/goodpractices.html#choosing-a-test-layout-import-rules
 # pythonpath = "src"
+
+# ------ tool.pytest ------

tests/conftest.py

@@ -75,7 +75,10 @@ def pytest_configure(config):
 
 @pytest.fixture(scope='session')
 def debug_mode(request):
-    return request.config.option.tractor_debug_mode
+    debug_mode: bool = request.config.option.tractor_debug_mode
+    # if debug_mode:
+    #     breakpoint()
+    return debug_mode
 
 
 @pytest.fixture(scope='session', autouse=True)
@@ -92,6 +95,12 @@ def spawn_backend(request) -> str:
     return request.config.option.spawn_backend
 
 
+# @pytest.fixture(scope='function', autouse=True)
+# def debug_enabled(request) -> str:
+#     from tractor import _state
+#     if _state._runtime_vars['_debug_mode']:
+#         breakpoint()
+
 _ci_env: bool = os.environ.get('CI', False)
 
 
@@ -150,6 +159,18 @@ def pytest_generate_tests(metafunc):
         metafunc.parametrize("start_method", [spawn_backend], scope='module')
 
 
+# TODO: a way to let test scripts (like from `examples/`)
+# guarantee they won't registry addr collide!
+# @pytest.fixture
+# def open_test_runtime(
+#     reg_addr: tuple,
+# ) -> AsyncContextManager:
+#     return partial(
+#         tractor.open_nursery,
+#         registry_addrs=[reg_addr],
+#     )
+
+
 def sig_prog(proc, sig):
     "Kill the actor-process with ``sig``."
     proc.send_signal(sig)

tests/devx/conftest.py

@@ -0,0 +1,243 @@
+'''
+`tractor.devx.*` tooling sub-pkg test space.
+
+'''
+import time
+from typing import (
+    Callable,
+)
+
+import pytest
+from pexpect.exceptions import (
+    TIMEOUT,
+)
+from pexpect.spawnbase import SpawnBase
+
+from tractor._testing import (
+    mk_cmd,
+)
+from tractor.devx._debug import (
+    _pause_msg as _pause_msg,
+    _crash_msg as _crash_msg,
+    _repl_fail_msg as _repl_fail_msg,
+    _ctlc_ignore_header as _ctlc_ignore_header,
+)
+from ..conftest import (
+    _ci_env,
+)
+
+
+@pytest.fixture
+def spawn(
+    start_method,
+    testdir: pytest.Pytester,
+    reg_addr: tuple[str, int],
+
+) -> Callable[[str], None]:
+    '''
+    Use the `pexpect` module shipped via `testdir.spawn()` to
+    run an `./examples/..` script by name.
+
+    '''
+    if start_method != 'trio':
+        pytest.skip(
+            '`pexpect` based tests only supported on `trio` backend'
+        )
+
+    def unset_colors():
+        '''
+        Python 3.13 introduced colored tracebacks that break patt
+        matching,
+
+        https://docs.python.org/3/using/cmdline.html#envvar-PYTHON_COLORS
+        https://docs.python.org/3/using/cmdline.html#using-on-controlling-color
+
+        '''
+        import os
+        os.environ['PYTHON_COLORS'] = '0'
+
+    def _spawn(
+        cmd: str,
+        **mkcmd_kwargs,
+    ):
+        unset_colors()
+        return testdir.spawn(
+            cmd=mk_cmd(
+                cmd,
+                **mkcmd_kwargs,
+            ),
+            expect_timeout=3,
+            # preexec_fn=unset_colors,
+            # ^TODO? get `pytest` core to expose underlying
+            # `pexpect.spawn()` stuff?
+        )
+
+    # such that test-dep can pass input script name.
+    return _spawn
+
+
+@pytest.fixture(
+    params=[False, True],
+    ids='ctl-c={}'.format,
+)
+def ctlc(
+    request,
+    ci_env: bool,
+
+) -> bool:
+
+    use_ctlc = request.param
+
+    node = request.node
+    markers = node.own_markers
+    for mark in markers:
+        if mark.name == 'has_nested_actors':
+            pytest.skip(
+                f'Test {node} has nested actors and fails with Ctrl-C.\n'
+                f'The test can sometimes run fine locally but until'
+                ' we solve' 'this issue this CI test will be xfail:\n'
+                'https://github.com/goodboy/tractor/issues/320'
+            )
+
+        if mark.name == 'ctlcs_bish':
+            pytest.skip(
+                f'Test {node} prolly uses something from the stdlib (namely `asyncio`..)\n'
+                f'The test and/or underlying example script can *sometimes* run fine '
+                f'locally but more then likely until the cpython peeps get their sh#$ together, '
+                f'this test will definitely not behave like `trio` under SIGINT..\n'
+            )
+
+    if use_ctlc:
+        # XXX: disable pygments highlighting for auto-tests
+        # since some envs (like actions CI) will struggle
+        # the the added color-char encoding..
+        from tractor.devx._debug import TractorConfig
+        TractorConfig.use_pygements = False
+
+    yield use_ctlc
+
+
+def expect(
+    child,
+
+    # normally a `pdb` prompt by default
+    patt: str,
+
+    **kwargs,
+
+) -> None:
+    '''
+    Expect wrapper that prints last seen console
+    data before failing.
+
+    '''
+    try:
+        child.expect(
+            patt,
+            **kwargs,
+        )
+    except TIMEOUT:
+        before = str(child.before.decode())
+        print(before)
+        raise
+
+
+PROMPT = r"\(Pdb\+\)"
+
+
+def in_prompt_msg(
+    child: SpawnBase,
+    parts: list[str],
+
+    pause_on_false: bool = False,
+    err_on_false: bool = False,
+    print_prompt_on_false: bool = True,
+
+) -> bool:
+    '''
+    Predicate check if (the prompt's) std-streams output has all
+    `str`-parts in it.
+
+    Can be used in test asserts for bulk matching expected
+    log/REPL output for a given `pdb` interact point.
+
+    '''
+    __tracebackhide__: bool = False
+
+    before: str = str(child.before.decode())
+    for part in parts:
+        if part not in before:
+            if pause_on_false:
+                import pdbp
+                pdbp.set_trace()
+
+            if print_prompt_on_false:
+                print(before)
+
+            if err_on_false:
+                raise ValueError(
+                    f'Could not find pattern in `before` output?\n'
+                    f'part: {part!r}\n'
+                )
+            return False
+
+    return True
+
+
+# TODO: todo support terminal color-chars stripping so we can match
+# against call stack frame output from the the 'll' command the like!
+# -[ ] SO answer for stipping ANSI codes: https://stackoverflow.com/a/14693789
+def assert_before(
+    child: SpawnBase,
+    patts: list[str],
+
+    **kwargs,
+
+) -> None:
+    __tracebackhide__: bool = False
+
+    assert in_prompt_msg(
+        child=child,
+        parts=patts,
+
+        # since this is an "assert" helper ;)
+        err_on_false=True,
+        **kwargs
+    )
+
+
+def do_ctlc(
+    child,
+    count: int = 3,
+    delay: float = 0.1,
+    patt: str|None = None,
+
+    # expect repl UX to reprint the prompt after every
+    # ctrl-c send.
+    # XXX: no idea but, in CI this never seems to work even on 3.10 so
+    # needs some further investigation potentially...
+    expect_prompt: bool = not _ci_env,
+
+) -> str|None:
+
+    before: str|None = None
+
+    # make sure ctl-c sends don't do anything but repeat output
+    for _ in range(count):
+        time.sleep(delay)
+        child.sendcontrol('c')
+
+        # TODO: figure out why this makes CI fail..
+        # if you run this test manually it works just fine..
+        if expect_prompt:
+            time.sleep(delay)
+            child.expect(PROMPT)
+            before = str(child.before.decode())
+            time.sleep(delay)
+
+            if patt:
+                # should see the last line on console
+                assert patt in before
+
+    # return the console content up to the final prompt
+    return before

tests/devx/test_debugger.py

@@ -13,26 +13,25 @@ TODO:
 from functools import partial
 import itertools
 import platform
-import pathlib
 import time
 
 import pytest
-import pexpect
 from pexpect.exceptions import (
     TIMEOUT,
     EOF,
 )
 
-from tractor._testing import (
-    examples_dir,
-)
-from tractor.devx._debug import (
+from .conftest import (
+    do_ctlc,
+    PROMPT,
     _pause_msg,
     _crash_msg,
     _repl_fail_msg,
 )
 from .conftest import (
-    _ci_env,
+    expect,
+    in_prompt_msg,
+    assert_before,
 )
 
 # TODO: The next great debugger audit could be done by you!
@@ -52,15 +51,6 @@ if platform.system() == 'Windows':
     )
 
 
-def mk_cmd(ex_name: str) -> str:
-    '''
-    Generate a command suitable to pass to ``pexpect.spawn()``.
-
-    '''
-    script_path: pathlib.Path = examples_dir() / 'debugging' / f'{ex_name}.py'
-    return ' '.join(['python', str(script_path)])
-
-
 # TODO: was trying to this xfail style but some weird bug i see in CI
 # that's happening at collect time.. pretty soon gonna dump actions i'm
 # thinkin...
@@ -79,142 +69,6 @@ has_nested_actors = pytest.mark.has_nested_actors
 # )
 
 
-@pytest.fixture
-def spawn(
-    start_method,
-    testdir,
-    reg_addr,
-) -> 'pexpect.spawn':
-
-    if start_method != 'trio':
-        pytest.skip(
-            "Debugger tests are only supported on the trio backend"
-        )
-
-    def _spawn(cmd):
-        return testdir.spawn(
-            cmd=mk_cmd(cmd),
-            expect_timeout=3,
-        )
-
-    return _spawn
-
-
-PROMPT = r"\(Pdb\+\)"
-
-
-def expect(
-    child,
-
-    # prompt by default
-    patt: str = PROMPT,
-
-    **kwargs,
-
-) -> None:
-    '''
-    Expect wrapper that prints last seen console
-    data before failing.
-
-    '''
-    try:
-        child.expect(
-            patt,
-            **kwargs,
-        )
-    except TIMEOUT:
-        before = str(child.before.decode())
-        print(before)
-        raise
-
-
-def in_prompt_msg(
-    prompt: str,
-    parts: list[str],
-
-    pause_on_false: bool = False,
-    print_prompt_on_false: bool = True,
-
-) -> bool:
-    '''
-    Predicate check if (the prompt's) std-streams output has all
-    `str`-parts in it.
-
-    Can be used in test asserts for bulk matching expected
-    log/REPL output for a given `pdb` interact point.
-
-    '''
-    __tracebackhide__: bool = False
-
-    for part in parts:
-        if part not in prompt:
-            if pause_on_false:
-                import pdbp
-                pdbp.set_trace()
-
-            if print_prompt_on_false:
-                print(prompt)
-
-            return False
-
-    return True
-
-
-# TODO: todo support terminal color-chars stripping so we can match
-# against call stack frame output from the the 'll' command the like!
-# -[ ] SO answer for stipping ANSI codes: https://stackoverflow.com/a/14693789
-def assert_before(
-    child,
-    patts: list[str],
-
-    **kwargs,
-
-) -> None:
-    __tracebackhide__: bool = False
-
-    # as in before the prompt end
-    before: str = str(child.before.decode())
-    assert in_prompt_msg(
-        prompt=before,
-        parts=patts,
-
-        **kwargs
-    )
-
-
-@pytest.fixture(
-    params=[False, True],
-    ids='ctl-c={}'.format,
-)
-def ctlc(
-    request,
-    ci_env: bool,
-
-) -> bool:
-
-    use_ctlc = request.param
-
-    node = request.node
-    markers = node.own_markers
-    for mark in markers:
-        if mark.name == 'has_nested_actors':
-            pytest.skip(
-                f'Test {node} has nested actors and fails with Ctrl-C.\n'
-                f'The test can sometimes run fine locally but until'
-                ' we solve' 'this issue this CI test will be xfail:\n'
-                'https://github.com/goodboy/tractor/issues/320'
-            )
-
-    if use_ctlc:
-        # XXX: disable pygments highlighting for auto-tests
-        # since some envs (like actions CI) will struggle
-        # the the added color-char encoding..
-        from tractor.devx._debug import TractorConfig
-        TractorConfig.use_pygements = False
-
-    yield use_ctlc
-
-
 @pytest.mark.parametrize(
     'user_in_out',
     [
@@ -238,14 +92,15 @@ def test_root_actor_error(
     # scan for the prompt
     expect(child, PROMPT)
 
-    before = str(child.before.decode())
-
     # make sure expected logging and error arrives
     assert in_prompt_msg(
-        before,
-        [_crash_msg, "('root'"]
+        child,
+        [
+            _crash_msg,
+            "('root'",
+            'AssertionError',
+        ]
     )
-    assert 'AssertionError' in before
 
     # send user command
     child.sendline(user_input)
@@ -264,8 +119,10 @@ def test_root_actor_error(
     ids=lambda item: f'{item[0]} -> {item[1]}',
 )
 def test_root_actor_bp(spawn, user_in_out):
-    """Demonstrate breakpoint from in root actor.
-    """
+    '''
+    Demonstrate breakpoint from in root actor.
+
+    '''
     user_input, expect_err_str = user_in_out
     child = spawn('root_actor_breakpoint')
 
@@ -279,7 +136,7 @@ def test_root_actor_bp(spawn, user_in_out):
     child.expect('\r\n')
 
     # process should exit
-    child.expect(pexpect.EOF)
+    child.expect(EOF)
 
     if expect_err_str is None:
         assert 'Error' not in str(child.before)
@@ -287,38 +144,6 @@ def test_root_actor_bp(spawn, user_in_out):
         assert expect_err_str in str(child.before)
 
 
-def do_ctlc(
-    child,
-    count: int = 3,
-    delay: float = 0.1,
-    patt: str|None = None,
-
-    # expect repl UX to reprint the prompt after every
-    # ctrl-c send.
-    # XXX: no idea but, in CI this never seems to work even on 3.10 so
-    # needs some further investigation potentially...
-    expect_prompt: bool = not _ci_env,
-
-) -> None:
-
-    # make sure ctl-c sends don't do anything but repeat output
-    for _ in range(count):
-        time.sleep(delay)
-        child.sendcontrol('c')
-
-        # TODO: figure out why this makes CI fail..
-        # if you run this test manually it works just fine..
-        if expect_prompt:
-            before = str(child.before.decode())
-            time.sleep(delay)
-            child.expect(PROMPT)
-            time.sleep(delay)
-
-            if patt:
-                # should see the last line on console
-                assert patt in before
-
-
 def test_root_actor_bp_forever(
     spawn,
     ctlc: bool,
@@ -358,7 +183,7 @@ def test_root_actor_bp_forever(
 
     # quit out of the loop
     child.sendline('q')
-    child.expect(pexpect.EOF)
+    child.expect(EOF)
 
 
 @pytest.mark.parametrize(
@@ -380,10 +205,12 @@ def test_subactor_error(
     # scan for the prompt
     child.expect(PROMPT)
 
-    before = str(child.before.decode())
     assert in_prompt_msg(
-        before,
-        [_crash_msg, "('name_error'"]
+        child,
+        [
+            _crash_msg,
+            "('name_error'",
+        ]
     )
 
     if do_next:
@@ -402,17 +229,15 @@ def test_subactor_error(
         child.sendline('continue')
 
     child.expect(PROMPT)
-    before = str(child.before.decode())
-
-    # root actor gets debugger engaged
     assert in_prompt_msg(
-        before,
-        [_crash_msg, "('root'"]
-    )
-    # error is a remote error propagated from the subactor
-    assert in_prompt_msg(
-        before,
-        [_crash_msg, "('name_error'"]
+        child,
+        [
+            _crash_msg,
+            # root actor gets debugger engaged
+            "('root'",
+            # error is a remote error propagated from the subactor
+            "('name_error'",
+        ]
     )
 
     # another round
@@ -423,7 +248,7 @@ def test_subactor_error(
     child.expect('\r\n')
 
     # process should exit
-    child.expect(pexpect.EOF)
+    child.expect(EOF)
 
 
 def test_subactor_breakpoint(
@@ -433,14 +258,11 @@ def test_subactor_breakpoint(
     "Single subactor with an infinite breakpoint loop"
 
     child = spawn('subactor_breakpoint')
-
-    # scan for the prompt
     child.expect(PROMPT)
-
-    before = str(child.before.decode())
     assert in_prompt_msg(
-        before,
-        [_pause_msg, "('breakpoint_forever'"]
+        child,
+        [_pause_msg,
+         "('breakpoint_forever'",]
     )
 
     # do some "next" commands to demonstrate recurrent breakpoint
@@ -456,9 +278,8 @@ def test_subactor_breakpoint(
     for _ in range(5):
         child.sendline('continue')
         child.expect(PROMPT)
-        before = str(child.before.decode())
         assert in_prompt_msg(
-            before,
+            child,
             [_pause_msg, "('breakpoint_forever'"]
         )
 
@@ -471,9 +292,8 @@ def test_subactor_breakpoint(
     # child process should exit but parent will capture pdb.BdbQuit
     child.expect(PROMPT)
 
-    before = str(child.before.decode())
     assert in_prompt_msg(
-        before,
+        child,
         ['RemoteActorError:',
          "('breakpoint_forever'",
          'bdb.BdbQuit',]
@@ -486,14 +306,16 @@ def test_subactor_breakpoint(
     child.sendline('c')
 
     # process should exit
-    child.expect(pexpect.EOF)
+    child.expect(EOF)
 
-    before = str(child.before.decode())
     assert in_prompt_msg(
-        before,
-        ['RemoteActorError:',
+        child, [
+        'MessagingError:',
+        'RemoteActorError:',
          "('breakpoint_forever'",
-         'bdb.BdbQuit',]
+         'bdb.BdbQuit',
+        ],
+        pause_on_false=True,
     )
 
 
@@ -514,7 +336,7 @@ def test_multi_subactors(
 
     before = str(child.before.decode())
     assert in_prompt_msg(
-        before,
+        child,
         [_pause_msg, "('breakpoint_forever'"]
     )
 
@@ -535,12 +357,14 @@ def test_multi_subactors(
 
     # first name_error failure
     child.expect(PROMPT)
-    before = str(child.before.decode())
     assert in_prompt_msg(
-        before,
-        [_crash_msg, "('name_error'"]
+        child,
+        [
+            _crash_msg,
+            "('name_error'",
+            "NameError",
+        ]
     )
-    assert "NameError" in before
 
     if ctlc:
         do_ctlc(child)
@@ -564,9 +388,8 @@ def test_multi_subactors(
     # breakpoint loop should re-engage
     child.sendline('c')
     child.expect(PROMPT)
-    before = str(child.before.decode())
     assert in_prompt_msg(
-        before,
+        child,
         [_pause_msg, "('breakpoint_forever'"]
     )
 
@@ -629,7 +452,7 @@ def test_multi_subactors(
 
     # process should exit
     child.sendline('c')
-    child.expect(pexpect.EOF)
+    child.expect(EOF)
 
     # repeat of previous multierror for final output
     assert_before(child, [
@@ -659,25 +482,28 @@ def test_multi_daemon_subactors(
     # the root's tty lock first so anticipate either crash
     # message on the first entry.
 
-    bp_forev_parts = [_pause_msg, "('bp_forever'"]
+    bp_forev_parts = [
+        _pause_msg,
+        "('bp_forever'",
+    ]
     bp_forev_in_msg = partial(
         in_prompt_msg,
         parts=bp_forev_parts,
     )
 
-    name_error_msg = "NameError: name 'doggypants' is not defined"
-    name_error_parts = [name_error_msg]
+    name_error_msg: str = "NameError: name 'doggypants' is not defined"
+    name_error_parts: list[str] = [name_error_msg]
 
     before = str(child.before.decode())
 
-    if bp_forev_in_msg(prompt=before):
+    if bp_forev_in_msg(child=child):
         next_parts = name_error_parts
 
     elif name_error_msg in before:
         next_parts = bp_forev_parts
 
     else:
-        raise ValueError("Neither log msg was found !?")
+        raise ValueError('Neither log msg was found !?')
 
     if ctlc:
         do_ctlc(child)
@@ -746,14 +572,12 @@ def test_multi_daemon_subactors(
     # wait for final error in root
     # where it crashs with boxed error
     while True:
-        try:
-            child.sendline('c')
-            child.expect(PROMPT)
-            assert_before(
-                child,
-                bp_forev_parts
-            )
-        except AssertionError:
+        child.sendline('c')
+        child.expect(PROMPT)
+        if not in_prompt_msg(
+            child,
+            bp_forev_parts
+        ):
             break
 
     assert_before(
@@ -769,7 +593,7 @@ def test_multi_daemon_subactors(
     )
 
     child.sendline('c')
-    child.expect(pexpect.EOF)
+    child.expect(EOF)
 
 
 @has_nested_actors
@@ -845,7 +669,7 @@ def test_multi_subactors_root_errors(
     ])
 
     child.sendline('c')
-    child.expect(pexpect.EOF)
+    child.expect(EOF)
 
     assert_before(child, [
         # "Attaching to pdb in crashed actor: ('root'",
@@ -934,10 +758,13 @@ def test_root_nursery_cancels_before_child_releases_tty_lock(
     child = spawn('root_cancelled_but_child_is_in_tty_lock')
 
     child.expect(PROMPT)
-
-    before = str(child.before.decode())
-    assert "NameError: name 'doggypants' is not defined" in before
-    assert "tractor._exceptions.RemoteActorError: ('name_error'" not in before
+    assert_before(
+        child,
+        [
+            "NameError: name 'doggypants' is not defined",
+            "tractor._exceptions.RemoteActorError: ('name_error'",
+        ],
+    )
     time.sleep(0.5)
 
     if ctlc:
@@ -975,7 +802,7 @@ def test_root_nursery_cancels_before_child_releases_tty_lock(
 
     for i in range(3):
         try:
-            child.expect(pexpect.EOF, timeout=0.5)
+            child.expect(EOF, timeout=0.5)
             break
         except TIMEOUT:
             child.sendline('c')
@@ -1017,7 +844,7 @@ def test_root_cancels_child_context_during_startup(
         do_ctlc(child)
 
     child.sendline('c')
-    child.expect(pexpect.EOF)
+    child.expect(EOF)
 
 
 def test_different_debug_mode_per_actor(
@@ -1028,9 +855,8 @@ def test_different_debug_mode_per_actor(
     child.expect(PROMPT)
 
     # only one actor should enter the debugger
-    before = str(child.before.decode())
     assert in_prompt_msg(
-        before,
+        child,
         [_crash_msg, "('debugged_boi'", "RuntimeError"],
     )
 
@@ -1038,9 +864,7 @@ def test_different_debug_mode_per_actor(
         do_ctlc(child)
 
     child.sendline('c')
-    child.expect(pexpect.EOF)
-
-    before = str(child.before.decode())
+    child.expect(EOF)
 
     # NOTE: this debugged actor error currently WON'T show up since the
     # root will actually cancel and terminate the nursery before the error
@@ -1059,103 +883,6 @@ def test_different_debug_mode_per_actor(
     )
 
 
-def test_pause_from_sync(
-    spawn,
-    ctlc: bool
-):
-    '''
-    Verify we can use the `pdbp` REPL from sync functions AND from
-    any thread spawned with `trio.to_thread.run_sync()`.
-
-    `examples/debugging/sync_bp.py`
-
-    '''
-    child = spawn('sync_bp')
-
-    # first `sync_pause()` after nurseries open
-    child.expect(PROMPT)
-    assert_before(
-        child,
-        [
-            # pre-prompt line
-            _pause_msg,
-            "<Task '__main__.main'",
-            "('root'",
-        ]
-    )
-    if ctlc:
-        do_ctlc(child)
-
-    child.sendline('c')
-
-
-    # first `await tractor.pause()` inside `p.open_context()` body
-    child.expect(PROMPT)
-
-    # XXX shouldn't see gb loaded message with PDB loglevel!
-    before = str(child.before.decode())
-    assert not in_prompt_msg(
-        before,
-        ['`greenback` portal opened!'],
-    )
-    # should be same root task
-    assert_before(
-        child,
-        [
-            _pause_msg,
-            "<Task '__main__.main'",
-            "('root'",
-        ]
-    )
-
-    if ctlc:
-        do_ctlc(child)
-
-    # one of the bg thread or subactor should have
-    # `Lock.acquire()`-ed
-    # (NOT both, which will result in REPL clobbering!)
-    attach_patts: dict[str, list[str]] = {
-        'subactor': [
-            "'start_n_sync_pause'",
-            "('subactor'",
-        ],
-        'inline_root_bg_thread': [
-            "<Thread(inline_root_bg_thread",
-            "('root'",
-        ],
-        'start_soon_root_bg_thread': [
-            "<Thread(start_soon_root_bg_thread",
-            "('root'",
-        ],
-    }
-    while attach_patts:
-        child.sendline('c')
-        child.expect(PROMPT)
-        before = str(child.before.decode())
-        for key in attach_patts.copy():
-            if key in before:
-                expected_patts: str = attach_patts.pop(key)
-                assert_before(
-                    child,
-                    [_pause_msg] + expected_patts
-                )
-                break
-
-        # ensure no other task/threads engaged a REPL
-        # at the same time as the one that was detected above.
-        for key, other_patts in attach_patts.items():
-            assert not in_prompt_msg(
-                before,
-                other_patts,
-            )
-
-        if ctlc:
-            do_ctlc(child)
-
-    child.sendline('c')
-    child.expect(pexpect.EOF)
-
-
 def test_post_mortem_api(
     spawn,
     ctlc: bool,
@@ -1258,7 +985,7 @@ def test_post_mortem_api(
     # )
 
     child.sendline('c')
-    child.expect(pexpect.EOF)
+    child.expect(EOF)
 
 
 def test_shield_pause(
@@ -1333,9 +1060,26 @@ def test_shield_pause(
         ]
     )
     child.sendline('c')
-    child.expect(pexpect.EOF)
+    child.expect(EOF)
 
 
+# TODO: better error for "non-ideal" usage from the root actor.
+# -[ ] if called from an async scope emit a message that suggests
+#    using `await tractor.pause()` instead since it's less overhead
+#    (in terms of `greenback` and/or extra threads) and if it's from
+#    a sync scope suggest that usage must first call
+#    `ensure_portal()` in the (eventual parent) async calling scope?
+def test_sync_pause_from_bg_task_in_root_actor_():
+    '''
+    When used from the root actor, normally we can only implicitly
+    support `.pause_from_sync()` from the main-parent-task (that
+    opens the runtime via `open_root_actor()`) since `greenback`
+    requires a `.ensure_portal()` call per `trio.Task` where it is
+    used.
+
+    '''
+    ...
+
 # TODO: needs ANSI code stripping tho, see `assert_before()` # above!
 def test_correct_frames_below_hidden():
     '''

tests/devx/test_pause_from_non_trio.py

@@ -0,0 +1,381 @@
+'''
+That "foreign loop/thread" debug REPL support better ALSO WORK!
+
+Same as `test_native_pause.py`.
+All these tests can be understood (somewhat) by running the
+equivalent `examples/debugging/` scripts manually.
+
+'''
+from contextlib import (
+    contextmanager as cm,
+)
+# from functools import partial
+# import itertools
+import time
+# from typing import (
+#     Iterator,
+# )
+
+import pytest
+from pexpect.exceptions import (
+    TIMEOUT,
+    EOF,
+)
+
+from .conftest import (
+    # _ci_env,
+    do_ctlc,
+    PROMPT,
+    # expect,
+    in_prompt_msg,
+    assert_before,
+    _pause_msg,
+    _crash_msg,
+    _ctlc_ignore_header,
+    # _repl_fail_msg,
+)
+
+@cm
+def maybe_expect_timeout(
+    ctlc: bool = False,
+) -> None:
+    try:
+        yield
+    except TIMEOUT:
+        # breakpoint()
+        if ctlc:
+            pytest.xfail(
+                'Some kinda redic threading SIGINT bug i think?\n'
+                'See the notes in `examples/debugging/sync_bp.py`..\n'
+            )
+        raise
+
+
+@pytest.mark.ctlcs_bish
+def test_pause_from_sync(
+    spawn,
+    ctlc: bool,
+):
+    '''
+    Verify we can use the `pdbp` REPL from sync functions AND from
+    any thread spawned with `trio.to_thread.run_sync()`.
+
+    `examples/debugging/sync_bp.py`
+
+    '''
+    child = spawn('sync_bp')
+
+    # first `sync_pause()` after nurseries open
+    child.expect(PROMPT)
+    assert_before(
+        child,
+        [
+            # pre-prompt line
+            _pause_msg,
+            "<Task '__main__.main'",
+            "('root'",
+        ]
+    )
+    if ctlc:
+        do_ctlc(child)
+        # ^NOTE^ subactor not spawned yet; don't need extra delay.
+
+    child.sendline('c')
+
+    # first `await tractor.pause()` inside `p.open_context()` body
+    child.expect(PROMPT)
+
+    # XXX shouldn't see gb loaded message with PDB loglevel!
+    # assert not in_prompt_msg(
+    #     child,
+    #     ['`greenback` portal opened!'],
+    # )
+    # should be same root task
+    assert_before(
+        child,
+        [
+            _pause_msg,
+            "<Task '__main__.main'",
+            "('root'",
+        ]
+    )
+
+    if ctlc:
+        do_ctlc(
+            child,
+            # NOTE: setting this to 0 (or some other sufficient
+            # small val) can cause the test to fail since the
+            # `subactor` suffers a race where the root/parent
+            # sends an actor-cancel prior to it hitting its pause
+            # point; by def the value is 0.1
+            delay=0.4,
+        )
+
+    # XXX, fwiw without a brief sleep here the SIGINT might actually
+    # trigger "subactor" cancellation by its parent  before the
+    # shield-handler is engaged.
+    #
+    # => similar to the `delay` input to `do_ctlc()` below, setting
+    # this too low can cause the test to fail since the `subactor`
+    # suffers a race where the root/parent sends an actor-cancel
+    # prior to the context task hitting its pause point (and thus
+    # engaging the `sigint_shield()` handler in time); this value
+    # seems be good enuf?
+    time.sleep(0.6)
+
+    # one of the bg thread or subactor should have
+    # `Lock.acquire()`-ed
+    # (NOT both, which will result in REPL clobbering!)
+    attach_patts: dict[str, list[str]] = {
+        'subactor': [
+            "'start_n_sync_pause'",
+            "('subactor'",
+        ],
+        'inline_root_bg_thread': [
+            "<Thread(inline_root_bg_thread",
+            "('root'",
+        ],
+        'start_soon_root_bg_thread': [
+            "<Thread(start_soon_root_bg_thread",
+            "('root'",
+        ],
+    }
+    conts: int = 0  # for debugging below matching logic on failure
+    while attach_patts:
+        child.sendline('c')
+        conts += 1
+        child.expect(PROMPT)
+        before = str(child.before.decode())
+        for key in attach_patts:
+            if key in before:
+                attach_key: str = key
+                expected_patts: str = attach_patts.pop(key)
+                assert_before(
+                    child,
+                    [_pause_msg]
+                    +
+                    expected_patts
+                )
+                break
+        else:
+            pytest.fail(
+                f'No keys found?\n\n'
+                f'{attach_patts.keys()}\n\n'
+                f'{before}\n'
+            )
+
+        # ensure no other task/threads engaged a REPL
+        # at the same time as the one that was detected above.
+        for key, other_patts in attach_patts.copy().items():
+            assert not in_prompt_msg(
+                child,
+                other_patts,
+            )
+
+        if ctlc:
+            do_ctlc(
+                child,
+                patt=attach_key,
+                # NOTE same as comment above
+                delay=0.4,
+            )
+
+    child.sendline('c')
+
+    # XXX TODO, weird threading bug it seems despite the
+    # `abandon_on_cancel: bool` setting to
+    # `trio.to_thread.run_sync()`..
+    with maybe_expect_timeout(
+        ctlc=ctlc,
+    ):
+        child.expect(EOF)
+
+
+def expect_any_of(
+    attach_patts: dict[str, list[str]],
+    child,   # what type?
+    ctlc: bool = False,
+    prompt: str = _ctlc_ignore_header,
+    ctlc_delay: float = .4,
+
+) -> list[str]:
+    '''
+    Receive any of a `list[str]` of patterns provided in
+    `attach_patts`.
+
+    Used to test racing prompts from multiple actors and/or
+    tasks using a common root process' `pdbp` REPL.
+
+    '''
+    assert attach_patts
+
+    child.expect(PROMPT)
+    before = str(child.before.decode())
+
+    for attach_key in attach_patts:
+        if attach_key in before:
+            expected_patts: str = attach_patts.pop(attach_key)
+            assert_before(
+                child,
+                expected_patts
+            )
+            break  # from for
+    else:
+        pytest.fail(
+            f'No keys found?\n\n'
+            f'{attach_patts.keys()}\n\n'
+            f'{before}\n'
+        )
+
+    # ensure no other task/threads engaged a REPL
+    # at the same time as the one that was detected above.
+    for key, other_patts in attach_patts.copy().items():
+        assert not in_prompt_msg(
+            child,
+            other_patts,
+        )
+
+    if ctlc:
+        do_ctlc(
+            child,
+            patt=prompt,
+            # NOTE same as comment above
+            delay=ctlc_delay,
+        )
+
+    return expected_patts
+
+
+@pytest.mark.ctlcs_bish
+def test_sync_pause_from_aio_task(
+    spawn,
+
+    ctlc: bool
+    # ^TODO, fix for `asyncio`!!
+):
+    '''
+    Verify we can use the `pdbp` REPL from an `asyncio.Task` spawned using
+    APIs in `.to_asyncio`.
+
+    `examples/debugging/asycio_bp.py`
+
+    '''
+    child = spawn('asyncio_bp')
+
+    # RACE on whether trio/asyncio task bps first
+    attach_patts: dict[str, list[str]] = {
+
+        # first pause in guest-mode (aka "infecting")
+        # `trio.Task`.
+        'trio-side': [
+            _pause_msg,
+            "<Task 'trio_ctx'",
+            "('aio_daemon'",
+        ],
+
+        # `breakpoint()` from `asyncio.Task`.
+        'asyncio-side': [
+            _pause_msg,
+            "<Task pending name='Task-2' coro=<greenback_shim()",
+            "('aio_daemon'",
+        ],
+    }
+
+    while attach_patts:
+        expect_any_of(
+            attach_patts=attach_patts,
+            child=child,
+            ctlc=ctlc,
+        )
+        child.sendline('c')
+
+    # NOW in race order,
+    # - the asyncio-task will error
+    # - the root-actor parent task will pause
+    #
+    attach_patts: dict[str, list[str]] = {
+
+        # error raised in `asyncio.Task`
+        "raise ValueError('asyncio side error!')": [
+            _crash_msg,
+            "<Task 'trio_ctx'",
+            "@ ('aio_daemon'",
+            "ValueError: asyncio side error!",
+
+            # XXX, we no longer show this frame by default!
+            # 'return await chan.receive()',  # `.to_asyncio` impl internals in tb
+        ],
+
+        # parent-side propagation via actor-nursery/portal
+        # "tractor._exceptions.RemoteActorError: remote task raised a 'ValueError'": [
+        "remote task raised a 'ValueError'": [
+            _crash_msg,
+            "src_uid=('aio_daemon'",
+            "('aio_daemon'",
+        ],
+
+        # a final pause in root-actor
+        "<Task '__main__.main'": [
+            _pause_msg,
+            "<Task '__main__.main'",
+            "('root'",
+        ],
+    }
+    while attach_patts:
+        expect_any_of(
+            attach_patts=attach_patts,
+            child=child,
+            ctlc=ctlc,
+        )
+        child.sendline('c')
+
+    assert not attach_patts
+
+    # final boxed error propagates to root
+    assert_before(
+        child,
+        [
+            _crash_msg,
+            "<Task '__main__.main'",
+            "('root'",
+            "remote task raised a 'ValueError'",
+            "ValueError: asyncio side error!",
+        ]
+    )
+
+    if ctlc:
+        do_ctlc(
+            child,
+            # NOTE: setting this to 0 (or some other sufficient
+            # small val) can cause the test to fail since the
+            # `subactor` suffers a race where the root/parent
+            # sends an actor-cancel prior to it hitting its pause
+            # point; by def the value is 0.1
+            delay=0.4,
+        )
+
+    child.sendline('c')
+    # with maybe_expect_timeout():
+    child.expect(EOF)
+
+
+def test_sync_pause_from_non_greenbacked_aio_task():
+    '''
+    Where the `breakpoint()` caller task is NOT spawned by
+    `tractor.to_asyncio` and thus never activates
+    a `greenback.ensure_portal()` beforehand, presumably bc the task
+    was started by some lib/dep as in often seen in the field.
+
+    Ensure sync pausing works when the pause is in,
+
+    - the root actor running in infected-mode?
+      |_ since we don't need any IPC to acquire the debug lock?
+      |_ is there some way to handle this like the non-main-thread case?
+
+    All other cases need to error out appropriately right?
+
+    - for any subactor we can't avoid needing the repl lock..
+      |_ is there a way to hook into `asyncio.ensure_future(obj)`?
+
+    '''
+    pass

tests/devx/test_tooling.py

@@ -0,0 +1,172 @@
+'''
+That "native" runtime-hackin toolset better be dang useful!
+
+Verify the funtion of a variety of "developer-experience" tools we
+offer from the `.devx` sub-pkg:
+
+- use of the lovely `stackscope` for dumping actor `trio`-task trees
+  during operation and hangs.
+
+TODO:
+- demonstration of `CallerInfo` call stack frame filtering such that
+  for logging and REPL purposes a user sees exactly the layers needed
+  when debugging a problem inside the stack vs. in their app.
+
+'''
+import os
+import signal
+import time
+
+from .conftest import (
+    expect,
+    assert_before,
+    in_prompt_msg,
+    PROMPT,
+    _pause_msg,
+)
+from pexpect.exceptions import (
+    # TIMEOUT,
+    EOF,
+)
+
+
+def test_shield_pause(
+    spawn,
+):
+    '''
+    Verify the `tractor.pause()/.post_mortem()` API works inside an
+    already cancelled `trio.CancelScope` and that you can step to the
+    next checkpoint wherein the cancelled will get raised.
+
+    '''
+    child = spawn(
+        'shield_hang_in_sub'
+    )
+    expect(
+        child,
+        'Yo my child hanging..?',
+    )
+    assert_before(
+        child,
+        [
+            'Entering shield sleep..',
+            'Enabling trace-trees on `SIGUSR1` since `stackscope` is installed @',
+        ]
+    )
+
+    script_pid: int = child.pid
+    print(
+        f'Sending SIGUSR1 to {script_pid}\n'
+        f'(kill -s SIGUSR1 {script_pid})\n'
+    )
+    os.kill(
+        script_pid,
+        signal.SIGUSR1,
+    )
+    time.sleep(0.2)
+    expect(
+        child,
+        # end-of-tree delimiter
+        "end-of-\('root'",
+    )
+    assert_before(
+        child,
+        [
+            # 'Srying to dump `stackscope` tree..',
+            # 'Dumping `stackscope` tree for actor',
+            "('root'",  # uid line
+
+            # TODO!? this used to show?
+            # -[ ] mk reproducable for @oremanj?
+            #
+            # parent block point (non-shielded)
+            # 'await trio.sleep_forever()  # in root',
+        ]
+    )
+    expect(
+        child,
+        # end-of-tree delimiter
+        "end-of-\('hanger'",
+    )
+    assert_before(
+        child,
+        [
+            # relay to the sub should be reported
+            'Relaying `SIGUSR1`[10] to sub-actor',
+
+            "('hanger'",  # uid line
+
+            # TODO!? SEE ABOVE
+            # hanger LOC where it's shield-halted
+            # 'await trio.sleep_forever()  # in subactor',
+        ]
+    )
+
+    # simulate the user sending a ctl-c to the hanging program.
+    # this should result in the terminator kicking in since
+    # the sub is shield blocking and can't respond to SIGINT.
+    os.kill(
+        child.pid,
+        signal.SIGINT,
+    )
+    expect(
+        child,
+        'Shutting down actor runtime',
+        timeout=6,
+    )
+    assert_before(
+        child,
+        [
+            'raise KeyboardInterrupt',
+            # 'Shutting down actor runtime',
+            '#T-800 deployed to collect zombie B0',
+            "'--uid', \"('hanger',",
+        ]
+    )
+
+
+def test_breakpoint_hook_restored(
+    spawn,
+):
+    '''
+    Ensures our actor runtime sets a custom `breakpoint()` hook
+    on open then restores the stdlib's default on close.
+
+    The hook state validation is done via `assert`s inside the
+    invoked script with only `breakpoint()` (not `tractor.pause()`)
+    calls used.
+
+    '''
+    child = spawn('restore_builtin_breakpoint')
+
+    child.expect(PROMPT)
+    assert_before(
+        child,
+        [
+            _pause_msg,
+            "<Task '__main__.main'",
+            "('root'",
+            "first bp, tractor hook set",
+        ]
+    )
+    child.sendline('c')
+    child.expect(PROMPT)
+    assert_before(
+        child,
+        [
+            "last bp, stdlib hook restored",
+        ]
+    )
+
+    # since the stdlib hook was already restored there should be NO
+    # `tractor` `log.pdb()` content from console!
+    assert not in_prompt_msg(
+        child,
+        [
+            _pause_msg,
+            "<Task '__main__.main'",
+            "('root'",
+        ],
+    )
+    child.sendline('c')
+    child.expect(EOF)

tests/test_advanced_faults.py

@@ -3,7 +3,6 @@ Sketchy network blackoutz, ugly byzantine gens, puedes eschuchar la
 cancelacion?..
 
 '''
-import itertools
 from functools import partial
 from types import ModuleType
 
@@ -230,13 +229,10 @@ def test_ipc_channel_break_during_stream(
     # get raw instance from pytest wrapper
     value = excinfo.value
     if isinstance(value, ExceptionGroup):
-        value = next(
-            itertools.dropwhile(
-                lambda exc: not isinstance(exc, expect_final_exc),
-                value.exceptions,
-            )
-        )
-        assert value
+        excs = value.exceptions
+        assert len(excs) == 1
+        final_exc = excs[0]
+        assert isinstance(final_exc, expect_final_exc)
 
 
 @tractor.context
@@ -259,15 +255,16 @@ async def break_ipc_after_started(
 
 def test_stream_closed_right_after_ipc_break_and_zombie_lord_engages():
     '''
-    Verify that is a subactor's IPC goes down just after bringing up a stream
-    the parent can trigger a SIGINT and the child will be reaped out-of-IPC by
-    the localhost process supervision machinery: aka "zombie lord".
+    Verify that is a subactor's IPC goes down just after bringing up
+    a stream the parent can trigger a SIGINT and the child will be
+    reaped out-of-IPC by the localhost process supervision machinery:
+    aka "zombie lord".
 
     '''
     async def main():
         with trio.fail_after(3):
-            async with tractor.open_nursery() as n:
-                portal = await n.start_actor(
+            async with tractor.open_nursery() as an:
+                portal = await an.start_actor(
                     'ipc_breaker',
                     enable_modules=[__name__],
                 )

tests/test_advanced_streaming.py

@@ -307,7 +307,15 @@ async def inf_streamer(
 
     async with (
         ctx.open_stream() as stream,
-        trio.open_nursery() as tn,
+
+        # XXX TODO, INTERESTING CASE!!
+        # - if we don't collapse the eg then the embedded
+        # `trio.EndOfChannel` doesn't propagate directly to the above
+        # .open_stream() parent, resulting in it also raising instead
+        # of gracefully absorbing as normal.. so how to handle?
+        trio.open_nursery(
+            strict_exception_groups=False,
+        ) as tn,
     ):
         async def close_stream_on_sentinel():
             async for msg in stream:

tests/test_cancellation.py

@@ -130,7 +130,7 @@ def test_multierror(
             try:
                 await portal2.result()
             except tractor.RemoteActorError as err:
-                assert err.boxed_type == AssertionError
+                assert err.boxed_type is AssertionError
                 print("Look Maa that first actor failed hard, hehh")
                 raise
 
@@ -182,7 +182,7 @@ def test_multierror_fast_nursery(reg_addr, start_method, num_subactors, delay):
 
     for exc in exceptions:
         assert isinstance(exc, tractor.RemoteActorError)
-        assert exc.boxed_type == AssertionError
+        assert exc.boxed_type is AssertionError
 
 
 async def do_nothing():
@@ -504,7 +504,9 @@ def test_cancel_via_SIGINT_other_task(
     if is_win():  # smh
         timeout += 1
 
-    async def spawn_and_sleep_forever(task_status=trio.TASK_STATUS_IGNORED):
+    async def spawn_and_sleep_forever(
+        task_status=trio.TASK_STATUS_IGNORED
+    ):
         async with tractor.open_nursery() as tn:
             for i in range(3):
                 await tn.run_in_actor(
@@ -517,7 +519,9 @@ def test_cancel_via_SIGINT_other_task(
     async def main():
         # should never timeout since SIGINT should cancel the current program
         with trio.fail_after(timeout):
-            async with trio.open_nursery() as n:
+            async with trio.open_nursery(
+                strict_exception_groups=False,
+            ) as n:
                 await n.start(spawn_and_sleep_forever)
                 if 'mp' in spawn_backend:
                     time.sleep(0.1)
@@ -610,6 +614,12 @@ def test_fast_graceful_cancel_when_spawn_task_in_soft_proc_wait_for_daemon(
                     nurse.start_soon(delayed_kbi)
 
                     await p.run(do_nuthin)
+
+        # need to explicitly re-raise the lone kbi..now
+        except* KeyboardInterrupt as kbi_eg:
+            assert (len(excs := kbi_eg.exceptions) == 1)
+            raise excs[0]
+
         finally:
             duration = time.time() - start
             if duration > timeout:

tests/test_caps_based_msging.py

@@ -874,13 +874,13 @@ def chk_pld_type(
     return roundtrip
 
 
-def test_limit_msgspec():
-
+def test_limit_msgspec(
+    debug_mode: bool,
+):
     async def main():
         async with tractor.open_root_actor(
-            debug_mode=True
+            debug_mode=debug_mode,
         ):
-
             # ensure we can round-trip a boxing `PayloadMsg`
             assert chk_pld_type(
                 payload_spec=Any,

tests/test_child_manages_service_nursery.py

@@ -95,8 +95,8 @@ async def trio_main(
 
     # stash a "service nursery" as "actor local" (aka a Python global)
     global _nursery
-    n = _nursery
-    assert n
+    tn = _nursery
+    assert tn
 
     async def consume_stream():
         async with wrapper_mngr() as stream:
@@ -104,10 +104,10 @@ async def trio_main(
                 print(msg)
 
     # run 2 tasks to ensure broadcaster chan use
-    n.start_soon(consume_stream)
-    n.start_soon(consume_stream)
+    tn.start_soon(consume_stream)
+    tn.start_soon(consume_stream)
 
-    n.start_soon(trio_sleep_and_err)
+    tn.start_soon(trio_sleep_and_err)
 
     await trio.sleep_forever()
 
@@ -117,8 +117,10 @@ async def open_actor_local_nursery(
     ctx: tractor.Context,
 ):
     global _nursery
-    async with trio.open_nursery() as n:
-        _nursery = n
+    async with trio.open_nursery(
+        strict_exception_groups=False,
+    ) as tn:
+        _nursery = tn
         await ctx.started()
         await trio.sleep(10)
         # await trio.sleep(1)
@@ -132,7 +134,7 @@ async def open_actor_local_nursery(
         # never yields back.. aka a scenario where the
         # ``tractor.context`` task IS NOT in the service n's cancel
         # scope.
-        n.cancel_scope.cancel()
+        tn.cancel_scope.cancel()
 
 
 @pytest.mark.parametrize(
@@ -157,7 +159,7 @@ def test_actor_managed_trio_nursery_task_error_cancels_aio(
         async with tractor.open_nursery() as n:
             p = await n.start_actor(
                 'nursery_mngr',
-                infect_asyncio=asyncio_mode,
+                infect_asyncio=asyncio_mode,  # TODO, is this enabling debug mode?
                 enable_modules=[__name__],
             )
             async with (

tests/test_context_stream_semantics.py

@@ -955,7 +955,7 @@ async def echo_back_sequence(
     )
 
     await ctx.started()
-    # await tractor.breakpoint()
+    # await tractor.pause()
     async with ctx.open_stream(
         msg_buffer_size=msg_buffer_size,
 

tests/test_discovery.py

@@ -181,7 +181,9 @@ async def spawn_and_check_registry(
 
             try:
                 async with tractor.open_nursery() as n:
-                    async with trio.open_nursery() as trion:
+                    async with trio.open_nursery(
+                        strict_exception_groups=False,
+                    ) as trion:
 
                         portals = {}
                         for i in range(3):
@@ -316,7 +318,9 @@ async def close_chans_before_nursery(
                         async with portal2.open_stream_from(
                             stream_forever
                         ) as agen2:
-                            async with trio.open_nursery() as n:
+                            async with trio.open_nursery(
+                                strict_exception_groups=False,
+                            ) as n:
                                 n.start_soon(streamer, agen1)
                                 n.start_soon(cancel, use_signal, .5)
                                 try:

tests/test_docs_examples.py

@@ -19,7 +19,7 @@ from tractor._testing import (
 @pytest.fixture
 def run_example_in_subproc(
     loglevel: str,
-    testdir: pytest.Testdir,
+    testdir: pytest.Pytester,
     reg_addr: tuple[str, int],
 ):
 
@@ -81,28 +81,36 @@ def run_example_in_subproc(
 
     # walk yields: (dirpath, dirnames, filenames)
     [
-        (p[0], f) for p in os.walk(examples_dir()) for f in p[2]
+        (p[0], f)
+        for p in os.walk(examples_dir())
+        for f in p[2]
 
-        if '__' not in f
-        and f[0] != '_'
-        and 'debugging' not in p[0]
-        and 'integration' not in p[0]
-        and 'advanced_faults' not in p[0]
-        and 'multihost' not in p[0]
+        if (
+            '__' not in f
+            and f[0] != '_'
+            and 'debugging' not in p[0]
+            and 'integration' not in p[0]
+            and 'advanced_faults' not in p[0]
+            and 'multihost' not in p[0]
+        )
     ],
-
     ids=lambda t: t[1],
 )
-def test_example(run_example_in_subproc, example_script):
-    """Load and run scripts from this repo's ``examples/`` dir as a user
+def test_example(
+    run_example_in_subproc,
+    example_script,
+):
+    '''
+    Load and run scripts from this repo's ``examples/`` dir as a user
     would copy and pasing them into their editor.
 
     On windows a little more "finessing" is done to make
     ``multiprocessing`` play nice: we copy the ``__main__.py`` into the
     test directory and invoke the script as a module with ``python -m
     test_example``.
-    """
-    ex_file = os.path.join(*example_script)
+
+    '''
+    ex_file: str = os.path.join(*example_script)
 
     if 'rpc_bidir_streaming' in ex_file and sys.version_info < (3, 9):
         pytest.skip("2-way streaming example requires py3.9 async with syntax")
@@ -128,7 +136,8 @@ def test_example(run_example_in_subproc, example_script):
                     # shouldn't eventually once we figure out what's
                     # a better way to be explicit about aio side
                     # cancels?
-                    and 'asyncio.exceptions.CancelledError' not in last_error
+                    and
+                    'asyncio.exceptions.CancelledError' not in last_error
                 ):
                     raise Exception(errmsg)
 

tests/test_inter_peer_cancellation.py

@@ -170,7 +170,7 @@ def test_do_not_swallow_error_before_started_by_remote_contextcancelled(
         trio.run(main)
 
     rae = excinfo.value
-    assert rae.boxed_type == TypeError
+    assert rae.boxed_type is TypeError
 
 
 @tractor.context

tests/test_root_infect_asyncio.py

@@ -0,0 +1,248 @@
+'''
+Special attention cases for using "infect `asyncio`" mode from a root
+actor; i.e. not using a std `trio.run()` bootstrap.
+
+'''
+import asyncio
+from functools import partial
+
+import pytest
+import trio
+import tractor
+from tractor import (
+    to_asyncio,
+)
+from tests.test_infected_asyncio import (
+    aio_echo_server,
+)
+
+
+@pytest.mark.parametrize(
+    'raise_error_mid_stream',
+    [
+        False,
+        Exception,
+        KeyboardInterrupt,
+    ],
+    ids='raise_error={}'.format,
+)
+def test_infected_root_actor(
+    raise_error_mid_stream: bool|Exception,
+
+    # conftest wide
+    loglevel: str,
+    debug_mode: bool,
+):
+    '''
+    Verify you can run the `tractor` runtime with `Actor.is_infected_aio() == True`
+    in the root actor.
+
+    '''
+    async def _trio_main():
+        with trio.fail_after(2 if not debug_mode else 999):
+            first: str
+            chan: to_asyncio.LinkedTaskChannel
+            async with (
+                tractor.open_root_actor(
+                    debug_mode=debug_mode,
+                    loglevel=loglevel,
+                ),
+                to_asyncio.open_channel_from(
+                    aio_echo_server,
+                ) as (first, chan),
+            ):
+                assert first == 'start'
+
+                for i in range(1000):
+                    await chan.send(i)
+                    out = await chan.receive()
+                    assert out == i
+                    print(f'asyncio echoing {i}')
+
+                    if (
+                        raise_error_mid_stream
+                        and
+                        i == 500
+                    ):
+                        raise raise_error_mid_stream
+
+                    if out is None:
+                        try:
+                            out = await chan.receive()
+                        except trio.EndOfChannel:
+                            break
+                        else:
+                            raise RuntimeError(
+                                'aio channel never stopped?'
+                            )
+
+    if raise_error_mid_stream:
+        with pytest.raises(raise_error_mid_stream):
+            tractor.to_asyncio.run_as_asyncio_guest(
+                trio_main=_trio_main,
+            )
+    else:
+        tractor.to_asyncio.run_as_asyncio_guest(
+            trio_main=_trio_main,
+        )
+
+
+
+async def sync_and_err(
+    # just signature placeholders for compat with
+    # ``to_asyncio.open_channel_from()``
+    to_trio: trio.MemorySendChannel,
+    from_trio: asyncio.Queue,
+    ev: asyncio.Event,
+
+):
+    if to_trio:
+        to_trio.send_nowait('start')
+
+    await ev.wait()
+    raise RuntimeError('asyncio-side')
+
+
+@pytest.mark.parametrize(
+    'aio_err_trigger',
+    [
+        'before_start_point',
+        'after_trio_task_starts',
+        'after_start_point',
+    ],
+    ids='aio_err_triggered={}'.format
+)
+def test_trio_prestarted_task_bubbles(
+    aio_err_trigger: str,
+
+    # conftest wide
+    loglevel: str,
+    debug_mode: bool,
+):
+    async def pre_started_err(
+        raise_err: bool = False,
+        pre_sleep: float|None = None,
+        aio_trigger: asyncio.Event|None = None,
+        task_status=trio.TASK_STATUS_IGNORED,
+    ):
+        '''
+        Maybe pre-started error then sleep.
+
+        '''
+        if pre_sleep is not None:
+            print(f'Sleeping from trio for {pre_sleep!r}s !')
+            await trio.sleep(pre_sleep)
+
+        # signal aio-task to raise JUST AFTER this task
+        # starts but has not yet `.started()`
+        if aio_trigger:
+            print('Signalling aio-task to raise from `trio`!!')
+            aio_trigger.set()
+
+        if raise_err:
+            print('Raising from trio!')
+            raise TypeError('trio-side')
+
+        task_status.started()
+        await trio.sleep_forever()
+
+    async def _trio_main():
+        # with trio.fail_after(2):
+        with trio.fail_after(999):
+            first: str
+            chan: to_asyncio.LinkedTaskChannel
+            aio_ev = asyncio.Event()
+
+            async with (
+                tractor.open_root_actor(
+                    debug_mode=False,
+                    loglevel=loglevel,
+                ),
+            ):
+                # TODO, tests for this with 3.13 egs?
+                # from tractor.devx import open_crash_handler
+                # with open_crash_handler():
+                async with (
+                    # where we'll start a sub-task that errors BEFORE
+                    # calling `.started()` such that the error should
+                    # bubble before the guest run terminates!
+                    trio.open_nursery() as tn,
+
+                    # THEN start an infect task which should error just
+                    # after the trio-side's task does.
+                    to_asyncio.open_channel_from(
+                        partial(
+                            sync_and_err,
+                            ev=aio_ev,
+                        )
+                    ) as (first, chan),
+                ):
+
+                    for i in range(5):
+                        pre_sleep: float|None = None
+                        last_iter: bool = (i == 4)
+
+                        # TODO, missing cases?
+                        # -[ ] error as well on
+                        #    'after_start_point' case as well for
+                        #    another case?
+                        raise_err: bool = False
+
+                        if last_iter:
+                            raise_err: bool = True
+
+                            # trigger aio task to error on next loop
+                            # tick/checkpoint
+                            if aio_err_trigger == 'before_start_point':
+                                aio_ev.set()
+
+                            pre_sleep: float = 0
+
+                        await tn.start(
+                            pre_started_err,
+                            raise_err,
+                            pre_sleep,
+                            (aio_ev if (
+                                    aio_err_trigger == 'after_trio_task_starts'
+                                    and
+                                    last_iter
+                                ) else None
+                            ),
+                        )
+
+                        if (
+                            aio_err_trigger == 'after_start_point'
+                            and
+                            last_iter
+                        ):
+                            aio_ev.set()
+
+    with pytest.raises(
+        expected_exception=ExceptionGroup,
+    ) as excinfo:
+        tractor.to_asyncio.run_as_asyncio_guest(
+            trio_main=_trio_main,
+        )
+
+    eg = excinfo.value
+    rte_eg, rest_eg = eg.split(RuntimeError)
+
+    # ensure the trio-task's error bubbled despite the aio-side
+    # having (maybe) errored first.
+    if aio_err_trigger in (
+        'after_trio_task_starts',
+        'after_start_point',
+    ):
+        assert len(errs := rest_eg.exceptions) == 1
+        typerr = errs[0]
+        assert (
+            type(typerr) is TypeError
+            and
+            'trio-side' in typerr.args
+        )
+
+    # when aio errors BEFORE (last) trio task is scheduled, we should
+    # never see anythinb but the aio-side.
+    else:
+        assert len(rtes := rte_eg.exceptions) == 1
+        assert 'asyncio-side' in rtes[0].args[0]

tests/test_task_broadcasting.py

@@ -2,7 +2,9 @@
 Broadcast channels for fan-out to local tasks.
 
 """
-from contextlib import asynccontextmanager
+from contextlib import (
+    asynccontextmanager as acm,
+)
 from functools import partial
 from itertools import cycle
 import time
@@ -15,6 +17,7 @@ import tractor
 from tractor.trionics import (
     broadcast_receiver,
     Lagged,
+    collapse_eg,
 )
 
 
@@ -62,7 +65,7 @@ async def ensure_sequence(
                 break
 
 
-@asynccontextmanager
+@acm
 async def open_sequence_streamer(
 
     sequence: list[int],
@@ -74,9 +77,9 @@ async def open_sequence_streamer(
     async with tractor.open_nursery(
         arbiter_addr=reg_addr,
         start_method=start_method,
-    ) as tn:
+    ) as an:
 
-        portal = await tn.start_actor(
+        portal = await an.start_actor(
             'sequence_echoer',
             enable_modules=[__name__],
         )
@@ -155,9 +158,12 @@ def test_consumer_and_parent_maybe_lag(
         ) as stream:
 
             try:
-                async with trio.open_nursery() as n:
+                async with (
+                    collapse_eg(),
+                    trio.open_nursery() as tn,
+                ):
 
-                    n.start_soon(
+                    tn.start_soon(
                         ensure_sequence,
                         stream,
                         sequence.copy(),
@@ -230,8 +236,8 @@ def test_faster_task_to_recv_is_cancelled_by_slower(
 
         ) as stream:
 
-            async with trio.open_nursery() as n:
-                n.start_soon(
+            async with trio.open_nursery() as tn:
+                tn.start_soon(
                     ensure_sequence,
                     stream,
                     sequence.copy(),
@@ -253,7 +259,7 @@ def test_faster_task_to_recv_is_cancelled_by_slower(
                         continue
 
                 print('cancelling faster subtask')
-                n.cancel_scope.cancel()
+                tn.cancel_scope.cancel()
 
             try:
                 value = await stream.receive()
@@ -271,7 +277,7 @@ def test_faster_task_to_recv_is_cancelled_by_slower(
                         # the faster subtask was cancelled
                         break
 
-                # await tractor.breakpoint()
+                # await tractor.pause()
                 # await stream.receive()
                 print(f'final value: {value}')
 
@@ -371,13 +377,13 @@ def test_ensure_slow_consumers_lag_out(
                                     f'on {lags}:{value}')
                                 return
 
-            async with trio.open_nursery() as nursery:
+            async with trio.open_nursery() as tn:
 
                 for i in range(1, num_laggers):
 
                     task_name = f'sub_{i}'
                     laggers[task_name] = 0
-                    nursery.start_soon(
+                    tn.start_soon(
                         partial(
                             sub_and_print,
                             delay=i*0.001,
@@ -497,6 +503,7 @@ def test_no_raise_on_lag():
                 # internals when the no raise flag is set.
                 loglevel='warning',
             ),
+            collapse_eg(),
             trio.open_nursery() as n,
         ):
             n.start_soon(slow)

tests/test_trioisms.py

@@ -3,6 +3,10 @@ Reminders for oddities in `trio` that we need to stay aware of and/or
 want to see changed.
 
 '''
+from contextlib import (
+    asynccontextmanager as acm,
+)
+
 import pytest
 import trio
 from trio import TaskStatus
@@ -60,7 +64,9 @@ def test_stashed_child_nursery(use_start_soon):
     async def main():
 
         async with (
-            trio.open_nursery() as pn,
+            trio.open_nursery(
+                strict_exception_groups=False,
+            ) as pn,
         ):
             cn = await pn.start(mk_child_nursery)
             assert cn
@@ -80,3 +86,118 @@ def test_stashed_child_nursery(use_start_soon):
 
     with pytest.raises(NameError):
         trio.run(main)
+
+
+@pytest.mark.parametrize(
+    ('unmask_from_canc', 'canc_from_finally'),
+    [
+        (True, False),
+        (True, True),
+        pytest.param(False, True,
+                     marks=pytest.mark.xfail(reason="never raises!")
+        ),
+    ],
+    # TODO, ask ronny how to impl this .. XD
+    # ids='unmask_from_canc={0}, canc_from_finally={1}',#.format,
+)
+def test_acm_embedded_nursery_propagates_enter_err(
+    canc_from_finally: bool,
+    unmask_from_canc: bool,
+    debug_mode: bool,
+):
+    '''
+    Demo how a masking `trio.Cancelled` could be handled by unmasking from the
+    `.__context__` field when a user (by accident) re-raises from a `finally:`.
+
+    '''
+    import tractor
+
+    @acm
+    async def maybe_raise_from_masking_exc(
+        tn: trio.Nursery,
+        unmask_from: BaseException|None = trio.Cancelled
+
+        # TODO, maybe offer a collection?
+        # unmask_from: set[BaseException] = {
+        #     trio.Cancelled,
+        # },
+    ):
+        if not unmask_from:
+            yield
+            return
+
+        try:
+            yield
+        except* unmask_from as be_eg:
+
+            # TODO, if we offer `unmask_from: set`
+            # for masker_exc_type in unmask_from:
+
+            matches, rest = be_eg.split(unmask_from)
+            if not matches:
+                raise
+
+            for exc_match in be_eg.exceptions:
+                if (
+                    (exc_ctx := exc_match.__context__)
+                    and
+                    type(exc_ctx) not in {
+                        # trio.Cancelled,  # always by default?
+                        unmask_from,
+                    }
+                ):
+                    exc_ctx.add_note(
+                        f'\n'
+                        f'WARNING: the above error was masked by a {unmask_from!r} !?!\n'
+                        f'Are you always cancelling? Say from a `finally:` ?\n\n'
+
+                        f'{tn!r}'
+                    )
+                    raise exc_ctx from exc_match
+
+
+    @acm
+    async def wraps_tn_that_always_cancels():
+        async with (
+            trio.open_nursery() as tn,
+            maybe_raise_from_masking_exc(
+                tn=tn,
+                unmask_from=(
+                    trio.Cancelled
+                    if unmask_from_canc
+                    else None
+                ),
+            )
+        ):
+            try:
+                yield tn
+            finally:
+                if canc_from_finally:
+                    tn.cancel_scope.cancel()
+                    await trio.lowlevel.checkpoint()
+
+    async def _main():
+        with tractor.devx.maybe_open_crash_handler(
+            pdb=debug_mode,
+        ) as bxerr:
+            assert not bxerr.value
+
+            async with (
+                wraps_tn_that_always_cancels() as tn,
+            ):
+                assert not tn.cancel_scope.cancel_called
+                assert 0
+
+        assert (
+            (err := bxerr.value)
+            and
+            type(err) is AssertionError
+        )
+
+    with pytest.raises(ExceptionGroup) as excinfo:
+        trio.run(_main)
+
+    eg: ExceptionGroup = excinfo.value
+    assert_eg, rest_eg = eg.split(AssertionError)
+
+    assert len(assert_eg.exceptions) == 1

tractor/__init__.py

@@ -44,6 +44,7 @@ from ._state import (
     current_actor as current_actor,
     is_root_process as is_root_process,
     current_ipc_ctx as current_ipc_ctx,
+    debug_mode as debug_mode
 )
 from ._exceptions import (
     ContextCancelled as ContextCancelled,
@@ -66,3 +67,4 @@ from ._root import (
 from ._ipc import Channel as Channel
 from ._portal import Portal as Portal
 from ._runtime import Actor as Actor
+from . import hilevel as hilevel

tractor/_clustering.py

@@ -19,10 +19,13 @@ Actor cluster helpers.
 
 '''
 from __future__ import annotations
-
-from contextlib import asynccontextmanager as acm
+from contextlib import (
+    asynccontextmanager as acm,
+)
 from multiprocessing import cpu_count
-from typing import AsyncGenerator, Optional
+from typing import (
+    AsyncGenerator,
+)
 
 import trio
 import tractor

tractor/_context.py

@@ -950,7 +950,7 @@ class Context:
             # f'Context.cancel() => {self.chan.uid}\n'
             f'c)=> {self.chan.uid}\n'
             # f'{self.chan.uid}\n'
-            f' |_ @{self.dst_maddr}\n'
+            f'  |_ @{self.dst_maddr}\n'
             f'    >> {self.repr_rpc}\n'
             # f'    >> {self._nsf}() -> {codec}[dict]:\n\n'
             # TODO: pull msg-type from spec re #320
@@ -1003,7 +1003,8 @@ class Context:
                     )
                 else:
                     log.cancel(
-                        'Timed out on cancel request of remote task?\n'
+                        f'Timed out on cancel request of remote task?\n'
+                        f'\n'
                         f'{reminfo}'
                     )
 
@@ -1560,12 +1561,12 @@ class Context:
                     strict_pld_parity=strict_pld_parity,
                     hide_tb=hide_tb,
                 )
-            except BaseException as err:
+            except BaseException as _bexc:
+                err = _bexc
                 if not isinstance(err, MsgTypeError):
                     __tracebackhide__: bool = False
 
-                raise
-
+                raise err
 
         # TODO: maybe a flag to by-pass encode op if already done
         # here in caller?
@@ -1703,15 +1704,28 @@ class Context:
         # TODO: expose as mod func instead!
         structfmt = pretty_struct.Struct.pformat
         if self._in_overrun:
-            log.warning(
-                f'Queueing OVERRUN msg on caller task:\n\n'
-
+            report: str = (
                 f'{flow_body}'
-
                 f'{structfmt(msg)}\n'
             )
+            over_q: deque = self._overflow_q
             self._overflow_q.append(msg)
 
+            if len(over_q) == over_q.maxlen:
+                report = (
+                    'FAILED to queue OVERRUN msg, OVERAN the OVERRUN QUEUE !!\n\n'
+                    + report
+                )
+                # log.error(report)
+                log.debug(report)
+
+            else:
+                report = (
+                    'Queueing OVERRUN msg on caller task:\n\n'
+                    + report
+                )
+                log.debug(report)
+
             # XXX NOTE XXX
             # overrun is the ONLY case where returning early is fine!
             return False
@@ -1969,7 +1983,10 @@ async def open_context_from_portal(
     ctxc_from_callee: ContextCancelled|None = None
     try:
         async with (
-            trio.open_nursery() as tn,
+            trio.open_nursery(
+                strict_exception_groups=False,
+            ) as tn,
+
             msgops.maybe_limit_plds(
                 ctx=ctx,
                 spec=ctx_meta.get('pld_spec'),

tractor/_entry.py

@@ -20,6 +20,7 @@ Sub-process entry points.
 """
 from __future__ import annotations
 from functools import partial
+import multiprocessing as mp
 import os
 import textwrap
 from typing import (
@@ -64,20 +65,22 @@ def _mp_main(
     '''
     actor._forkserver_info = forkserver_info
     from ._spawn import try_set_start_method
-    spawn_ctx = try_set_start_method(start_method)
+    spawn_ctx: mp.context.BaseContext = try_set_start_method(start_method)
+    assert spawn_ctx
 
     if actor.loglevel is not None:
         log.info(
-            f"Setting loglevel for {actor.uid} to {actor.loglevel}")
+            f'Setting loglevel for {actor.uid} to {actor.loglevel}'
+        )
         get_console_log(actor.loglevel)
 
-    assert spawn_ctx
+    # TODO: use scops headers like for `trio` below!
+    # (well after we libify it maybe..)
     log.info(
-        f"Started new {spawn_ctx.current_process()} for {actor.uid}")
-
-    _state._current_actor = actor
-
-    log.debug(f"parent_addr is {parent_addr}")
+        f'Started new {spawn_ctx.current_process()} for {actor.uid}'
+    #     f"parent_addr is {parent_addr}"
+    )
+    _state._current_actor: Actor = actor
     trio_main = partial(
         async_main,
         actor=actor,
@@ -94,7 +97,9 @@ def _mp_main(
         pass  # handle it the same way trio does?
 
     finally:
-        log.info(f"Subactor {actor.uid} terminated")
+        log.info(
+            f'`mp`-subactor {actor.uid} exited'
+        )
 
 
 # TODO: move this func to some kinda `.devx._conc_lang.py` eventually
@@ -233,7 +238,7 @@ def _trio_main(
             nest_from_op(
                 input_op='>(',  # see syntax ideas above
                 tree_str=actor_info,
-                back_from_op=1,
+                back_from_op=2,  # since "complete"
             )
         )
     logmeth = log.info

tractor/_exceptions.py

@@ -22,6 +22,7 @@ from __future__ import annotations
 import builtins
 import importlib
 from pprint import pformat
+from pdb import bdb
 import sys
 from types import (
     TracebackType,
@@ -82,6 +83,48 @@ class InternalError(RuntimeError):
 
     '''
 
+class AsyncioCancelled(Exception):
+    '''
+    Asyncio cancelled translation (non-base) error
+    for use with the ``to_asyncio`` module
+    to be raised in the ``trio`` side task
+
+    NOTE: this should NOT inherit from `asyncio.CancelledError` or
+    tests should break!
+
+    '''
+
+
+class AsyncioTaskExited(Exception):
+    '''
+    asyncio.Task "exited" translation error for use with the
+    `to_asyncio` APIs to be raised in the `trio` side task indicating
+    on `.run_task()`/`.open_channel_from()` exit that the aio side
+    exited early/silently.
+
+    '''
+
+class TrioCancelled(Exception):
+    '''
+    Trio cancelled translation (non-base) error
+    for use with the `to_asyncio` module
+    to be raised in the `asyncio.Task` to indicate
+    that the `trio` side raised `Cancelled` or an error.
+
+    '''
+
+class TrioTaskExited(Exception):
+    '''
+    The `trio`-side task exited without explicitly cancelling the
+    `asyncio.Task` peer.
+
+    This is very similar to how `trio.ClosedResource` acts as
+    a "clean shutdown" signal to the consumer side of a mem-chan,
+
+    https://trio.readthedocs.io/en/stable/reference-core.html#clean-shutdown-with-channels
+
+    '''
+
 
 # NOTE: more or less should be close to these:
 # 'boxed_type',
@@ -127,8 +170,8 @@ _body_fields: list[str] = list(
 
 def get_err_type(type_name: str) -> BaseException|None:
     '''
-    Look up an exception type by name from the set of locally
-    known namespaces:
+    Look up an exception type by name from the set of locally known
+    namespaces:
 
     - `builtins`
     - `tractor._exceptions`
@@ -139,6 +182,7 @@ def get_err_type(type_name: str) -> BaseException|None:
         builtins,
         _this_mod,
         trio,
+        bdb,
     ]:
         if type_ref := getattr(
             ns,
@@ -358,6 +402,13 @@ class RemoteActorError(Exception):
                 self._ipc_msg.src_type_str
             )
 
+            if not self._src_type:
+                raise TypeError(
+                    f'Failed to lookup src error type with '
+                    f'`tractor._exceptions.get_err_type()` :\n'
+                    f'{self.src_type_str}'
+                )
+
         return self._src_type
 
     @property
@@ -366,6 +417,9 @@ class RemoteActorError(Exception):
         String-name of the (last hop's) boxed error type.
 
         '''
+        # TODO, maybe support also serializing the
+        # `ExceptionGroup.exeptions: list[BaseException]` set under
+        # certain conditions?
         bt: Type[BaseException] = self.boxed_type
         if bt:
             return str(bt.__name__)
@@ -609,6 +663,7 @@ class RemoteActorError(Exception):
                 # just after <Type(
                 #             |___ ..
                 tb_body_indent=1,
+                boxer_header=self.relay_uid,
             )
 
         tail = ''
@@ -651,16 +706,10 @@ class RemoteActorError(Exception):
         failing actor's remote env.
 
         '''
-        src_type_ref: Type[BaseException] = self.src_type
-        if not src_type_ref:
-            raise TypeError(
-                'Failed to lookup src error type:\n'
-                f'{self.src_type_str}'
-            )
-
         # TODO: better tb insertion and all the fancier dunder
         # metadata stuff as per `.__context__` etc. and friends:
         # https://github.com/python-trio/trio/issues/611
+        src_type_ref: Type[BaseException] = self.src_type
         return src_type_ref(self.tb_str)
 
     # TODO: local recontruction of nested inception for a given
@@ -786,8 +835,11 @@ class MsgTypeError(
         '''
         if (
             (_bad_msg := self.msgdata.get('_bad_msg'))
-            and
-            isinstance(_bad_msg, PayloadMsg)
+            and (
+                isinstance(_bad_msg, PayloadMsg)
+                or
+                isinstance(_bad_msg, msgtypes.Start)
+            )
         ):
             return _bad_msg
 
@@ -973,15 +1025,6 @@ class NoRuntime(RuntimeError):
     "The root actor has not been initialized yet"
 
 
-
-class AsyncioCancelled(Exception):
-    '''
-    Asyncio cancelled translation (non-base) error
-    for use with the ``to_asyncio`` module
-    to be raised in the ``trio`` side task
-
-    '''
-
 class MessagingError(Exception):
     '''
     IPC related msg (typing), transaction (ordering) or dialog
@@ -989,7 +1032,6 @@ class MessagingError(Exception):
 
     '''
 
-
 def pack_error(
     exc: BaseException|RemoteActorError,
 
@@ -1143,19 +1185,51 @@ def unpack_error(
 
 
 def is_multi_cancelled(
-    exc: BaseException|BaseExceptionGroup
-) -> bool:
+    exc: BaseException|BaseExceptionGroup,
+
+    ignore_nested: set[BaseException] = set(),
+
+) -> bool|BaseExceptionGroup:
     '''
-    Predicate to determine if a possible ``BaseExceptionGroup`` contains
-    only ``trio.Cancelled`` sub-exceptions (and is likely the result of
-    cancelling a collection of subtasks.
+    Predicate to determine if an `BaseExceptionGroup` only contains
+    some (maybe nested) set of sub-grouped exceptions (like only
+    `trio.Cancelled`s which get swallowed silently by default) and is
+    thus the result of "gracefully cancelling" a collection of
+    sub-tasks (or other conc primitives) and receiving a "cancelled
+    ACK" from each after termination.
+
+    Docs:
+    ----
+    - https://docs.python.org/3/library/exceptions.html#exception-groups
+    - https://docs.python.org/3/library/exceptions.html#BaseExceptionGroup.subgroup
 
     '''
+
+    if (
+        not ignore_nested
+        or
+        trio.Cancelled in ignore_nested
+        # XXX always count-in `trio`'s native signal
+    ):
+        ignore_nested.update({trio.Cancelled})
+
     if isinstance(exc, BaseExceptionGroup):
-        return exc.subgroup(
-            lambda exc: isinstance(exc, trio.Cancelled)
-        ) is not None
+        matched_exc: BaseExceptionGroup|None = exc.subgroup(
+            tuple(ignore_nested),
 
+            # TODO, complain about why not allowed XD
+            # condition=tuple(ignore_nested),
+        )
+        if matched_exc is not None:
+            return matched_exc
+
+    # NOTE, IFF no excs types match (throughout the error-tree)
+    # -> return `False`, OW return the matched sub-eg.
+    #
+    # IOW, for the inverse of ^ for the purpose of
+    # maybe-enter-REPL--logic: "only debug when the err-tree contains
+    # at least one exc-type NOT in `ignore_nested`" ; i.e. the case where
+    # we fallthrough and return `False` here.
     return False
 
 
@@ -1375,7 +1449,9 @@ def _mk_recv_mte(
         any_pld: Any = msgpack.decode(msg.pld)
         message: str = (
             f'invalid `{msg_type.__qualname__}` msg payload\n\n'
-            f'value: `{any_pld!r}` does not match type-spec: '
+            f'{any_pld!r}\n\n'
+            f'has type {type(any_pld)!r}\n\n'
+            f'and does not match type-spec '
             f'`{type(msg).__qualname__}.pld: {codec.pld_spec_str}`'
         )
         bad_msg = msg

tractor/_ipc.py

@@ -255,8 +255,8 @@ class MsgpackTCPStream(MsgTransport):
                 raise TransportClosed(
                     message=(
                         f'IPC transport already closed by peer\n'
-                        f'x)> {type(trans_err)}\n'
-                        f' |_{self}\n'
+                        f'x]> {type(trans_err)}\n'
+                        f'  |_{self}\n'
                     ),
                     loglevel=loglevel,
                 ) from trans_err
@@ -273,8 +273,8 @@ class MsgpackTCPStream(MsgTransport):
                 raise TransportClosed(
                     message=(
                         f'IPC transport already manually closed locally?\n'
-                        f'x)> {type(closure_err)} \n'
-                        f' |_{self}\n'
+                        f'x]> {type(closure_err)} \n'
+                        f'  |_{self}\n'
                     ),
                     loglevel='error',
                     raise_on_report=(
@@ -289,8 +289,8 @@ class MsgpackTCPStream(MsgTransport):
                 raise TransportClosed(
                     message=(
                         f'IPC transport already gracefully closed\n'
-                        f')>\n'
-                        f'|_{self}\n'
+                        f']>\n'
+                        f' |_{self}\n'
                     ),
                     loglevel='transport',
                     # cause=???  # handy or no?

tractor/_portal.py

@@ -533,6 +533,10 @@ async def open_portal(
     async with maybe_open_nursery(
         tn,
         shield=shield,
+        strict_exception_groups=False,
+        # ^XXX^ TODO? soo roll our own then ??
+        # -> since we kinda want the "if only one `.exception` then
+        # just raise that" interface?
     ) as tn:
 
         if not channel.connected():

tractor/_root.py

@@ -80,7 +80,7 @@ async def open_root_actor(
 
     # enables the multi-process debugger support
     debug_mode: bool = False,
-    maybe_enable_greenback: bool = False,  # `.pause_from_sync()/breakpoint()` support
+    maybe_enable_greenback: bool = True,  # `.pause_from_sync()/breakpoint()` support
     enable_stack_on_sig: bool = False,
 
     # internal logging
@@ -95,13 +95,24 @@ async def open_root_actor(
 
     hide_tb: bool = True,
 
+    # XXX, proxied directly to `.devx._debug._maybe_enter_pm()`
+    # for REPL-entry logic.
+    debug_filter: Callable[
+        [BaseException|BaseExceptionGroup],
+        bool,
+    ] = lambda err: not is_multi_cancelled(err),
+
+    # TODO, a way for actors to augment passing derived
+    # read-only state to sublayers?
+    # extra_rt_vars: dict|None = None,
+
 ) -> Actor:
     '''
     Runtime init entry point for ``tractor``.
 
     '''
-    __tracebackhide__: bool = hide_tb
     _debug.hide_runtime_frames()
+    __tracebackhide__: bool = hide_tb
 
     # TODO: stick this in a `@cm` defined in `devx._debug`?
     #
@@ -233,14 +244,8 @@ async def open_root_actor(
         and
         enable_stack_on_sig
     ):
-        try:
-            logger.info('Enabling `stackscope` traces on SIGUSR1')
-            from .devx import enable_stack_on_sig
-            enable_stack_on_sig()
-        except ImportError:
-            logger.warning(
-                '`stackscope` not installed for use in debug mode!'
-            )
+        from .devx._stackscope import enable_stack_on_sig
+        enable_stack_on_sig()
 
     # closed into below ping task-func
     ponged_addrs: list[tuple[str, int]] = []
@@ -336,6 +341,10 @@ async def open_root_actor(
             loglevel=loglevel,
             enable_modules=enable_modules,
         )
+        # XXX, in case the root actor runtime was actually run from
+        # `tractor.to_asyncio.run_as_asyncio_guest()` and NOt
+        # `.trio.run()`.
+        actor._infected_aio = _state._runtime_vars['_is_infected_aio']
 
     # Start up main task set via core actor-runtime nurseries.
     try:
@@ -353,7 +362,10 @@ async def open_root_actor(
         )
 
         # start the actor runtime in a new task
-        async with trio.open_nursery() as nursery:
+        async with trio.open_nursery(
+            strict_exception_groups=False,
+            # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
+        ) as nursery:
 
             # ``_runtime.async_main()`` creates an internal nursery
             # and blocks here until any underlying actor(-process)
@@ -377,6 +389,13 @@ async def open_root_actor(
                 Exception,
                 BaseExceptionGroup,
             ) as err:
+
+                # TODO, in beginning to handle the subsubactor with
+                # crashed grandparent cases..
+                #
+                # was_locked: bool = await _debug.maybe_wait_for_debugger(
+                #     child_in_debug=True,
+                # )
                 # XXX NOTE XXX see equiv note inside
                 # `._runtime.Actor._stream_handler()` where in the
                 # non-root or root-that-opened-this-mahually case we
@@ -385,11 +404,15 @@ async def open_root_actor(
                 entered: bool = await _debug._maybe_enter_pm(
                     err,
                     api_frame=inspect.currentframe(),
+                    debug_filter=debug_filter,
                 )
+
                 if (
                     not entered
                     and
-                    not is_multi_cancelled(err)
+                    not is_multi_cancelled(
+                        err,
+                    )
                 ):
                     logger.exception('Root actor crashed\n')
 
@@ -443,12 +466,19 @@ def run_daemon(
 
     start_method: str | None = None,
     debug_mode: bool = False,
+
+    # TODO, support `infected_aio=True` mode by,
+    # - calling the appropriate entrypoint-func from `.to_asyncio`
+    # - maybe init-ing `greenback` as done above in
+    #   `open_root_actor()`.
+
     **kwargs
 
 ) -> None:
     '''
-    Spawn daemon actor which will respond to RPC; the main task simply
-    starts the runtime and then sleeps forever.
+    Spawn a root (daemon) actor which will respond to RPC; the main
+    task simply starts the runtime and then blocks via embedded
+    `trio.sleep_forever()`.
 
     This is a very minimal convenience wrapper around starting
     a "run-until-cancelled" root actor which can be started with a set
@@ -461,7 +491,6 @@ def run_daemon(
         importlib.import_module(path)
 
     async def _main():
-
         async with open_root_actor(
             registry_addrs=registry_addrs,
             name=name,

tractor/_rpc.py

@@ -620,7 +620,11 @@ async def _invoke(
             tn: trio.Nursery
             rpc_ctx_cs: CancelScope
             async with (
-                trio.open_nursery() as tn,
+                trio.open_nursery(
+                    strict_exception_groups=False,
+                    # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
+
+                ) as tn,
                 msgops.maybe_limit_plds(
                     ctx=ctx,
                     spec=ctx_meta.get('pld_spec'),
@@ -733,8 +737,8 @@ async def _invoke(
         # XXX: do we ever trigger this block any more?
         except (
             BaseExceptionGroup,
-            trio.Cancelled,
             BaseException,
+            trio.Cancelled,
 
         ) as scope_error:
             if (
@@ -847,8 +851,8 @@ async def try_ship_error_to_remote(
             log.critical(
                 'IPC transport failure -> '
                 f'failed to ship error to {remote_descr}!\n\n'
-                f'X=> {channel.uid}\n\n'
-
+                f'{type(msg)!r}[{msg.boxed_type_str}] X=> {channel.uid}\n'
+                f'\n'
                 # TODO: use `.msg.preetty_struct` for this!
                 f'{msg}\n'
             )

tractor/_runtime.py

@@ -59,6 +59,7 @@ from types import ModuleType
 import warnings
 
 import trio
+from trio._core import _run as trio_runtime
 from trio import (
     CancelScope,
     Nursery,
@@ -80,6 +81,7 @@ from ._context import (
 from .log import get_logger
 from ._exceptions import (
     ContextCancelled,
+    InternalError,
     ModuleNotExposed,
     MsgTypeError,
     unpack_error,
@@ -98,6 +100,7 @@ from ._rpc import (
 
 if TYPE_CHECKING:
     from ._supervise import ActorNursery
+    from trio._channel import MemoryChannelState
 
 
 log = get_logger('tractor')
@@ -896,11 +899,15 @@ class Actor:
                 f'peer: {chan.uid}\n'
                 f'cid:{cid}\n'
             )
-            ctx._allow_overruns = allow_overruns
+            ctx._allow_overruns: bool = allow_overruns
 
             # adjust buffer size if specified
-            state = ctx._send_chan._state  # type: ignore
-            if msg_buffer_size and state.max_buffer_size != msg_buffer_size:
+            state: MemoryChannelState  = ctx._send_chan._state  # type: ignore
+            if (
+                msg_buffer_size
+                and
+                state.max_buffer_size != msg_buffer_size
+            ):
                 state.max_buffer_size = msg_buffer_size
 
         except KeyError:
@@ -1094,7 +1101,36 @@ class Actor:
                                 '`tractor.pause_from_sync()` not available!'
                             )
 
-                rvs['_is_root'] = False
+                # XXX ensure the "infected `asyncio` mode" setting
+                # passed down from our spawning parent is consistent
+                # with `trio`-runtime initialization:
+                # - during sub-proc boot, the entrypoint func
+                #   (`._entry.<spawn_backend>_main()`) should set
+                #   `._infected_aio = True` before calling
+                #   `run_as_asyncio_guest()`,
+                # - the value of `infect_asyncio: bool = True` as
+                #   passed to `ActorNursery.start_actor()` must be
+                #   the same as `_runtime_vars['_is_infected_aio']`
+                if (
+                    (aio_rtv := rvs['_is_infected_aio'])
+                    !=
+                    (aio_attr := self._infected_aio)
+                ):
+                    raise InternalError(
+                        'Parent sent runtime-vars that mismatch for the '
+                        '"infected `asyncio` mode" settings ?!?\n\n'
+
+                        f'rvs["_is_infected_aio"] = {aio_rtv}\n'
+                        f'self._infected_aio = {aio_attr}\n'
+                    )
+                if aio_rtv:
+                    assert trio_runtime.GLOBAL_RUN_CONTEXT.runner.is_guest
+                    # ^TODO^ possibly add a `sniffio` or
+                    # `trio` pub-API for `is_guest_mode()`?
+
+                rvs['_is_root'] = False  # obvi XD
+
+                # update process-wide globals
                 _state._runtime_vars.update(rvs)
 
                 # XXX: ``msgspec`` doesn't support serializing tuples
@@ -1247,7 +1283,8 @@ class Actor:
         msg: str = (
             f'Actor-runtime cancel request from {requester_type}\n\n'
             f'<=c) {requesting_uid}\n'
-            f' |_{self}\n'
+            f'  |_{self}\n'
+            f'\n'
         )
 
         # TODO: what happens here when we self-cancel tho?
@@ -1267,13 +1304,15 @@ class Actor:
                 lock_req_ctx.has_outcome
             ):
                 msg += (
-                    '-> Cancelling active debugger request..\n'
+                    f'\n'
+                    f'-> Cancelling active debugger request..\n'
                     f'|_{_debug.Lock.repr()}\n\n'
                     f'|_{lock_req_ctx}\n\n'
                 )
                 # lock_req_ctx._scope.cancel()
                 # TODO: wrap this in a method-API..
                 debug_req.req_cs.cancel()
+                # if lock_req_ctx:
 
             # self-cancel **all** ongoing RPC tasks
             await self.cancel_rpc_tasks(
@@ -1682,11 +1721,15 @@ async def async_main(
         # parent is kept alive as a resilient service until
         # cancellation steps have (mostly) occurred in
         # a deterministic way.
-        async with trio.open_nursery() as root_nursery:
+        async with trio.open_nursery(
+            strict_exception_groups=False,
+        ) as root_nursery:
             actor._root_n = root_nursery
             assert actor._root_n
 
-            async with trio.open_nursery() as service_nursery:
+            async with trio.open_nursery(
+                strict_exception_groups=False,
+            ) as service_nursery:
                 # This nursery is used to handle all inbound
                 # connections to us such that if the TCP server
                 # is killed, connections can continue to process

tractor/_spawn.py

@@ -327,9 +327,10 @@ async def soft_kill(
     uid: tuple[str, str] = portal.channel.uid
     try:
         log.cancel(
-            'Soft killing sub-actor via portal request\n'
-            f'c)> {portal.chan.uid}\n'
-            f' |_{proc}\n'
+            f'Soft killing sub-actor via portal request\n'
+            f'\n'
+            f'(c=> {portal.chan.uid}\n'
+            f'  |_{proc}\n'
         )
         # wait on sub-proc to signal termination
         await wait_func(proc)

tractor/_state.py

@@ -44,6 +44,8 @@ _runtime_vars: dict[str, Any] = {
     '_root_mailbox': (None, None),
     '_registry_addrs': [],
 
+    '_is_infected_aio': False,
+
     # for `tractor.pause_from_sync()` & `breakpoint()` support
     'use_greenback': False,
 }
@@ -70,7 +72,8 @@ def current_actor(
     '''
     if (
         err_on_no_runtime
-        and _current_actor is None
+        and
+        _current_actor is None
     ):
         msg: str = 'No local actor has been initialized yet?\n'
         from ._exceptions import NoRuntime
@@ -105,6 +108,7 @@ def is_main_process() -> bool:
     return mp.current_process().name == 'MainProcess'
 
 
+# TODO, more verby name?
 def debug_mode() -> bool:
     '''
     Bool determining if "debug mode" is on which enables

tractor/_streaming.py

@@ -376,7 +376,7 @@ class MsgStream(trio.abc.Channel):
                 f'Stream self-closed by {self._ctx.side!r}-side before EoC\n'
                 # } bc a stream is a "scope"/msging-phase inside an IPC
                 f'x}}>\n'
-                f'|_{self}\n'
+                f'  |_{self}\n'
             )
             log.cancel(message)
             self._eoc = trio.EndOfChannel(message)

tractor/_supervise.py

@@ -158,6 +158,7 @@ class ActorNursery:
         # configure and pass runtime state
         _rtv = _state._runtime_vars.copy()
         _rtv['_is_root'] = False
+        _rtv['_is_infected_aio'] = infect_asyncio
 
         # allow setting debug policy per actor
         if debug_mode is not None:
@@ -394,17 +395,23 @@ async def _open_and_supervise_one_cancels_all_nursery(
     # `ActorNursery.start_actor()`).
 
     # errors from this daemon actor nursery bubble up to caller
-    async with trio.open_nursery() as da_nursery:
+    async with trio.open_nursery(
+        strict_exception_groups=False,
+        # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
+    ) as da_nursery:
         try:
             # This is the inner level "run in actor" nursery. It is
             # awaited first since actors spawned in this way (using
-            # ``ActorNusery.run_in_actor()``) are expected to only
+            # `ActorNusery.run_in_actor()`) are expected to only
             # return a single result and then complete (i.e. be canclled
             # gracefully). Errors collected from these actors are
             # immediately raised for handling by a supervisor strategy.
             # As such if the strategy propagates any error(s) upwards
             # the above "daemon actor" nursery will be notified.
-            async with trio.open_nursery() as ria_nursery:
+            async with trio.open_nursery(
+                strict_exception_groups=False,
+                # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
+            ) as ria_nursery:
 
                 an = ActorNursery(
                     actor,
@@ -471,8 +478,8 @@ async def _open_and_supervise_one_cancels_all_nursery(
                             ContextCancelled,
                         }:
                             log.cancel(
-                                'Actor-nursery caught remote cancellation\n\n'
-
+                                'Actor-nursery caught remote cancellation\n'
+                                '\n'
                                 f'{inner_err.tb_str}'
                             )
                         else:
@@ -564,7 +571,9 @@ async def _open_and_supervise_one_cancels_all_nursery(
 @acm
 # @api_frame
 async def open_nursery(
+    hide_tb: bool = True,
     **kwargs,
+    # ^TODO, paramspec for `open_root_actor()`
 
 ) -> typing.AsyncGenerator[ActorNursery, None]:
     '''
@@ -582,7 +591,7 @@ async def open_nursery(
     which cancellation scopes correspond to each spawned subactor set.
 
     '''
-    __tracebackhide__: bool = True
+    __tracebackhide__: bool = hide_tb
     implicit_runtime: bool = False
     actor: Actor = current_actor(err_on_no_runtime=False)
     an: ActorNursery|None = None
@@ -598,7 +607,10 @@ async def open_nursery(
             # mark us for teardown on exit
             implicit_runtime: bool = True
 
-            async with open_root_actor(**kwargs) as actor:
+            async with open_root_actor(
+                hide_tb=hide_tb,
+                **kwargs,
+            ) as actor:
                 assert actor is current_actor()
 
                 try:
@@ -636,8 +648,10 @@ async def open_nursery(
         # show frame on any internal runtime-scope error
         if (
             an
-            and not an.cancelled
-            and an._scope_error
+            and
+            not an.cancelled
+            and
+            an._scope_error
         ):
             __tracebackhide__: bool = False
 

tractor/_testing/__init__.py

@@ -19,7 +19,10 @@ Various helpers/utils for auditing your `tractor` app and/or the
 core runtime.
 
 '''
-from contextlib import asynccontextmanager as acm
+from contextlib import (
+    asynccontextmanager as acm,
+)
+import os
 import pathlib
 
 import tractor
@@ -54,6 +57,35 @@ def examples_dir() -> pathlib.Path:
     return repodir() / 'examples'
 
 
+def mk_cmd(
+    ex_name: str,
+    exs_subpath: str = 'debugging',
+) -> str:
+    '''
+    Generate a shell command suitable to pass to `pexpect.spawn()`
+    which runs the script as a python program's entrypoint.
+
+    In particular ensure we disable the new tb coloring via unsetting
+    `$PYTHON_COLORS` so that `pexpect` can pattern match without
+    color-escape-codes.
+
+    '''
+    script_path: pathlib.Path = (
+        examples_dir()
+        / exs_subpath
+        / f'{ex_name}.py'
+    )
+    py_cmd: str = ' '.join([
+        'python',
+        str(script_path)
+    ])
+    # XXX, required for py 3.13+
+    # https://docs.python.org/3/using/cmdline.html#using-on-controlling-color
+    # https://docs.python.org/3/using/cmdline.html#envvar-PYTHON_COLORS
+    os.environ['PYTHON_COLORS'] = '0'
+    return py_cmd
+
+
 @acm
 async def expect_ctxc(
     yay: bool,

tractor/devx/__init__.py

@@ -26,7 +26,7 @@ from ._debug import (
     breakpoint as breakpoint,
     pause as pause,
     pause_from_sync as pause_from_sync,
-    shield_sigint_handler as shield_sigint_handler,
+    sigint_shield as sigint_shield,
     open_crash_handler as open_crash_handler,
     maybe_open_crash_handler as maybe_open_crash_handler,
     maybe_init_greenback as maybe_init_greenback,

tractor/devx/_frame_stack.py

@@ -234,7 +234,7 @@ def find_caller_info(
 _frame2callerinfo_cache: dict[FrameType, CallerInfo] = {}
 
 
-# TODO: -[x] move all this into new `.devx._code`!
+# TODO: -[x] move all this into new `.devx._frame_stack`!
 # -[ ] consider rename to _callstack?
 # -[ ] prolly create a `@runtime_api` dec?
 #   |_ @api_frame seems better?
@@ -286,3 +286,18 @@ def api_frame(
     wrapped._call_infos: dict[FrameType, CallerInfo] = _frame2callerinfo_cache
     wrapped.__api_func__: bool = True
     return wrapper(wrapped)
+
+
+# TODO: something like this instead of the adhoc frame-unhiding
+# blocks all over the runtime!! XD
+# -[ ] ideally we can expect a certain error (set) and if something
+#     else is raised then all frames below the wrapped one will be
+#     un-hidden via `__tracebackhide__: bool = False`.
+# |_ might need to dynamically mutate the code objs like
+#    `pdbp.hideframe()` does?
+# -[ ] use this as a `@acm` decorator as introed in 3.10?
+# @acm
+# async def unhide_frame_when_not(
+#     error_set: set[BaseException],
+# ) -> TracebackType:
+#     ...

tractor/devx/_stackscope.py

@@ -24,19 +24,32 @@ disjoint, parallel executing tasks in separate actors.
 
 '''
 from __future__ import annotations
+# from functools import partial
+from threading import (
+    current_thread,
+    Thread,
+    RLock,
+)
 import multiprocessing as mp
 from signal import (
     signal,
+    getsignal,
     SIGUSR1,
+    SIGINT,
+)
+# import traceback
+from types import ModuleType
+from typing import (
+    Callable,
+    TYPE_CHECKING,
 )
-import traceback
-from typing import TYPE_CHECKING
 
 import trio
 from tractor import (
     _state,
     log as logmod,
 )
+from tractor.devx import _debug
 
 log = logmod.get_logger(__name__)
 
@@ -51,26 +64,68 @@ if TYPE_CHECKING:
 
 @trio.lowlevel.disable_ki_protection
 def dump_task_tree() -> None:
-    import stackscope
-    from tractor.log import get_console_log
+    '''
+    Do a classic `stackscope.extract()` task-tree dump to console at
+    `.devx()` level.
 
+    '''
+    import stackscope
     tree_str: str = str(
         stackscope.extract(
             trio.lowlevel.current_root_task(),
             recurse_child_tasks=True
         )
     )
-    log = get_console_log(
-        name=__name__,
-        level='cancel',
-    )
     actor: Actor = _state.current_actor()
+    thr: Thread = current_thread()
+    current_sigint_handler: Callable = getsignal(SIGINT)
+    if (
+        current_sigint_handler
+        is not
+        _debug.DebugStatus._trio_handler
+    ):
+        sigint_handler_report: str = (
+            'The default `trio` SIGINT handler was replaced?!'
+        )
+    else:
+        sigint_handler_report: str = (
+            'The default `trio` SIGINT handler is in use?!'
+        )
+
+    # sclang symbology
+    # |_<object>
+    # |_(Task/Thread/Process/Actor
+    # |_{Supervisor/Scope
+    # |_[Storage/Memory/IPC-Stream/Data-Struct
+
     log.devx(
         f'Dumping `stackscope` tree for actor\n'
-        f'{actor.name}: {actor}\n'
-        f' |_{mp.current_process()}\n\n'
-        f'{tree_str}\n'
+        f'(>: {actor.uid!r}\n'
+        f' |_{mp.current_process()}\n'
+        f'   |_{thr}\n'
+        f'     |_{actor}\n'
+        f'\n'
+        f'{sigint_handler_report}\n'
+        f'signal.getsignal(SIGINT) -> {current_sigint_handler!r}\n'
+        # f'\n'
+        # start-of-trace-tree delimiter (mostly for testing)
+        # f'------ {actor.uid!r} ------\n'
+        f'\n'
+        f'------ start-of-{actor.uid!r} ------\n'
+        f'|\n'
+        f'{tree_str}'
+        # end-of-trace-tree delimiter (mostly for testing)
+        f'|\n'
+        f'|_____ end-of-{actor.uid!r} ______\n'
     )
+    # TODO: can remove this right?
+    # -[ ] was original code from author
+    #
+    # print(
+    #     'DUMPING FROM PRINT\n'
+    #     +
+    #     content
+    # )
     # import logging
     # try:
     #     with open("/dev/tty", "w") as tty:
@@ -80,58 +135,130 @@ def dump_task_tree() -> None:
     #         "task_tree"
     #     ).exception("Error printing task tree")
 
+_handler_lock = RLock()
+_tree_dumped: bool = False
 
-def signal_handler(
+
+def dump_tree_on_sig(
     sig: int,
     frame: object,
 
     relay_to_subs: bool = True,
 
 ) -> None:
-    try:
-        trio.lowlevel.current_trio_token(
-        ).run_sync_soon(dump_task_tree)
-    except RuntimeError:
-        # not in async context -- print a normal traceback
-        traceback.print_stack()
+    global _tree_dumped, _handler_lock
+    with _handler_lock:
+        # if _tree_dumped:
+        #     log.warning(
+        #         'Already dumped for this actor...??'
+        #     )
+        #     return
+
+        _tree_dumped = True
+
+        # actor: Actor = _state.current_actor()
+        log.devx(
+            'Trying to dump `stackscope` tree..\n'
+        )
+        try:
+            dump_task_tree()
+            # await actor._service_n.start_soon(
+            #     partial(
+            #         trio.to_thread.run_sync,
+            #         dump_task_tree,
+            #     )
+            # )
+            # trio.lowlevel.current_trio_token().run_sync_soon(
+            #     dump_task_tree
+            # )
+
+        except RuntimeError:
+            log.exception(
+                'Failed to dump `stackscope` tree..\n'
+            )
+            # not in async context -- print a normal traceback
+            # traceback.print_stack()
+            raise
+
+        except BaseException:
+            log.exception(
+                'Failed to dump `stackscope` tree..\n'
+            )
+            raise
+
+        # log.devx(
+        #     'Supposedly we dumped just fine..?'
+        # )
 
     if not relay_to_subs:
         return
 
     an: ActorNursery
     for an in _state.current_actor()._actoruid2nursery.values():
-
         subproc: ProcessType
         subactor: Actor
         for subactor, subproc, _ in an._children.values():
-            log.devx(
+            log.warning(
                 f'Relaying `SIGUSR1`[{sig}] to sub-actor\n'
                 f'{subactor}\n'
                 f' |_{subproc}\n'
             )
 
-            if isinstance(subproc, trio.Process):
-                subproc.send_signal(sig)
+            # bc of course stdlib can't have a std API.. XD
+            match subproc:
+                case trio.Process():
+                    subproc.send_signal(sig)
 
-            elif isinstance(subproc, mp.Process):
-                subproc._send_signal(sig)
+                case mp.Process():
+                    subproc._send_signal(sig)
 
 
 def enable_stack_on_sig(
-    sig: int = SIGUSR1
-) -> None:
+    sig: int = SIGUSR1,
+) -> ModuleType:
     '''
     Enable `stackscope` tracing on reception of a signal; by
     default this is SIGUSR1.
 
+    HOT TIP: a task/ctx-tree dump can be triggered from a shell with
+    fancy cmds.
+
+    For ex. from `bash` using `pgrep` and cmd-sustitution
+    (https://www.gnu.org/software/bash/manual/bash.html#Command-Substitution)
+    you could use:
+
+    >> kill -SIGUSR1 $(pgrep -f <part-of-cmd: str>)
+
+    OR without a sub-shell,
+
+    >> pkill --signal SIGUSR1 -f <part-of-cmd: str>
+
     '''
+    try:
+        import stackscope
+    except ImportError:
+        log.warning(
+            '`stackscope` not installed for use in debug mode!'
+        )
+        return None
+
+    handler: Callable|int = getsignal(sig)
+    if handler is dump_tree_on_sig:
+        log.devx(
+            'A `SIGUSR1` handler already exists?\n'
+            f'|_ {handler!r}\n'
+        )
+        return
+
     signal(
         sig,
-        signal_handler,
+        dump_tree_on_sig,
     )
-    # NOTE: not the above can be triggered from
-    # a (xonsh) shell using:
-    # kill -SIGUSR1 @$(pgrep -f '<cmd>')
-    #
-    # for example if you were looking to trace a `pytest` run
-    # kill -SIGUSR1 @$(pgrep -f 'pytest')
+    log.devx(
+        'Enabling trace-trees on `SIGUSR1` '
+        'since `stackscope` is installed @ \n'
+        f'{stackscope!r}\n\n'
+        f'With `SIGUSR1` handler\n'
+        f'|_{dump_tree_on_sig}\n'
+    )
+    return stackscope

tractor/devx/pformat.py

@@ -53,6 +53,7 @@ def pformat_boxed_tb(
 
     tb_box_indent: int|None = None,
     tb_body_indent: int = 1,
+    boxer_header: str = '-'
 
 ) -> str:
     '''
@@ -88,10 +89,10 @@ def pformat_boxed_tb(
 
     tb_box: str = (
         f'|\n'
-        f' ------ - ------\n'
+        f' ------ {boxer_header} ------\n'
         f'{tb_body}'
-        f' ------ - ------\n'
-        f'_|\n'
+        f' ------ {boxer_header}- ------\n'
+        f'_|'
     )
     tb_box_indent: str = (
         tb_box_indent

tractor/hilevel/__init__.py

@@ -0,0 +1,26 @@
+# tractor: structured concurrent "actors".
+# Copyright 2024-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+High level design patterns, APIs and runtime extensions built on top
+of the `tractor` runtime core.
+
+'''
+from ._service import (
+    open_service_mngr as open_service_mngr,
+    get_service_mngr as get_service_mngr,
+    ServiceMngr as ServiceMngr,
+)

tractor/hilevel/_service.py

@@ -0,0 +1,592 @@
+# tractor: structured concurrent "actors".
+# Copyright 2024-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Daemon subactor as service(s) management and supervision primitives
+and API.
+
+'''
+from __future__ import annotations
+from contextlib import (
+    asynccontextmanager as acm,
+    # contextmanager as cm,
+)
+from collections import defaultdict
+from dataclasses import (
+    dataclass,
+    field,
+)
+import functools
+import inspect
+from typing import (
+    Callable,
+    Any,
+)
+
+import tractor
+import trio
+from trio import TaskStatus
+from tractor import (
+    log,
+    ActorNursery,
+    current_actor,
+    ContextCancelled,
+    Context,
+    Portal,
+)
+
+log = log.get_logger('tractor')
+
+
+# TODO: implement a `@singleton` deco-API for wrapping the below
+# factory's impl for general actor-singleton use?
+#
+# -[ ] go through the options peeps on SO did?
+#  * https://stackoverflow.com/questions/6760685/what-is-the-best-way-of-implementing-singleton-in-python
+#  * including @mikenerone's answer
+#   |_https://stackoverflow.com/questions/6760685/what-is-the-best-way-of-implementing-singleton-in-python/39186313#39186313
+#
+# -[ ] put it in `tractor.lowlevel._globals` ?
+#  * fits with our oustanding actor-local/global feat req?
+#   |_ https://github.com/goodboy/tractor/issues/55
+#  * how can it relate to the `Actor.lifetime_stack` that was
+#    silently patched in?
+#   |_ we could implicitly call both of these in the same
+#     spot in the runtime using the lifetime stack?
+#    - `open_singleton_cm().__exit__()`
+#    -`del_singleton()`
+#   |_ gives SC fixtue semantics to sync code oriented around
+#     sub-process lifetime?
+#  * what about with `trio.RunVar`?
+#   |_https://trio.readthedocs.io/en/stable/reference-lowlevel.html#trio.lowlevel.RunVar
+#    - which we'll need for no-GIL cpython (right?) presuming
+#      multiple `trio.run()` calls in process?
+#
+#
+# @singleton
+# async def open_service_mngr(
+#     **init_kwargs,
+# ) -> ServiceMngr:
+#     '''
+#     Note this function body is invoke IFF no existing singleton instance already
+#     exists in this proc's memory.
+
+#     '''
+#     # setup
+#     yield ServiceMngr(**init_kwargs)
+#     # teardown
+
+
+# a deletion API for explicit instance de-allocation?
+# @open_service_mngr.deleter
+# def del_service_mngr() -> None:
+#     mngr = open_service_mngr._singleton[0]
+#     open_service_mngr._singleton[0] = None
+#     del mngr
+
+
+
+# TODO: implement a singleton deco-API for wrapping the below
+# factory's impl for general actor-singleton use?
+#
+# @singleton
+# async def open_service_mngr(
+#     **init_kwargs,
+# ) -> ServiceMngr:
+#     '''
+#     Note this function body is invoke IFF no existing singleton instance already
+#     exists in this proc's memory.
+
+#     '''
+#     # setup
+#     yield ServiceMngr(**init_kwargs)
+#     # teardown
+
+
+
+# TODO: singleton factory API instead of a class API
+@acm
+async def open_service_mngr(
+    *,
+    debug_mode: bool = False,
+
+    # NOTE; since default values for keyword-args are effectively
+    # module-vars/globals as per the note from,
+    # https://docs.python.org/3/tutorial/controlflow.html#default-argument-values
+    #
+    # > "The default value is evaluated only once. This makes
+    #   a difference when the default is a mutable object such as
+    #   a list, dictionary, or instances of most classes"
+    #
+    _singleton: list[ServiceMngr|None] = [None],
+    **init_kwargs,
+
+) -> ServiceMngr:
+    '''
+    Open an actor-global "service-manager" for supervising a tree
+    of subactors and/or actor-global tasks.
+
+    The delivered `ServiceMngr` is singleton instance for each
+    actor-process, that is, allocated on first open and never
+    de-allocated unless explicitly deleted by al call to
+    `del_service_mngr()`.
+
+    '''
+    # TODO: factor this an allocation into
+    # a `._mngr.open_service_mngr()` and put in the
+    # once-n-only-once setup/`.__aenter__()` part!
+    # -[ ] how to make this only happen on the `mngr == None` case?
+    #  |_ use `.trionics.maybe_open_context()` (for generic
+    #     async-with-style-only-once of the factory impl, though
+    #     what do we do for the allocation case?
+    #    / `.maybe_open_nursery()` (since for this specific case
+    #    it's simpler?) to activate
+    async with (
+        tractor.open_nursery() as an,
+        trio.open_nursery() as tn,
+    ):
+        # impl specific obvi..
+        init_kwargs.update({
+            'an': an,
+            'tn': tn,
+        })
+
+        mngr: ServiceMngr|None
+        if (mngr := _singleton[0]) is None:
+
+            log.info('Allocating a new service mngr!')
+            mngr = _singleton[0] = ServiceMngr(**init_kwargs)
+
+            # TODO: put into `.__aenter__()` section of
+            # eventual `@singleton_acm` API wrapper.
+            #
+            # assign globally for future daemon/task creation
+            mngr.an = an
+            mngr.tn = tn
+
+        else:
+            assert (mngr.an and mngr.tn)
+            log.info(
+                'Using extant service mngr!\n\n'
+                f'{mngr!r}\n'  # it has a nice `.__repr__()` of services state
+            )
+
+        try:
+            # NOTE: this is a singleton factory impl specific detail
+            # which should be supported in the condensed
+            # `@singleton_acm` API?
+            mngr.debug_mode = debug_mode
+
+            yield mngr
+        finally:
+            # TODO: is this more clever/efficient?
+            # if 'samplerd' in mngr.service_ctxs:
+            #     await mngr.cancel_service('samplerd')
+            tn.cancel_scope.cancel()
+
+
+
+def get_service_mngr() -> ServiceMngr:
+    '''
+    Try to get the singleton service-mngr for this actor presuming it
+    has already been allocated using,
+
+    .. code:: python
+
+        async with open_<@singleton_acm(func)>() as mngr`
+            ... this block kept open ...
+
+    If not yet allocated raise a `ServiceError`.
+
+    '''
+    # https://stackoverflow.com/a/12627202
+    # https://docs.python.org/3/library/inspect.html#inspect.Signature
+    maybe_mngr: ServiceMngr|None = inspect.signature(
+        open_service_mngr
+    ).parameters['_singleton'].default[0]
+
+    if maybe_mngr is None:
+        raise RuntimeError(
+            'Someone must allocate a `ServiceMngr` using\n\n'
+            '`async with open_service_mngr()` beforehand!!\n'
+        )
+
+    return maybe_mngr
+
+
+async def _open_and_supervise_service_ctx(
+    serman: ServiceMngr,
+    name: str,
+    ctx_fn: Callable,  # TODO, type for `@tractor.context` requirement
+    portal: Portal,
+
+    allow_overruns: bool = False,
+    task_status: TaskStatus[
+        tuple[
+            trio.CancelScope,
+            Context,
+            trio.Event,
+            Any,
+        ]
+    ] = trio.TASK_STATUS_IGNORED,
+    **ctx_kwargs,
+
+) -> Any:
+    '''
+    Open a remote IPC-context defined by `ctx_fn` in the
+    (service) actor accessed via `portal` and supervise the
+    (local) parent task to termination at which point the remote
+    actor runtime is cancelled alongside it.
+
+    The main application is for allocating long-running
+    "sub-services" in a main daemon and explicitly controlling
+    their lifetimes from an actor-global singleton.
+
+    '''
+    # TODO: use the ctx._scope directly here instead?
+    # -[ ] actually what semantics do we expect for this
+    #   usage!?
+    with trio.CancelScope() as cs:
+        try:
+            async with portal.open_context(
+                ctx_fn,
+                allow_overruns=allow_overruns,
+                **ctx_kwargs,
+
+            ) as (ctx, started):
+
+                # unblock once the remote context has started
+                complete = trio.Event()
+                task_status.started((
+                    cs,
+                    ctx,
+                    complete,
+                    started,
+                ))
+                log.info(
+                    f'`pikerd` service {name} started with value {started}'
+                )
+                # wait on any context's return value
+                # and any final portal result from the
+                # sub-actor.
+                ctx_res: Any = await ctx.wait_for_result()
+
+                # NOTE: blocks indefinitely until cancelled
+                # either by error from the target context
+                # function or by being cancelled here by the
+                # surrounding cancel scope.
+                return (
+                    await portal.wait_for_result(),
+                    ctx_res,
+                )
+
+        except ContextCancelled as ctxe:
+            canceller: tuple[str, str] = ctxe.canceller
+            our_uid: tuple[str, str] = current_actor().uid
+            if (
+                canceller != portal.chan.uid
+                and
+                canceller != our_uid
+            ):
+                log.cancel(
+                    f'Actor-service `{name}` was remotely cancelled by a peer?\n'
+
+                    # TODO: this would be a good spot to use
+                    # a respawn feature Bo
+                    f'-> Keeping `pikerd` service manager alive despite this inter-peer cancel\n\n'
+
+                    f'cancellee: {portal.chan.uid}\n'
+                    f'canceller: {canceller}\n'
+                )
+            else:
+                raise
+
+        finally:
+            # NOTE: the ctx MUST be cancelled first if we
+            # don't want the above `ctx.wait_for_result()` to
+            # raise a self-ctxc. WHY, well since from the ctx's
+            # perspective the cancel request will have
+            # arrived out-out-of-band at the `Actor.cancel()`
+            # level, thus `Context.cancel_called == False`,
+            # meaning `ctx._is_self_cancelled() == False`.
+            # with trio.CancelScope(shield=True):
+            # await ctx.cancel()
+            await portal.cancel_actor()  # terminate (remote) sub-actor
+            complete.set()  # signal caller this task is done
+            serman.service_ctxs.pop(name)  # remove mngr entry
+
+
+# TODO: we need remote wrapping and a general soln:
+# - factor this into a ``tractor.highlevel`` extension # pack for the
+#   library.
+# - wrap a "remote api" wherein you can get a method proxy
+#   to the pikerd actor for starting services remotely!
+# - prolly rename this to ActorServicesNursery since it spawns
+#   new actors and supervises them to completion?
+@dataclass
+class ServiceMngr:
+    '''
+    A multi-subactor-as-service manager.
+
+    Spawn, supervise and monitor service/daemon subactors in a SC
+    process tree.
+
+    '''
+    an: ActorNursery
+    tn: trio.Nursery
+    debug_mode: bool = False # tractor sub-actor debug mode flag
+
+    service_tasks: dict[
+        str,
+        tuple[
+            trio.CancelScope,
+            trio.Event,
+        ]
+    ] = field(default_factory=dict)
+
+    service_ctxs: dict[
+        str,
+        tuple[
+            trio.CancelScope,
+            Context,
+            Portal,
+            trio.Event,
+        ]
+    ] = field(default_factory=dict)
+
+    # internal per-service task mutexs
+    _locks = defaultdict(trio.Lock)
+
+    # TODO, unify this interface with our `TaskManager` PR!
+    #
+    #
+    async def start_service_task(
+        self,
+        name: str,
+        # TODO: typevar for the return type of the target and then
+        # use it below for `ctx_res`?
+        fn: Callable,
+
+        allow_overruns: bool = False,
+        **ctx_kwargs,
+
+    ) -> tuple[
+        trio.CancelScope,
+        Any,
+        trio.Event,
+    ]:
+        async def _task_manager_start(
+            task_status: TaskStatus[
+                tuple[
+                    trio.CancelScope,
+                    trio.Event,
+                ]
+            ] = trio.TASK_STATUS_IGNORED,
+        ) -> Any:
+
+            task_cs = trio.CancelScope()
+            task_complete = trio.Event()
+
+            with task_cs as cs:
+                task_status.started((
+                    cs,
+                    task_complete,
+                ))
+                try:
+                    await fn()
+                except trio.Cancelled as taskc:
+                    log.cancel(
+                        f'Service task for `{name}` was cancelled!\n'
+                        # TODO: this would be a good spot to use
+                        # a respawn feature Bo
+                    )
+                    raise taskc
+                finally:
+                    task_complete.set()
+        (
+            cs,
+            complete,
+        ) = await self.tn.start(_task_manager_start)
+
+        # store the cancel scope and portal for later cancellation or
+        # retstart if needed.
+        self.service_tasks[name] = (
+            cs,
+            complete,
+        )
+        return (
+            cs,
+            complete,
+        )
+
+    async def cancel_service_task(
+        self,
+        name: str,
+
+    ) -> Any:
+        log.info(f'Cancelling `pikerd` service {name}')
+        cs, complete = self.service_tasks[name]
+
+        cs.cancel()
+        await complete.wait()
+        # TODO, if we use the `TaskMngr` from #346
+        # we can also get the return value from the task!
+
+        if name in self.service_tasks:
+            # TODO: custom err?
+            # raise ServiceError(
+            raise RuntimeError(
+                f'Service task {name!r} not terminated!?\n'
+            )
+
+    async def start_service_ctx(
+        self,
+        name: str,
+        portal: Portal,
+        # TODO: typevar for the return type of the target and then
+        # use it below for `ctx_res`?
+        ctx_fn: Callable,
+        **ctx_kwargs,
+
+    ) -> tuple[
+        trio.CancelScope,
+        Context,
+        Any,
+    ]:
+        '''
+        Start a remote IPC-context defined by `ctx_fn` in a background
+        task and immediately return supervision primitives to manage it:
+
+        - a `cs: CancelScope` for the newly allocated bg task
+        - the `ipc_ctx: Context` to manage the remotely scheduled
+          `trio.Task`.
+        - the `started: Any` value returned by the remote endpoint
+          task's `Context.started(<value>)` call.
+
+        The bg task supervises the ctx such that when it terminates the supporting
+        actor runtime is also cancelled, see `_open_and_supervise_service_ctx()`
+        for details.
+
+        '''
+        cs, ipc_ctx, complete, started = await self.tn.start(
+            functools.partial(
+                _open_and_supervise_service_ctx,
+                serman=self,
+                name=name,
+                ctx_fn=ctx_fn,
+                portal=portal,
+                **ctx_kwargs,
+            )
+        )
+
+        # store the cancel scope and portal for later cancellation or
+        # retstart if needed.
+        self.service_ctxs[name] = (cs, ipc_ctx, portal, complete)
+        return (
+            cs,
+            ipc_ctx,
+            started,
+        )
+
+    async def start_service(
+        self,
+        daemon_name: str,
+        ctx_ep: Callable,  # kwargs must `partial`-ed in!
+        # ^TODO, type for `@tractor.context` deco-ed funcs!
+
+        debug_mode: bool = False,
+        **start_actor_kwargs,
+
+    ) -> Context:
+        '''
+        Start new subactor and schedule a supervising "service task"
+        in it which explicitly defines the sub's lifetime.
+
+        "Service daemon subactors" are cancelled (and thus
+        terminated) using the paired `.cancel_service()`.
+
+        Effectively this API can be used to manage "service daemons"
+        spawned under a single parent actor with supervision
+        semantics equivalent to a one-cancels-one style actor-nursery
+        or "(subactor) task manager" where each subprocess's (and
+        thus its embedded actor runtime) lifetime is synced to that
+        of the remotely spawned task defined by `ctx_ep`.
+
+        The funcionality can be likened to a "daemonized" version of
+        `.hilevel.worker.run_in_actor()` but with supervision
+        controls offered by `tractor.Context` where the main/root
+        remotely scheduled `trio.Task` invoking `ctx_ep` determines
+        the underlying subactor's lifetime.
+
+        '''
+        entry: tuple|None = self.service_ctxs.get(daemon_name)
+        if entry:
+            (cs, sub_ctx, portal, complete) = entry
+            return sub_ctx
+
+        if daemon_name not in self.service_ctxs:
+            portal: Portal = await self.an.start_actor(
+                daemon_name,
+                debug_mode=(  # maybe set globally during allocate
+                    debug_mode
+                    or
+                    self.debug_mode
+                ),
+                **start_actor_kwargs,
+            )
+            ctx_kwargs: dict[str, Any] = {}
+            if isinstance(ctx_ep, functools.partial):
+                ctx_kwargs: dict[str, Any] = ctx_ep.keywords
+                ctx_ep: Callable = ctx_ep.func
+
+            (
+                cs,
+                sub_ctx,
+                started,
+            ) = await self.start_service_ctx(
+                name=daemon_name,
+                portal=portal,
+                ctx_fn=ctx_ep,
+                **ctx_kwargs,
+            )
+
+            return sub_ctx
+
+    async def cancel_service(
+        self,
+        name: str,
+
+    ) -> Any:
+        '''
+        Cancel the service task and actor for the given ``name``.
+
+        '''
+        log.info(f'Cancelling `pikerd` service {name}')
+        cs, sub_ctx, portal, complete = self.service_ctxs[name]
+
+        # cs.cancel()
+        await sub_ctx.cancel()
+        await complete.wait()
+
+        if name in self.service_ctxs:
+            # TODO: custom err?
+            # raise ServiceError(
+            raise RuntimeError(
+                f'Service actor for {name} not terminated and/or unknown?'
+            )
+
+        # assert name not in self.service_ctxs, \
+        #     f'Serice task for {name} not terminated?'

tractor/log.py

@@ -258,20 +258,28 @@ class ActorContextInfo(Mapping):
 
 
 def get_logger(
-
-    name: str | None = None,
+    name: str|None = None,
     _root_name: str = _proj_name,
 
+    logger: Logger|None = None,
+
+    # TODO, using `.config.dictConfig()` api?
+    # -[ ] SO answer with docs links
+    #  |_https://stackoverflow.com/questions/7507825/where-is-a-complete-example-of-logging-config-dictconfig
+    #  |_https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema
+    subsys_spec: str|None = None,
+
 ) -> StackLevelAdapter:
     '''Return the package log or a sub-logger for ``name`` if provided.
 
     '''
     log: Logger
-    log = rlog = logging.getLogger(_root_name)
+    log = rlog = logger or logging.getLogger(_root_name)
 
     if (
         name
-        and name != _proj_name
+        and
+        name != _proj_name
     ):
 
         # NOTE: for handling for modules that use ``get_logger(__name__)``
@@ -283,7 +291,7 @@ def get_logger(
         #   since in python the {filename} is always this same
         #   module-file.
 
-        sub_name: None | str = None
+        sub_name: None|str = None
         rname, _, sub_name = name.partition('.')
         pkgpath, _, modfilename = sub_name.rpartition('.')
 
@@ -306,7 +314,10 @@ def get_logger(
 
     # add our actor-task aware adapter which will dynamically look up
     # the actor and task names at each log emit
-    logger = StackLevelAdapter(log, ActorContextInfo())
+    logger = StackLevelAdapter(
+        log,
+        ActorContextInfo(),
+    )
 
     # additional levels
     for name, val in CUSTOM_LEVELS.items():
@@ -319,15 +330,25 @@ def get_logger(
 
 
 def get_console_log(
-    level: str | None = None,
+    level: str|None = None,
+    logger: Logger|None = None,
     **kwargs,
-) -> LoggerAdapter:
-    '''Get the package logger and enable a handler which writes to stderr.
 
-    Yeah yeah, i know we can use ``DictConfig``. You do it.
+) -> LoggerAdapter:
     '''
-    log = get_logger(**kwargs)  # our root logger
-    logger = log.logger
+    Get a `tractor`-style logging instance: a `Logger` wrapped in
+    a `StackLevelAdapter` which injects various concurrency-primitive
+    (process, thread, task) fields and enables a `StreamHandler` that
+    writes on stderr using `colorlog` formatting.
+
+    Yeah yeah, i know we can use `logging.config.dictConfig()`. You do it.
+
+    '''
+    log = get_logger(
+        logger=logger,
+        **kwargs
+    )  # set a root logger
+    logger: Logger = log.logger
 
     if not level:
         return log
@@ -346,9 +367,13 @@ def get_console_log(
             None,
         )
     ):
+        fmt = LOG_FORMAT
+        # if logger:
+        #     fmt = None
+
         handler = StreamHandler()
         formatter = colorlog.ColoredFormatter(
-            LOG_FORMAT,
+            fmt=fmt,
             datefmt=DATE_FORMAT,
             log_colors=STD_PALETTE,
             secondary_log_colors=BOLD_PALETTE,
@@ -365,7 +390,7 @@ def get_loglevel() -> str:
 
 
 # global module logger for tractor itself
-log = get_logger('tractor')
+log: StackLevelAdapter = get_logger('tractor')
 
 
 def at_least_level(

tractor/msg/_codec.py

@@ -41,8 +41,10 @@ import textwrap
 from typing import (
     Any,
     Callable,
+    Protocol,
     Type,
     TYPE_CHECKING,
+    TypeVar,
     Union,
 )
 from types import ModuleType
@@ -181,7 +183,11 @@ def mk_dec(
     dec_hook: Callable|None = None,
 
 ) -> MsgDec:
+    '''
+    Create an IPC msg decoder, normally used as the
+    `PayloadMsg.pld: PayloadT` field decoder inside a `PldRx`.
 
+    '''
     return MsgDec(
         _dec=msgpack.Decoder(
             type=spec,  # like `MsgType[Any]`
@@ -227,6 +233,13 @@ def pformat_msgspec(
     join_char: str = '\n',
 
 ) -> str:
+    '''
+    Pretty `str` format the `msgspec.msgpack.Decoder.type` attribute
+    for display in (console) log messages as a nice (maybe multiline)
+    presentation of all supported `Struct`s (subtypes) available for
+    typed decoding.
+
+    '''
     dec: msgpack.Decoder = getattr(codec, 'dec', codec)
     return join_char.join(
         mk_msgspec_table(
@@ -630,31 +643,57 @@ def limit_msg_spec(
 #         # import pdbp; pdbp.set_trace()
 #         assert ext_codec.pld_spec == extended_spec
 #         yield ext_codec
+#
+# ^-TODO-^ is it impossible to make something like this orr!?
+
+# TODO: make an auto-custom hook generator from a set of input custom
+# types?
+# -[ ] below is a proto design using a `TypeCodec` idea?
+#
+# type var for the expected interchange-lib's
+# IPC-transport type when not available as a built-in
+# serialization output.
+WireT = TypeVar('WireT')
 
 
-# TODO: make something similar to this inside `._codec` such that
-# user can just pass a type table of some sort?
-# -[ ] we would need to decode all msgs to `pretty_struct.Struct`
-#     and then call `.to_dict()` on them?
-# -[x] we're going to need to re-impl all the stuff changed in the
-#    runtime port such that it can handle dicts or `Msg`s?
-#
-# def mk_dict_msg_codec_hooks() -> tuple[Callable, Callable]:
-#     '''
-#     Deliver a `enc_hook()`/`dec_hook()` pair which does
-#     manual convertion from our above native `Msg` set
-#     to `dict` equivalent (wire msgs) in order to keep legacy compat
-#     with the original runtime implementation.
-#
-#     Note: this is is/was primarly used while moving the core
-#     runtime over to using native `Msg`-struct types wherein we
-#     start with the send side emitting without loading
-#     a typed-decoder and then later flipping the switch over to
-#     load to the native struct types once all runtime usage has
-#     been adjusted appropriately.
-#
-#     '''
-#     return (
-#         # enc_to_dict,
-#         dec_from_dict,
-#     )
+# TODO: some kinda (decorator) API for built-in subtypes
+# that builds this implicitly by inspecting the `mro()`?
+class TypeCodec(Protocol):
+    '''
+    A per-custom-type wire-transport serialization translator
+    description type.
+
+    '''
+    src_type: Type
+    wire_type: WireT
+
+    def encode(obj: Type) -> WireT:
+        ...
+
+    def decode(
+        obj_type: Type[WireT],
+        obj: WireT,
+    ) -> Type:
+        ...
+
+
+class MsgpackTypeCodec(TypeCodec):
+    ...
+
+
+def mk_codec_hooks(
+    type_codecs: list[TypeCodec],
+
+) -> tuple[Callable, Callable]:
+    '''
+    Deliver a `enc_hook()`/`dec_hook()` pair which handle
+    manual convertion from an input `Type` set such that whenever
+    the `TypeCodec.filter()` predicate matches the
+    `TypeCodec.decode()` is called on the input native object by
+    the `dec_hook()` and whenever the
+    `isiinstance(obj, TypeCodec.type)` matches against an
+    `enc_hook(obj=obj)` the return value is taken from a
+    `TypeCodec.encode(obj)` callback.
+
+    '''
+    ...

tractor/msg/_ops.py

@@ -258,6 +258,9 @@ class PldRx(Struct):
                         f'|_pld={pld!r}\n'
                     )
                     return pld
+                except TypeError as typerr:
+                    __tracebackhide__: bool = False
+                    raise typerr
 
                 # XXX pld-value type failure
                 except ValidationError as valerr:
@@ -796,8 +799,14 @@ def validate_payload_msg(
     __tracebackhide__: bool = hide_tb
     codec: MsgCodec = current_codec()
     msg_bytes: bytes = codec.encode(pld_msg)
+    roundtripped: Started|None = None
     try:
         roundtripped: Started = codec.decode(msg_bytes)
+    except TypeError as typerr:
+        __tracebackhide__: bool = False
+        raise typerr
+
+    try:
         ctx: Context = getattr(ipc, 'ctx', ipc)
         pld: PayloadT = ctx.pld_rx.decode_pld(
             msg=roundtripped,
@@ -822,6 +831,11 @@ def validate_payload_msg(
             )
             raise ValidationError(complaint)
 
+    # usually due to `.decode()` input type
+    except TypeError as typerr:
+        __tracebackhide__: bool = False
+        raise typerr
+
     # raise any msg type error NO MATTER WHAT!
     except ValidationError as verr:
         try:
@@ -832,9 +846,13 @@ def validate_payload_msg(
                 verb_header='Trying to send ',
                 is_invalid_payload=True,
             )
-        except BaseException:
+        except BaseException as _be:
+            if not roundtripped:
+                raise verr
+
+            be = _be
             __tracebackhide__: bool = False
-            raise
+            raise be
 
         if not raise_mte:
             return mte

tractor/msg/pretty_struct.py

@@ -30,9 +30,9 @@ from msgspec import (
     Struct as _Struct,
     structs,
 )
-from pprint import (
-    saferepr,
-)
+# from pprint import (
+#     saferepr,
+# )
 
 from tractor.log import get_logger
 
@@ -75,8 +75,8 @@ class DiffDump(UserList):
         for k, left, right in self:
             repstr += (
                 f'({k},\n'
-                f'\t{repr(left)},\n'
-                f'\t{repr(right)},\n'
+                f' |_{repr(left)},\n'
+                f' |_{repr(right)},\n'
                 ')\n'
             )
         repstr += ']\n'
@@ -144,15 +144,22 @@ def pformat(
                 field_indent=indent + field_indent,
             )
 
-        else:  # the `pprint` recursion-safe format:
+        else:
+            val_str: str = repr(v)
+
+            # XXX LOL, below just seems to be f#$%in causing
+            # recursion errs..
+            #
+            # the `pprint` recursion-safe format:
             # https://docs.python.org/3.11/library/pprint.html#pprint.saferepr
-            try:
-                val_str: str = saferepr(v)
-            except Exception:
-                log.exception(
-                    'Failed to `saferepr({type(struct)})` !?\n'
-                )
-            return _Struct.__repr__(struct)
+            # try:
+            #     val_str: str = saferepr(v)
+            # except Exception:
+            #     log.exception(
+            #         'Failed to `saferepr({type(struct)})` !?\n'
+            #     )
+                # raise
+                # return _Struct.__repr__(struct)
 
         # TODO: LOLOL use `textwrap.indent()` instead dawwwwwg!
         obj_str += (field_ws + f'{k}: {typ_name} = {val_str},\n')
@@ -203,12 +210,7 @@ class Struct(
         return sin_props
 
     pformat = pformat
-    # __repr__ = pformat
-    # __str__ = __repr__ = pformat
-    # TODO: use a pprint.PrettyPrinter instance around ONLY rendering
-    # inside a known tty?
-    # def __repr__(self) -> str:
-    #     ...
+
     def __repr__(self) -> str:
         try:
             return pformat(self)
@@ -218,6 +220,13 @@ class Struct(
             )
             return _Struct.__repr__(self)
 
+    # __repr__ = pformat
+    # __str__ = __repr__ = pformat
+    # TODO: use a pprint.PrettyPrinter instance around ONLY rendering
+    # inside a known tty?
+    # def __repr__(self) -> str:
+    #     ...
+
     def copy(
         self,
         update: dict | None = None,
@@ -267,13 +276,15 @@ class Struct(
                 fi.type(getattr(self, fi.name)),
             )
 
+    # TODO: make a mod func instead and just point to it here for
+    # method impl?
     def __sub__(
         self,
         other: Struct,
 
     ) -> DiffDump[tuple[str, Any, Any]]:
         '''
-        Compare fields/items key-wise and return a ``DiffDump``
+        Compare fields/items key-wise and return a `DiffDump`
         for easy visual REPL comparison B)
 
         '''
@@ -290,3 +301,42 @@ class Struct(
                 ))
 
         return diffs
+
+    @classmethod
+    def fields_diff(
+        cls,
+        other: dict|Struct,
+
+    ) -> DiffDump[tuple[str, Any, Any]]:
+        '''
+        Very similar to `PrettyStruct.__sub__()` except accepts an
+        input `other: dict` (presumably that would normally be called
+        like `Struct(**other)`) which returns a `DiffDump` of the
+        fields of the struct and the `dict`'s fields.
+
+        '''
+        nullish = object()
+        consumed: dict = other.copy()
+        diffs: DiffDump[tuple[str, Any, Any]] = DiffDump()
+        for fi in structs.fields(cls):
+            field_name: str = fi.name
+            # ours: Any = getattr(self, field_name)
+            theirs: Any = consumed.pop(field_name, nullish)
+            if theirs is nullish:
+                diffs.append((
+                    field_name,
+                    f'{fi.type!r}',
+                    'NOT-DEFINED in `other: dict`',
+                ))
+
+        # when there are lingering fields in `other` that this struct
+        # DOES NOT define we also append those.
+        if consumed:
+            for k, v in consumed.items():
+                diffs.append((
+                    k,
+                    f'NOT-DEFINED for `{cls.__name__}`',
+                    f'`other: dict` has value = {v!r}',
+                ))
+
+        return diffs

tractor/trionics/__init__.py

@@ -29,3 +29,6 @@ from ._broadcast import (
     BroadcastReceiver as BroadcastReceiver,
     Lagged as Lagged,
 )
+from ._beg import (
+    collapse_eg as collapse_eg,
+)

tractor/trionics/_beg.py

@@ -0,0 +1,58 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+`BaseExceptionGroup` related utils and helpers pertaining to
+first-class-`trio` from a historical perspective B)
+
+'''
+from contextlib import (
+    asynccontextmanager as acm,
+)
+
+
+def maybe_collapse_eg(
+    beg: BaseExceptionGroup,
+) -> BaseException:
+    '''
+    If the input beg can collapse to a single non-eg sub-exception,
+    return it instead.
+
+    '''
+    if len(excs := beg.exceptions) == 1:
+        return excs[0]
+
+    return beg
+
+
+@acm
+async def collapse_eg():
+    '''
+    If `BaseExceptionGroup` raised in the body scope is
+    "collapse-able" (in the same way that
+    `trio.open_nursery(strict_exception_groups=False)` works) then
+    only raise the lone emedded non-eg in in place.
+
+    '''
+    try:
+        yield
+    except* BaseException as beg:
+        if (
+            exc := maybe_collapse_eg(beg)
+        ) is not beg:
+            raise exc
+
+        raise beg

tractor/trionics/_broadcast.py

@@ -15,14 +15,14 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 '''
-``tokio`` style broadcast channel.
+`tokio` style broadcast channel.
 https://docs.rs/tokio/1.11.0/tokio/sync/broadcast/index.html
 
 '''
 from __future__ import annotations
 from abc import abstractmethod
 from collections import deque
-from contextlib import asynccontextmanager
+from contextlib import asynccontextmanager as acm
 from functools import partial
 from operator import ne
 from typing import (
@@ -382,7 +382,7 @@ class BroadcastReceiver(ReceiveChannel):
                         # likely it makes sense to unwind back to the
                         # underlying?
                         # import tractor
-                        # await tractor.breakpoint()
+                        # await tractor.pause()
                         log.warning(
                             f'Only one sub left for {self}?\n'
                             'We can probably unwind from breceiver?'
@@ -398,7 +398,7 @@ class BroadcastReceiver(ReceiveChannel):
 
             return await self._receive_from_underlying(key, state)
 
-    @asynccontextmanager
+    @acm
     async def subscribe(
         self,
         raise_on_lag: bool = True,

tractor/trionics/_mngrs.py

@@ -57,6 +57,8 @@ async def maybe_open_nursery(
     shield: bool = False,
     lib: ModuleType = trio,
 
+    **kwargs,  # proxy thru
+
 ) -> AsyncGenerator[trio.Nursery, Any]:
     '''
     Create a new nursery if None provided.
@@ -67,7 +69,7 @@ async def maybe_open_nursery(
     if nursery is not None:
         yield nursery
     else:
-        async with lib.open_nursery() as nursery:
+        async with lib.open_nursery(**kwargs) as nursery:
             nursery.cancel_scope.shield = shield
             yield nursery
 
@@ -143,9 +145,14 @@ async def gather_contexts(
             'Use a non-lazy iterator or sequence type intead!'
         )
 
-    async with trio.open_nursery() as n:
+    async with trio.open_nursery(
+        strict_exception_groups=False,
+        # ^XXX^ TODO? soo roll our own then ??
+        # -> since we kinda want the "if only one `.exception` then
+        # just raise that" interface?
+    ) as tn:
         for mngr in mngrs:
-            n.start_soon(
+            tn.start_soon(
                 _enter_and_wait,
                 mngr,
                 unwrapped,

tractor/trionics/_supervisor.py

@@ -0,0 +1,322 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Erlang-style (ish) "one-cancels-one" nursery.
+
+'''
+from __future__ import annotations
+from contextlib import (
+    asynccontextmanager as acm,
+    contextmanager as cm,
+)
+from functools import partial
+from typing import (
+    Generator,
+    Any,
+)
+
+from outcome import (
+    Outcome,
+    acapture,
+)
+from msgspec import Struct
+import trio
+from trio._core._run import (
+    Task,
+    CancelScope,
+    Nursery,
+)
+
+
+class TaskOutcome(Struct):
+    '''
+    The outcome of a scheduled ``trio`` task which includes an interface
+    for synchronizing to the completion of the task's runtime and access
+    to the eventual boxed result/value or raised exception.
+
+    '''
+    lowlevel_task: Task
+    _exited = trio.Event()  # as per `trio.Runner.task_exited()`
+    _outcome: Outcome | None = None  # as per `outcome.Outcome`
+    _result: Any | None = None  # the eventual maybe-returned-value
+
+    @property
+    def result(self) -> Any:
+        '''
+        Either Any or None depending on whether the Outcome has compeleted.
+
+        '''
+        if self._outcome is None:
+            raise RuntimeError(
+                f'Task {self.lowlevel_task.name} is not complete.\n'
+                'First wait on `await TaskOutcome.wait_for_result()`!'
+            )
+        return self._result
+
+    def _set_outcome(
+        self,
+        outcome: Outcome,
+    ):
+        '''
+        Set the ``Outcome`` for this task.
+
+        This method should only ever be called by the task's supervising
+        nursery implemenation.
+
+        '''
+        self._outcome = outcome
+        self._result = outcome.unwrap()
+        self._exited.set()
+
+    async def wait_for_result(self) -> Any:
+        '''
+        Unwind the underlying task's ``Outcome`` by async waiting for
+        the task to first complete and then unwrap it's result-value.
+
+        '''
+        if self._exited.is_set():
+            return self._result
+
+        await self._exited.wait()
+
+        out = self._outcome
+        if out is None:
+            raise ValueError(f'{out} is not an outcome!?')
+
+        return self.result
+
+
+class TaskManagerNursery(Struct):
+    _n: Nursery
+    _scopes: dict[
+        Task,
+        tuple[CancelScope, Outcome]
+    ] = {}
+
+    task_manager: Generator[Any, Outcome, None] | None = None
+
+    async def start_soon(
+        self,
+        async_fn,
+        *args,
+
+        name=None,
+        task_manager: Generator[Any, Outcome, None] | None = None
+
+    ) -> tuple[CancelScope, Task]:
+
+        # NOTE: internals of a nursery don't let you know what
+        # the most recently spawned task is by order.. so we'd
+        # have to either change that or do set ops.
+        # pre_start_tasks: set[Task] = n._children.copy()
+        # new_tasks = n._children - pre_start_Tasks
+        # assert len(new_tasks) == 1
+        # task = new_tasks.pop()
+
+        n: Nursery = self._n
+
+        sm = self.task_manager
+        # we do default behavior of a scope-per-nursery
+        # if the user did not provide a task manager.
+        if sm is None:
+            return n.start_soon(async_fn, *args, name=None)
+
+        new_task: Task | None = None
+        to_return: tuple[Any] | None = None
+
+        # NOTE: what do we enforce as a signature for the
+        # `@task_scope_manager` here?
+        mngr = sm(nursery=n)
+
+        async def _start_wrapped_in_scope(
+            task_status: TaskStatus[
+                tuple[CancelScope, Task]
+            ] = trio.TASK_STATUS_IGNORED,
+
+        ) -> None:
+
+            # TODO: this was working before?! and, do we need something
+            # like it to implement `.start()`?
+            # nonlocal to_return
+
+            # execute up to the first yield
+            try:
+                to_return: tuple[Any] = next(mngr)
+            except StopIteration:
+                raise RuntimeError("task manager didn't yield") from None
+
+            # TODO: how do we support `.start()` style?
+            # - relay through whatever the
+            #   started task passes back via `.started()` ?
+            #   seems like that won't work with also returning
+            #   a "task handle"?
+            # - we were previously binding-out this `to_return` to
+            #   the parent's lexical scope, why isn't that working
+            #   now?
+            task_status.started(to_return)
+
+            # invoke underlying func now that cs is entered.
+            outcome = await acapture(async_fn, *args)
+
+            # execute from the 1st yield to return and expect
+            # generator-mngr `@task_scope_manager` thinger to
+            # terminate!
+            try:
+                mngr.send(outcome)
+
+
+                # I would presume it's better to have a handle to
+                # the `Outcome` entirely? This method sends *into*
+                # the mngr this `Outcome.value`; seems like kinda
+                # weird semantics for our purposes?
+                # outcome.send(mngr)
+
+            except StopIteration:
+                return
+            else:
+                raise RuntimeError(f"{mngr} didn't stop!")
+
+        to_return = await n.start(_start_wrapped_in_scope)
+        assert to_return is not None
+
+        # TODO: use the fancy type-check-time type signature stuff from
+        # mypy i guess..to like, relay the type of whatever the
+        # generator yielded through? betcha that'll be un-grokable XD
+        return to_return
+
+
+# TODO: define a decorator to runtime type check that this a generator
+# with a single yield that also delivers a value (of some std type) from
+# the yield expression?
+# @trio.task_manager
+def add_task_handle_and_crash_handling(
+    nursery: Nursery,
+
+    debug_mode: bool = False,
+
+) -> Generator[
+    Any,
+    Outcome,
+    None,
+]:
+    '''
+    A customizable, user defined "task scope manager".
+
+    With this specially crafted single-yield generator function you can
+    add more granular controls around every task spawned by `trio` B)
+
+    '''
+    # if you need it you can ask trio for the task obj
+    task: Task = trio.lowlevel.current_task()
+    print(f'Spawning task: {task.name}')
+
+    # User defined "task handle" for more granular supervision
+    # of each spawned task as needed for their particular usage.
+    task_outcome = TaskOutcome(task)
+
+    # NOTE: if wanted the user could wrap the output task handle however
+    # they want!
+    # class TaskHandle(Struct):
+    #     task: Task
+    #     cs: CancelScope
+    #     outcome: TaskOutcome
+
+    # this yields back when the task is terminated, cancelled or returns.
+    try:
+        with CancelScope() as cs:
+
+            # the yielded value(s) here are what are returned to the
+            # nursery's `.start_soon()` caller B)
+            lowlevel_outcome: Outcome = yield (task_outcome, cs)
+            task_outcome._set_outcome(lowlevel_outcome)
+
+    # Adds "crash handling" from `pdbp` by entering
+    # a REPL on std errors.
+    except Exception as err:
+        print(f'{task.name} crashed, entering debugger!')
+        if debug_mode:
+            import pdbp
+            pdbp.xpm()
+        raise
+
+    finally:
+        print(f'{task.name} Exitted')
+
+
+@acm
+async def open_nursery(
+    task_manager: Generator[Any, Outcome, None] | None = None,
+
+    **lowlevel_nursery_kwargs,
+):
+    async with trio.open_nursery(**lowlevel_nursery_kwargs) as nurse:
+        yield TaskManagerNursery(
+            nurse,
+            task_manager=task_manager,
+        )
+
+
+async def sleep_then_return_val(val: str):
+    await trio.sleep(0.2)
+    return val
+
+
+async def ensure_cancelled():
+    try:
+        await trio.sleep_forever()
+
+    except trio.Cancelled:
+        task = trio.lowlevel.current_task()
+        print(f'heyyo ONLY {task.name} was cancelled as expected B)')
+        assert 0
+
+    except BaseException:
+        raise RuntimeError("woa woa woa this ain't right!")
+
+
+if __name__ == '__main__':
+
+    async def main():
+        async with open_nursery(
+            task_manager=partial(
+                add_task_handle_and_crash_handling,
+                debug_mode=True,
+            ),
+        ) as sn:
+            for _ in range(3):
+                outcome, _ = await sn.start_soon(trio.sleep_forever)
+
+            # extra task we want to engage in debugger post mortem.
+            err_outcome, cs = await sn.start_soon(ensure_cancelled)
+
+            val: str = 'yoyoyo'
+            val_outcome, _ = await sn.start_soon(
+                sleep_then_return_val,
+                val,
+            )
+            res = await val_outcome.wait_for_result()
+            assert res == val
+            print(f'{res} -> GOT EXPECTED TASK VALUE')
+
+            await trio.sleep(0.6)
+            print(
+                f'Cancelling and waiting on {err_outcome.lowlevel_task} '
+                'to CRASH..'
+            )
+            cs.cancel()
+
+    trio.run(main)

uv.lock

@@ -126,7 +126,31 @@ wheels = [
 [[package]]
 name = "msgspec"
 version = "0.19.0"
-source = { git = "https://github.com/jcrist/msgspec.git#dd965dce22e5278d4935bea923441ecde31b5325" }
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/cf/9b/95d8ce458462b8b71b8a70fa94563b2498b89933689f3a7b8911edfae3d7/msgspec-0.19.0.tar.gz", hash = "sha256:604037e7cd475345848116e89c553aa9a233259733ab51986ac924ab1b976f8e", size = 216934 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/24/d4/2ec2567ac30dab072cce3e91fb17803c52f0a37aab6b0c24375d2b20a581/msgspec-0.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aa77046904db764b0462036bc63ef71f02b75b8f72e9c9dd4c447d6da1ed8f8e", size = 187939 },
+    { url = "https://files.pythonhosted.org/packages/2b/c0/18226e4328897f4f19875cb62bb9259fe47e901eade9d9376ab5f251a929/msgspec-0.19.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:047cfa8675eb3bad68722cfe95c60e7afabf84d1bd8938979dd2b92e9e4a9551", size = 182202 },
+    { url = "https://files.pythonhosted.org/packages/81/25/3a4b24d468203d8af90d1d351b77ea3cffb96b29492855cf83078f16bfe4/msgspec-0.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e78f46ff39a427e10b4a61614a2777ad69559cc8d603a7c05681f5a595ea98f7", size = 209029 },
+    { url = "https://files.pythonhosted.org/packages/85/2e/db7e189b57901955239f7689b5dcd6ae9458637a9c66747326726c650523/msgspec-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c7adf191e4bd3be0e9231c3b6dc20cf1199ada2af523885efc2ed218eafd011", size = 210682 },
+    { url = "https://files.pythonhosted.org/packages/03/97/7c8895c9074a97052d7e4a1cc1230b7b6e2ca2486714eb12c3f08bb9d284/msgspec-0.19.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:f04cad4385e20be7c7176bb8ae3dca54a08e9756cfc97bcdb4f18560c3042063", size = 214003 },
+    { url = "https://files.pythonhosted.org/packages/61/61/e892997bcaa289559b4d5869f066a8021b79f4bf8e955f831b095f47a4cd/msgspec-0.19.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:45c8fb410670b3b7eb884d44a75589377c341ec1392b778311acdbfa55187716", size = 216833 },
+    { url = "https://files.pythonhosted.org/packages/ce/3d/71b2dffd3a1c743ffe13296ff701ee503feaebc3f04d0e75613b6563c374/msgspec-0.19.0-cp311-cp311-win_amd64.whl", hash = "sha256:70eaef4934b87193a27d802534dc466778ad8d536e296ae2f9334e182ac27b6c", size = 186184 },
+    { url = "https://files.pythonhosted.org/packages/b2/5f/a70c24f075e3e7af2fae5414c7048b0e11389685b7f717bb55ba282a34a7/msgspec-0.19.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f98bd8962ad549c27d63845b50af3f53ec468b6318400c9f1adfe8b092d7b62f", size = 190485 },
+    { url = "https://files.pythonhosted.org/packages/89/b0/1b9763938cfae12acf14b682fcf05c92855974d921a5a985ecc197d1c672/msgspec-0.19.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:43bbb237feab761b815ed9df43b266114203f53596f9b6e6f00ebd79d178cdf2", size = 183910 },
+    { url = "https://files.pythonhosted.org/packages/87/81/0c8c93f0b92c97e326b279795f9c5b956c5a97af28ca0fbb9fd86c83737a/msgspec-0.19.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cfc033c02c3e0aec52b71710d7f84cb3ca5eb407ab2ad23d75631153fdb1f12", size = 210633 },
+    { url = "https://files.pythonhosted.org/packages/d0/ef/c5422ce8af73928d194a6606f8ae36e93a52fd5e8df5abd366903a5ca8da/msgspec-0.19.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d911c442571605e17658ca2b416fd8579c5050ac9adc5e00c2cb3126c97f73bc", size = 213594 },
+    { url = "https://files.pythonhosted.org/packages/19/2b/4137bc2ed45660444842d042be2cf5b18aa06efd2cda107cff18253b9653/msgspec-0.19.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:757b501fa57e24896cf40a831442b19a864f56d253679f34f260dcb002524a6c", size = 214053 },
+    { url = "https://files.pythonhosted.org/packages/9d/e6/8ad51bdc806aac1dc501e8fe43f759f9ed7284043d722b53323ea421c360/msgspec-0.19.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5f0f65f29b45e2816d8bded36e6b837a4bf5fb60ec4bc3c625fa2c6da4124537", size = 219081 },
+    { url = "https://files.pythonhosted.org/packages/b1/ef/27dd35a7049c9a4f4211c6cd6a8c9db0a50647546f003a5867827ec45391/msgspec-0.19.0-cp312-cp312-win_amd64.whl", hash = "sha256:067f0de1c33cfa0b6a8206562efdf6be5985b988b53dd244a8e06f993f27c8c0", size = 187467 },
+    { url = "https://files.pythonhosted.org/packages/3c/cb/2842c312bbe618d8fefc8b9cedce37f773cdc8fa453306546dba2c21fd98/msgspec-0.19.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f12d30dd6266557aaaf0aa0f9580a9a8fbeadfa83699c487713e355ec5f0bd86", size = 190498 },
+    { url = "https://files.pythonhosted.org/packages/58/95/c40b01b93465e1a5f3b6c7d91b10fb574818163740cc3acbe722d1e0e7e4/msgspec-0.19.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82b2c42c1b9ebc89e822e7e13bbe9d17ede0c23c187469fdd9505afd5a481314", size = 183950 },
+    { url = "https://files.pythonhosted.org/packages/e8/f0/5b764e066ce9aba4b70d1db8b087ea66098c7c27d59b9dd8a3532774d48f/msgspec-0.19.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19746b50be214a54239aab822964f2ac81e38b0055cca94808359d779338c10e", size = 210647 },
+    { url = "https://files.pythonhosted.org/packages/9d/87/bc14f49bc95c4cb0dd0a8c56028a67c014ee7e6818ccdce74a4862af259b/msgspec-0.19.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60ef4bdb0ec8e4ad62e5a1f95230c08efb1f64f32e6e8dd2ced685bcc73858b5", size = 213563 },
+    { url = "https://files.pythonhosted.org/packages/53/2f/2b1c2b056894fbaa975f68f81e3014bb447516a8b010f1bed3fb0e016ed7/msgspec-0.19.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac7f7c377c122b649f7545810c6cd1b47586e3aa3059126ce3516ac7ccc6a6a9", size = 213996 },
+    { url = "https://files.pythonhosted.org/packages/aa/5a/4cd408d90d1417e8d2ce6a22b98a6853c1b4d7cb7669153e4424d60087f6/msgspec-0.19.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5bc1472223a643f5ffb5bf46ccdede7f9795078194f14edd69e3aab7020d327", size = 219087 },
+    { url = "https://files.pythonhosted.org/packages/23/d8/f15b40611c2d5753d1abb0ca0da0c75348daf1252220e5dda2867bd81062/msgspec-0.19.0-cp313-cp313-win_amd64.whl", hash = "sha256:317050bc0f7739cb30d257ff09152ca309bf5a369854bbf1e57dffc310c1f20f", size = 187432 },
+]
 
 [[package]]
 name = "outcome"
@@ -240,7 +264,7 @@ wheels = [
 
 [[package]]
 name = "pytest"
-version = "8.3.4"
+version = "8.3.5"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "colorama", marker = "sys_platform == 'win32'" },
@@ -248,9 +272,9 @@ dependencies = [
     { name = "packaging" },
     { name = "pluggy" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/05/35/30e0d83068951d90a01852cb1cef56e5d8a09d20c7f511634cc2f7e0372a/pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761", size = 1445919 }
+sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/11/92/76a1c94d3afee238333bc0a42b82935dd8f9cf8ce9e336ff87ee14d9e1cf/pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6", size = 343083 },
+    { url = "https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", size = 343634 },
 ]
 
 [[package]]
@@ -314,17 +338,15 @@ dev = [
     { name = "pytest" },
     { name = "stackscope" },
     { name = "xonsh" },
-    { name = "xonsh-vox-tabcomplete" },
-    { name = "xontrib-vox" },
 ]
 
 [package.metadata]
 requires-dist = [
     { name = "colorlog", specifier = ">=6.8.2,<7" },
-    { name = "msgspec", git = "https://github.com/jcrist/msgspec.git" },
-    { name = "pdbp", specifier = ">=1.5.0,<2" },
+    { name = "msgspec", specifier = ">=0.19.0" },
+    { name = "pdbp", specifier = ">=1.6,<2" },
     { name = "tricycle", specifier = ">=0.4.1,<0.5" },
-    { name = "trio", specifier = ">=0.24,<0.25" },
+    { name = "trio", specifier = ">0.27" },
     { name = "wrapt", specifier = ">=1.16.0,<2" },
 ]
 
@@ -332,13 +354,11 @@ requires-dist = [
 dev = [
     { name = "greenback", specifier = ">=1.2.1,<2" },
     { name = "pexpect", specifier = ">=4.9.0,<5" },
-    { name = "prompt-toolkit", specifier = ">=3.0.43,<4" },
+    { name = "prompt-toolkit", specifier = ">=3.0.50" },
     { name = "pyperclip", specifier = ">=1.9.0" },
-    { name = "pytest", specifier = ">=8.2.0,<9" },
+    { name = "pytest", specifier = ">=8.3.5" },
     { name = "stackscope", specifier = ">=0.2.2,<0.3" },
-    { name = "xonsh", specifier = ">=0.19.1" },
-    { name = "xonsh-vox-tabcomplete", specifier = ">=0.5,<0.6" },
-    { name = "xontrib-vox", specifier = ">=0.0.1,<0.0.2" },
+    { name = "xonsh", specifier = ">=0.19.2" },
 ]
 
 [[package]]
@@ -355,7 +375,7 @@ wheels = [
 
 [[package]]
 name = "trio"
-version = "0.24.0"
+version = "0.29.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "attrs" },
@@ -365,9 +385,9 @@ dependencies = [
     { name = "sniffio" },
     { name = "sortedcontainers" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/8a/f3/07c152213222c615fe2391b8e1fea0f5af83599219050a549c20fcbd9ba2/trio-0.24.0.tar.gz", hash = "sha256:ffa09a74a6bf81b84f8613909fb0beaee84757450183a7a2e0b47b455c0cac5d", size = 545131 }
+sdist = { url = "https://files.pythonhosted.org/packages/a1/47/f62e62a1a6f37909aed0bf8f5d5411e06fa03846cfcb64540cd1180ccc9f/trio-0.29.0.tar.gz", hash = "sha256:ea0d3967159fc130acb6939a0be0e558e364fee26b5deeecc893a6b08c361bdf", size = 588952 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/14/fb/9299cf74953f473a15accfdbe2c15218e766bae8c796f2567c83bae03e98/trio-0.24.0-py3-none-any.whl", hash = "sha256:c3bd3a4e3e3025cd9a2241eae75637c43fe0b9e88b4c97b9161a55b9e54cd72c", size = 460205 },
+    { url = "https://files.pythonhosted.org/packages/c9/55/c4d9bea8b3d7937901958f65124123512419ab0eb73695e5f382521abbfb/trio-0.29.0-py3-none-any.whl", hash = "sha256:d8c463f1a9cc776ff63e331aba44c125f423a5a13c684307e828d930e625ba66", size = 492920 },
 ]
 
 [[package]]
@@ -434,33 +454,13 @@ wheels = [
 
 [[package]]
 name = "xonsh"
-version = "0.19.1"
+version = "0.19.2"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/98/6e/b54a0b2685535995ee50f655103c463f9d339455c9b08c4bce3e03e7bb17/xonsh-0.19.1.tar.gz", hash = "sha256:5d3de649c909f6d14bc69232219bcbdb8152c830e91ddf17ad169c672397fb97", size = 796468 }
+sdist = { url = "https://files.pythonhosted.org/packages/68/4e/56e95a5e607eb3b0da37396f87cde70588efc8ef819ab16f02d5b8378dc4/xonsh-0.19.2.tar.gz", hash = "sha256:cfdd0680d954a2c3aefd6caddcc7143a3d06aa417ed18365a08219bb71b960b0", size = 799960 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/8c/e6/db44068c5725af9678e37980ae9503165393d51b80dc8517fa4ec74af1cf/xonsh-0.19.1-py310-none-any.whl", hash = "sha256:83eb6610ed3535f8542abd80af9554fb7e2805b0b3f96e445f98d4b5cf1f7046", size = 640686 },
-    { url = "https://files.pythonhosted.org/packages/77/4e/e487e82349866b245c559433c9ba626026a2e66bd17d7f9ac1045082f146/xonsh-0.19.1-py311-none-any.whl", hash = "sha256:c176e515b0260ab803963d1f0924f1e32f1064aa6fd5d791aa0cf6cda3a924ae", size = 640680 },
-    { url = "https://files.pythonhosted.org/packages/5d/88/09060815548219b8f6953a06c247cb5c92d03cbdf7a02a980bda1b5754db/xonsh-0.19.1-py312-none-any.whl", hash = "sha256:fe1266c86b117aced3bdc4d5972420bda715864435d0bd3722d63451e8001036", size = 640604 },
-    { url = "https://files.pythonhosted.org/packages/83/ff/7873cb8184cffeafddbf861712831c2baa2e9dbecdbfd33b1228f0db0019/xonsh-0.19.1-py313-none-any.whl", hash = "sha256:3f158b6fc0bba954e0b989004d4261bafc4bd94c68c2abd75b825da23e5a869c", size = 641166 },
-    { url = "https://files.pythonhosted.org/packages/cc/03/b9f8dd338df0a330011d104e63d4d0acd8bbbc1e990ff049487b6bdf585d/xonsh-0.19.1-py39-none-any.whl", hash = "sha256:a900a6eb87d881a7ef90b1ac8522ba3699582f0bcb1e9abd863d32f6d63faf04", size = 632912 },
-]
-
-[[package]]
-name = "xonsh-vox-tabcomplete"
-version = "0.5"
-source = { registry = "https://pypi.org/simple" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ab/fd/af0c2ee6c067c2a4dc64ec03598c94de1f6ec5984b3116af917f3add4a16/xonsh_vox_tabcomplete-0.5-py3-none-any.whl", hash = "sha256:9701b198180f167071234e77eab87b7befa97c1873b088d0b3fbbe6d6d8dcaad", size = 14381 },
-]
-
-[[package]]
-name = "xontrib-vox"
-version = "0.0.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "xonsh" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/6c/ac/a5db68a1f2e4036f7ff4c8546b1cbe29edee2ff40e0ff931836745988b79/xontrib-vox-0.0.1.tar.gz", hash = "sha256:c1f0b155992b4b0ebe6dcfd651084a8707ade7372f7e456c484d2a85339d9907", size = 16504 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/23/58/dcdf11849c8340033da00669527ce75d8292a4e8d82605c082ed236a081a/xontrib_vox-0.0.1-py3-none-any.whl", hash = "sha256:df2bbb815832db5b04d46684f540eac967ee40ef265add2662a95d6947d04c70", size = 13467 },
+    { url = "https://files.pythonhosted.org/packages/6c/13/281094759df87b23b3c02dc4a16603ab08ea54d7f6acfeb69f3341137c7a/xonsh-0.19.2-py310-none-any.whl", hash = "sha256:ec7f163fd3a4943782aa34069d4e72793328c916a5975949dbec8536cbfc089b", size = 642301 },
+    { url = "https://files.pythonhosted.org/packages/29/41/a51e4c3918fe9a293b150cb949b1b8c6d45eb17dfed480dcb76ea43df4e7/xonsh-0.19.2-py311-none-any.whl", hash = "sha256:53c45f7a767901f2f518f9b8dd60fc653e0498e56e89825e1710bb0859985049", size = 642286 },
+    { url = "https://files.pythonhosted.org/packages/0a/93/9a77b731f492fac27c577dea2afb5a2bcc2a6a1c79be0c86c95498060270/xonsh-0.19.2-py312-none-any.whl", hash = "sha256:b24c619aa52b59eae4d35c4195dba9b19a2c548fb5c42c6f85f2b8ccb96807b5", size = 642386 },
+    { url = "https://files.pythonhosted.org/packages/be/75/070324769c1ff88d971ce040f4f486339be98e0a365c8dd9991eb654265b/xonsh-0.19.2-py313-none-any.whl", hash = "sha256:c53ef6c19f781fbc399ed1b382b5c2aac2125010679a3b61d643978273c27df0", size = 642873 },
+    { url = "https://files.pythonhosted.org/packages/fa/cb/2c7ccec54f5b0e73fdf7650e8336582ff0347d9001c5ef8271dc00c034fe/xonsh-0.19.2-py39-none-any.whl", hash = "sha256:bcc0225dc3847f1ed2f175dac6122fbcc54cea67d9c2dc2753d9615e2a5ff284", size = 634602 },
 ]