Merge pull request #385 from goodboy/repl_fixture

Add a `repl_fixture` for debugger-interaction-hooks and a newly re-factored subsys pkg: `tractor.devx.debug`

examples/debugging/asyncio_bp.py

@@ -29,7 +29,7 @@ async def bp_then_error(
     to_trio.send_nowait('start')
 
     # NOTE: what happens here inside the hook needs some refinement..
-    # => seems like it's still `._debug._set_trace()` but
+    # => seems like it's still `.debug._set_trace()` but
     #    we set `Lock.local_task_in_debug = 'sync'`, we probably want
     #    some further, at least, meta-data about the task/actor in debug
     #    in terms of making it clear it's `asyncio` mucking about.

examples/debugging/restore_builtin_breakpoint.py

@@ -5,7 +5,7 @@ import trio
 import tractor
 
 # ensure mod-path is correct!
-from tractor.devx._debug import (
+from tractor.devx.debug import (
     _sync_pause_from_builtin as _sync_pause_from_builtin,
 )
 
@@ -19,19 +19,22 @@ async def main() -> None:
     async with tractor.open_nursery(
         debug_mode=True,
         loglevel='devx',
-    ) as an:
+        maybe_enable_greenback=True,
-        assert an
+        # ^XXX REQUIRED to enable `breakpoint()` support (from sync
+        # fns) and thus required here to avoid an assertion err
+        # on the next line
+    ):
         assert (
             (pybp_var := os.environ['PYTHONBREAKPOINT'])
             ==
-            'tractor.devx._debug._sync_pause_from_builtin'
+            'tractor.devx.debug._sync_pause_from_builtin'
         )
 
         # TODO: an assert that verifies the hook has indeed been, hooked
         # XD
         assert (
             (pybp_hook := sys.breakpointhook)
-            is not tractor.devx._debug._set_trace
+            is not tractor.devx.debug._set_trace
         )
 
         print(

examples/debugging/subactor_bp_in_ctx.py

@@ -33,8 +33,11 @@ async def just_bp(
 
 
 async def main():
+
     async with tractor.open_nursery(
         debug_mode=True,
+        enable_transports=['uds'],
+        loglevel='devx',
     ) as n:
         p = await n.start_actor(
             'bp_boi',

examples/debugging/sync_bp.py

@@ -6,7 +6,7 @@ import tractor
 
 # TODO: only import these when not running from test harness?
 # can we detect `pexpect` usage maybe?
-# from tractor.devx._debug import (
+# from tractor.devx.debug import (
 #     get_lock,
 #     get_debug_req,
 # )

pyproject.toml

@@ -63,6 +63,7 @@ dev = [
   "stackscope>=0.2.2,<0.3",
   # ^ requires this?
   "typing-extensions>=4.14.1",
+
   "pyperclip>=1.9.0",
   "prompt-toolkit>=3.0.50",
   "xonsh>=0.19.2",

tests/conftest.py

@@ -103,7 +103,7 @@ def sig_prog(
 def daemon(
     debug_mode: bool,
     loglevel: str,
-    testdir,
+    testdir: pytest.Pytester,
     reg_addr: tuple[str, int],
     tpt_proto: str,
 

tests/devx/conftest.py

@@ -2,9 +2,11 @@
 `tractor.devx.*` tooling sub-pkg test space.
 
 '''
+from __future__ import annotations
 import time
 from typing import (
     Callable,
+    TYPE_CHECKING,
 )
 
 import pytest
@@ -16,7 +18,7 @@ from pexpect.spawnbase import SpawnBase
 from tractor._testing import (
     mk_cmd,
 )
-from tractor.devx._debug import (
+from tractor.devx.debug import (
     _pause_msg as _pause_msg,
     _crash_msg as _crash_msg,
     _repl_fail_msg as _repl_fail_msg,
@@ -26,14 +28,22 @@ from ..conftest import (
     _ci_env,
 )
 
+if TYPE_CHECKING:
+    from pexpect import pty_spawn
+
+
+# a fn that sub-instantiates a `pexpect.spawn()`
+# and returns it.
+type PexpectSpawner = Callable[[str], pty_spawn.spawn]
+
 
 @pytest.fixture
 def spawn(
-    start_method,
+    start_method: str,
     testdir: pytest.Pytester,
     reg_addr: tuple[str, int],
 
-) -> Callable[[str], None]:
+) -> PexpectSpawner:
     '''
     Use the `pexpect` module shipped via `testdir.spawn()` to
     run an `./examples/..` script by name.
@@ -59,7 +69,7 @@ def spawn(
     def _spawn(
         cmd: str,
         **mkcmd_kwargs,
-    ):
+    ) -> pty_spawn.spawn:
         unset_colors()
         return testdir.spawn(
             cmd=mk_cmd(
@@ -73,7 +83,7 @@ def spawn(
         )
 
     # such that test-dep can pass input script name.
-    return _spawn
+    return _spawn  # the `PexpectSpawner`, type alias.
 
 
 @pytest.fixture(
@@ -111,7 +121,7 @@ def ctlc(
         # XXX: disable pygments highlighting for auto-tests
         # since some envs (like actions CI) will struggle
         # the the added color-char encoding..
-        from tractor.devx._debug import TractorConfig
+        from tractor.devx.debug import TractorConfig
         TractorConfig.use_pygements = False
 
     yield use_ctlc

tests/devx/test_debugger.py

@@ -10,10 +10,14 @@ TODO:
     - wonder if any of it'll work on OS X?
 
 """
+from __future__ import annotations
 from functools import partial
 import itertools
 import platform
 import time
+from typing import (
+    TYPE_CHECKING,
+)
 
 import pytest
 from pexpect.exceptions import (
@@ -34,6 +38,9 @@ from .conftest import (
     assert_before,
 )
 
+if TYPE_CHECKING:
+    from ..conftest import PexpectSpawner
+
 # TODO: The next great debugger audit could be done by you!
 # - recurrent entry to breakpoint() from single actor *after* and an
 #   error in another task?
@@ -528,7 +535,7 @@ def test_multi_daemon_subactors(
     # now the root actor won't clobber the bp_forever child
     # during it's first access to the debug lock, but will instead
     # wait for the lock to release, by the edge triggered
-    # ``devx._debug.Lock.no_remote_has_tty`` event before sending cancel messages
+    # ``devx.debug.Lock.no_remote_has_tty`` event before sending cancel messages
     # (via portals) to its underlings B)
 
     # at some point here there should have been some warning msg from
@@ -1063,6 +1070,88 @@ def test_shield_pause(
     child.expect(EOF)
 
 
+@pytest.mark.parametrize(
+    'quit_early', [False, True]
+)
+def test_ctxep_pauses_n_maybe_ipc_breaks(
+    spawn: PexpectSpawner,
+    quit_early: bool,
+):
+    '''
+    Audit generator embedded `.pause()`es from within a `@context`
+    endpoint with a chan close at the end, requiring that ctl-c is
+    mashed and zombie reaper kills sub with no hangs.
+
+    '''
+    child = spawn('subactor_bp_in_ctx')
+    child.expect(PROMPT)
+
+    # 3 iters for the `gen()` pause-points
+    for i in range(3):
+        assert_before(
+            child,
+            [
+                _pause_msg,
+                "('bp_boi'",  # actor name
+                "<Task 'just_bp'",  # task name
+            ]
+        )
+        if (
+            i == 1
+            and
+            quit_early
+        ):
+            child.sendline('q')
+            child.expect(PROMPT)
+            assert_before(
+                child,
+                ["tractor._exceptions.RemoteActorError: remote task raised a 'BdbQuit'",
+                 "bdb.BdbQuit",
+                 "('bp_boi'",
+                ]
+            )
+            child.sendline('c')
+            child.expect(EOF)
+            assert_before(
+                child,
+                ["tractor._exceptions.RemoteActorError: remote task raised a 'BdbQuit'",
+                 "bdb.BdbQuit",
+                 "('bp_boi'",
+                ]
+            )
+            break  # end-of-test
+
+        child.sendline('c')
+        try:
+            child.expect(PROMPT)
+        except TIMEOUT:
+            # no prompt since we hang due to IPC chan purposely
+            # closed so verify we see error reporting as well as
+            # a failed crash-REPL request msg and can CTL-c our way
+            # out.
+            assert_before(
+                child,
+                ['peer IPC channel closed abruptly?',
+                 'another task closed this fd',
+                 'Debug lock request was CANCELLED?',
+                 "TransportClosed: 'MsgpackUDSStream' was already closed locally ?",]
+
+                # XXX races on whether these show/hit?
+                 # 'Failed to REPl via `_pause()` You called `tractor.pause()` from an already cancelled scope!',
+                 # 'AssertionError',
+            )
+            # OSc(ancel) the hanging tree
+            do_ctlc(
+                child=child,
+                expect_prompt=False,
+            )
+            child.expect(EOF)
+            assert_before(
+                child,
+                ['KeyboardInterrupt'],
+            )
+
+
 # TODO: better error for "non-ideal" usage from the root actor.
 # -[ ] if called from an async scope emit a message that suggests
 #    using `await tractor.pause()` instead since it's less overhead

tests/devx/test_tooling.py

@@ -13,9 +13,16 @@ TODO:
   when debugging a problem inside the stack vs. in their app.
 
 '''
+from __future__ import annotations
+from contextlib import (
+    contextmanager as cm,
+)
 import os
 import signal
 import time
+from typing import (
+    TYPE_CHECKING,
+)
 
 from .conftest import (
     expect,
@@ -24,14 +31,19 @@ from .conftest import (
     PROMPT,
     _pause_msg,
 )
+
+import pytest
 from pexpect.exceptions import (
     # TIMEOUT,
     EOF,
 )
 
+if TYPE_CHECKING:
+    from ..conftest import PexpectSpawner
+
 
 def test_shield_pause(
-    spawn,
+    spawn: PexpectSpawner,
 ):
     '''
     Verify the `tractor.pause()/.post_mortem()` API works inside an
@@ -126,7 +138,7 @@ def test_shield_pause(
 
 
 def test_breakpoint_hook_restored(
-    spawn,
+    spawn: PexpectSpawner,
 ):
     '''
     Ensures our actor runtime sets a custom `breakpoint()` hook
@@ -140,16 +152,22 @@ def test_breakpoint_hook_restored(
     child = spawn('restore_builtin_breakpoint')
 
     child.expect(PROMPT)
-    assert_before(
+    try:
-        child,
+        assert_before(
-        [
+            child,
-            _pause_msg,
+            [
-            "<Task '__main__.main'",
+                _pause_msg,
-            "('root'",
+                "<Task '__main__.main'",
-            "first bp, tractor hook set",
+                "('root'",
-        ]
+                "first bp, tractor hook set",
-    )
+            ]
-    child.sendline('c')
+        )
+    # XXX if the above raises `AssertionError`, without sending
+    # the final 'continue' cmd to the REPL-active sub-process,
+    # we'll hang waiting for that pexpect instance to terminate..
+    finally:
+        child.sendline('c')
+
     child.expect(PROMPT)
     assert_before(
         child,
@@ -170,3 +188,117 @@ def test_breakpoint_hook_restored(
     )
     child.sendline('c')
     child.expect(EOF)
+
+
+
+_to_raise = Exception('Triggering a crash')
+
+
+@pytest.mark.parametrize(
+    'to_raise',
+    [
+        None,
+        _to_raise,
+        RuntimeError('Never crash handle this!'),
+    ],
+)
+@pytest.mark.parametrize(
+    'raise_on_exit',
+    [
+        True,
+        [type(_to_raise)],
+        False,
+    ]
+)
+def test_crash_handler_cms(
+    debug_mode: bool,
+    to_raise: Exception,
+    raise_on_exit: bool|list[Exception],
+):
+    '''
+    Verify the `.devx.open_crash_handler()` API(s) by also
+    (conveniently enough) tesing its `repl_fixture: ContextManager`
+    param support which for this suite allows use to avoid use of
+    a `pexpect`-style-test since we use the fixture to avoid actually
+    entering `PdbpREPL.iteract()` :smirk:
+
+    '''
+    import tractor
+    # import trio
+
+    # state flags
+    repl_acquired: bool = False
+    repl_released: bool = False
+
+    @cm
+    def block_repl_ux(
+        repl: tractor.devx.debug.PdbREPL,
+        maybe_bxerr: (
+            tractor.devx._debug.BoxedMaybeException
+            |None
+        ) = None,
+        enter_repl: bool = True,
+
+    ) -> bool:
+        '''
+        Set pre/post-REPL state vars and bypass actual conole
+        interaction.
+
+        '''
+        nonlocal repl_acquired, repl_released
+
+        # task: trio.Task = trio.lowlevel.current_task()
+        # print(f'pre-REPL active_task={task.name}')
+
+        print('pre-REPL')
+        repl_acquired = True
+        yield False  # never actually .interact()
+        print('post-REPL')
+        repl_released = True
+
+    try:
+        # TODO, with runtime's `debug_mode` setting
+        # -[ ] need to open runtime tho obvi..
+        #
+        # with tractor.devx.maybe_open_crash_handler(
+        #     pdb=True,
+
+        with tractor.devx.open_crash_handler(
+            raise_on_exit=raise_on_exit,
+            repl_fixture=block_repl_ux
+        ) as bxerr:
+            if to_raise is not None:
+                raise to_raise
+
+    except Exception as _exc:
+        exc = _exc
+        if (
+            raise_on_exit is True
+            or
+            type(to_raise) in raise_on_exit
+        ):
+            assert (
+                exc
+                is
+                to_raise
+                is
+                bxerr.value
+            )
+
+        else:
+            raise
+    else:
+        assert (
+            to_raise is None
+            or
+            not raise_on_exit
+            or
+            type(to_raise) not in raise_on_exit
+        )
+        assert bxerr.value is to_raise
+
+    assert bxerr.raise_on_exit == raise_on_exit
+
+    if to_raise is not None:
+        assert repl_acquired
+        assert repl_released

tests/test_infected_asyncio.py

@@ -889,7 +889,7 @@ async def manage_file(
 
         # NOTE: turns out you don't even need to sched an aio task
         # since the original issue, even though seemingly was due to
-        # the guest-run being abandoned + a `._debug.pause()` inside
+        # the guest-run being abandoned + a `.debug.pause()` inside
         # `._runtime._async_main()` (which was originally trying to
         # debug the `.lifetime_stack` not closing), IS NOT actually
         # the core issue?
@@ -1101,7 +1101,7 @@ def test_sigint_closes_lifetime_stack(
 #    => completed using `.bestow_portal(task)` inside
 #     `.to_asyncio._run_asyncio_task()` right?
 #   -[ ] translation func to get from `asyncio` task calling to 
-#     `._debug.wait_for_parent_stdin_hijack()` which does root
+#     `.debug.wait_for_parent_stdin_hijack()` which does root
 #     call to do TTY locking.
 #
 def test_sync_breakpoint():

tests/test_root_runtime.py

@@ -83,3 +83,26 @@ def test_implicit_root_via_first_nursery(
             assert tractor.current_actor().aid.name == 'root'
 
     trio.run(main)
+
+
+def test_runtime_vars_unset(
+    reg_addr: tuple,
+    debug_mode: bool
+):
+    '''
+    Ensure any `._state._runtime_vars` are restored to default values
+    after the root actor-runtime exits!
+
+    '''
+    assert not tractor._state._runtime_vars['_debug_mode']
+    async def main():
+        assert not tractor._state._runtime_vars['_debug_mode']
+        async with tractor.open_nursery(
+            debug_mode=True,
+        ):
+            assert tractor._state._runtime_vars['_debug_mode']
+
+        # after runtime closure, should be reverted!
+        assert not tractor._state._runtime_vars['_debug_mode']
+
+    trio.run(main)

tractor/_context.py

@@ -292,7 +292,7 @@ class Context:
     # - `._runtime._invoke()` will check this flag before engaging
     #   the crash handler REPL in such cases where the "callee"
     #   raises the cancellation,
-    # - `.devx._debug.lock_stdio_for_peer()` will set it to `False` if
+    # - `.devx.debug.lock_stdio_for_peer()` will set it to `False` if
     #   the global tty-lock has been configured to filter out some
     #   actors from being able to acquire the debugger lock.
     _enter_debugger_on_cancel: bool = True
@@ -1251,8 +1251,8 @@ class Context:
 
             # ?XXX, should already be set in `._deliver_msg()` right?
             if self._outcome_msg is not Unresolved:
-                # from .devx import _debug
+                # from .devx import debug
-                # await _debug.pause()
+                # await debug.pause()
                 assert self._outcome_msg is outcome_msg
             else:
                 self._outcome_msg = outcome_msg
@@ -2187,7 +2187,7 @@ async def open_context_from_portal(
         #   debugging the tractor-runtime itself using it's
         #   own `.devx.` tooling!
         # 
-        # await _debug.pause()
+        # await debug.pause()
 
         # CASE 2: context was cancelled by local task calling
         # `.cancel()`, we don't raise and the exit block should
@@ -2254,7 +2254,7 @@ async def open_context_from_portal(
         # NOTE: `Context.cancel()` is conversely NEVER CALLED in
         # the `ContextCancelled` "self cancellation absorbed" case
         # handled in the block above ^^^ !!
-        # await _debug.pause()
+        # await debug.pause()
         # log.cancel(
         match scope_err:
             case trio.Cancelled:
@@ -2269,11 +2269,11 @@ async def open_context_from_portal(
         )
 
         if debug_mode():
-            # async with _debug.acquire_debug_lock(portal.actor.uid):
+            # async with debug.acquire_debug_lock(portal.actor.uid):
             #     pass
             # TODO: factor ^ into below for non-root cases?
             #
-            from .devx._debug import maybe_wait_for_debugger
+            from .devx.debug import maybe_wait_for_debugger
             was_acquired: bool = await maybe_wait_for_debugger(
                 # header_msg=(
                 #     'Delaying `ctx.cancel()` until debug lock '
@@ -2336,8 +2336,8 @@ async def open_context_from_portal(
                 raise
 
             # yes this worx!
-            # from .devx import _debug
+            # from .devx import debug
-            # await _debug.pause()
+            # await debug.pause()
 
             # an exception type boxed in a `RemoteActorError`
             # is returned (meaning it was obvi not raised)
@@ -2372,7 +2372,7 @@ async def open_context_from_portal(
         # where the root is waiting on the lock to clear but the
         # child has already cleared it and clobbered IPC.
         if debug_mode():
-            from .devx._debug import maybe_wait_for_debugger
+            from .devx.debug import maybe_wait_for_debugger
             await maybe_wait_for_debugger()
 
         # though it should be impossible for any tasks

tractor/_entry.py

@@ -35,7 +35,7 @@ from .log import (
 )
 from . import _state
 from .devx import (
-    _debug,
+    _frame_stack,
     pformat,
 )
 from .to_asyncio import run_as_asyncio_guest
@@ -116,7 +116,7 @@ def _trio_main(
     Entry point for a `trio_run_in_process` subactor.
 
     '''
-    _debug.hide_runtime_frames()
+    _frame_stack.hide_runtime_frames()
 
     _state._current_actor = actor
     trio_main = partial(

tractor/_root.py

@@ -44,7 +44,10 @@ from ._runtime import (
     # Arbiter as Registry,
     async_main,
 )
-from .devx import _debug
+from .devx import (
+    debug,
+    _frame_stack,
+)
 from . import _spawn
 from . import _state
 from . import log
@@ -67,7 +70,7 @@ from ._exceptions import (
 logger = log.get_logger('tractor')
 
 
-# TODO: stick this in a `@acm` defined in `devx._debug`?
+# TODO: stick this in a `@acm` defined in `devx.debug`?
 # -[ ] also maybe consider making this a `wrapt`-deco to
 #     save an indent level?
 #
@@ -89,7 +92,7 @@ async def maybe_block_bp(
         debug_mode
         and maybe_enable_greenback
         and (
-            maybe_mod := await _debug.maybe_init_greenback(
+            maybe_mod := await debug.maybe_init_greenback(
                 raise_not_found=False,
             )
         )
@@ -99,7 +102,7 @@ async def maybe_block_bp(
             'Enabling `tractor.pause_from_sync()` support!\n'
         )
         os.environ['PYTHONBREAKPOINT'] = (
-            'tractor.devx._debug._sync_pause_from_builtin'
+            'tractor.devx.debug._sync_pause_from_builtin'
         )
         _state._runtime_vars['use_greenback'] = True
         bp_blocked = False
@@ -163,7 +166,9 @@ async def open_root_actor(
 
     # enables the multi-process debugger support
     debug_mode: bool = False,
-    maybe_enable_greenback: bool = True,  # `.pause_from_sync()/breakpoint()` support
+    maybe_enable_greenback: bool = False,  # `.pause_from_sync()/breakpoint()` support
+    # ^XXX NOTE^ the perf implications of use,
+    # https://greenback.readthedocs.io/en/latest/principle.html#performance
     enable_stack_on_sig: bool = False,
 
     # internal logging
@@ -178,7 +183,7 @@ async def open_root_actor(
 
     hide_tb: bool = True,
 
-    # XXX, proxied directly to `.devx._debug._maybe_enter_pm()`
+    # XXX, proxied directly to `.devx.debug._maybe_enter_pm()`
     # for REPL-entry logic.
     debug_filter: Callable[
         [BaseException|BaseExceptionGroup],
@@ -228,12 +233,12 @@ async def open_root_actor(
                 f'enable_transports={enable_transports!r}\n'
             )
 
-        _debug.hide_runtime_frames()
+        _frame_stack.hide_runtime_frames()
         __tracebackhide__: bool = hide_tb
 
         # attempt to retreive ``trio``'s sigint handler and stash it
         # on our debugger lock state.
-        _debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
+        debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
 
         # mark top most level process as root actor
         _state._runtime_vars['_is_root'] = True
@@ -281,13 +286,14 @@ async def open_root_actor(
 
         if (
             debug_mode
-            and _spawn._spawn_method == 'trio'
+            and
+            _spawn._spawn_method == 'trio'
         ):
             _state._runtime_vars['_debug_mode'] = True
 
             # expose internal debug module to every actor allowing for
             # use of ``await tractor.pause()``
-            enable_modules.append('tractor.devx._debug')
+            enable_modules.append('tractor.devx.debug._tty_lock')
 
             # if debug mode get's enabled *at least* use that level of
             # logging for some informative console prompts.
@@ -469,7 +475,7 @@ async def open_root_actor(
                     # TODO, in beginning to handle the subsubactor with
                     # crashed grandparent cases..
                     #
-                    # was_locked: bool = await _debug.maybe_wait_for_debugger(
+                    # was_locked: bool = await debug.maybe_wait_for_debugger(
                     #     child_in_debug=True,
                     # )
                     # XXX NOTE XXX see equiv note inside
@@ -477,7 +483,7 @@ async def open_root_actor(
                     # non-root or root-that-opened-this-mahually case we
                     # wait for the local actor-nursery to exit before
                     # exiting the transport channel handler.
-                    entered: bool = await _debug._maybe_enter_pm(
+                    entered: bool = await debug._maybe_enter_pm(
                         err,
                         api_frame=inspect.currentframe(),
                         debug_filter=debug_filter,
@@ -517,8 +523,17 @@ async def open_root_actor(
                     )
                     await actor.cancel(None)  # self cancel
         finally:
+            # revert all process-global runtime state
+            if (
+                debug_mode
+                and
+                _spawn._spawn_method == 'trio'
+            ):
+                _state._runtime_vars['_debug_mode'] = False
+
             _state._current_actor = None
             _state._last_actor_terminated = actor
+
             logger.runtime(
                 f'Root actor terminated\n'
                 f')>\n'

tractor/_rpc.py

@@ -57,7 +57,7 @@ from ._exceptions import (
     unpack_error,
 )
 from .devx import (
-    _debug,
+    debug,
     add_div,
 )
 from . import _state
@@ -266,7 +266,7 @@ async def _errors_relayed_via_ipc(
 
     # TODO: a debug nursery when in debug mode!
     # async with maybe_open_debugger_nursery() as debug_tn:
-    # => see matching comment in side `._debug._pause()`
+    # => see matching comment in side `.debug._pause()`
     rpc_err: BaseException|None = None
     try:
         yield  # run RPC invoke body
@@ -318,7 +318,7 @@ async def _errors_relayed_via_ipc(
                     'RPC task crashed, attempting to enter debugger\n'
                     f'|_{ctx}'
                 )
-                entered_debug = await _debug._maybe_enter_pm(
+                entered_debug = await debug._maybe_enter_pm(
                     err,
                     api_frame=inspect.currentframe(),
                 )
@@ -462,7 +462,7 @@ async def _invoke(
     ):
         # XXX for .pause_from_sync()` usage we need to make sure
         # `greenback` is boostrapped in the subactor!
-        await _debug.maybe_init_greenback()
+        await debug.maybe_init_greenback()
 
     # TODO: possibly a specially formatted traceback
     # (not sure what typing is for this..)?
@@ -751,7 +751,7 @@ async def _invoke(
                 and 'Cancel scope stack corrupted' in scope_error.args[0]
             ):
                 log.exception('Cancel scope stack corrupted!?\n')
-                # _debug.mk_pdb().set_trace()
+                # debug.mk_pdb().set_trace()
 
             # always set this (child) side's exception as the
             # local error on the context
@@ -779,7 +779,7 @@ async def _invoke(
 
             # don't pop the local context until we know the
             # associated child isn't in debug any more
-            await _debug.maybe_wait_for_debugger()
+            await debug.maybe_wait_for_debugger()
             ctx: Context = actor._contexts.pop((
                 chan.uid,
                 cid,
@@ -983,7 +983,7 @@ async def process_messages(
                         # XXX NOTE XXX don't start entire actor
                         # runtime cancellation if this actor is
                         # currently in debug mode!
-                        pdb_complete: trio.Event|None = _debug.DebugStatus.repl_release
+                        pdb_complete: trio.Event|None = debug.DebugStatus.repl_release
                         if pdb_complete:
                             await pdb_complete.wait()
 

tractor/_runtime.py

@@ -97,7 +97,7 @@ from ._exceptions import (
     MsgTypeError,
     unpack_error,
 )
-from .devx import _debug
+from .devx import debug
 from ._discovery import get_registry
 from ._portal import Portal
 from . import _state
@@ -237,7 +237,7 @@ class Actor:
         # TODO? only add this when `is_debug_mode() == True` no?
         # always include debugging tools module
         if _state.is_root_process():
-            enable_modules.append('tractor.devx._debug')
+            enable_modules.append('tractor.devx.debug._tty_lock')
 
         self.enable_modules: dict[str, str] = get_mod_nsps2fps(
             mod_ns_paths=enable_modules,
@@ -968,7 +968,7 @@ class Actor:
 
             # kill any debugger request task to avoid deadlock
             # with the root actor in this tree
-            debug_req = _debug.DebugStatus
+            debug_req = debug.DebugStatus
             lock_req_ctx: Context = debug_req.req_ctx
             if (
                 lock_req_ctx
@@ -978,7 +978,7 @@ class Actor:
                 msg += (
                     f'\n'
                     f'-> Cancelling active debugger request..\n'
-                    f'|_{_debug.Lock.repr()}\n\n'
+                    f'|_{debug.Lock.repr()}\n\n'
                     f'|_{lock_req_ctx}\n\n'
                 )
                 # lock_req_ctx._scope.cancel()
@@ -1308,7 +1308,7 @@ async def async_main(
 
     # attempt to retreive ``trio``'s sigint handler and stash it
     # on our debugger state.
-    _debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
+    debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
 
     is_registered: bool = False
     try:
@@ -1401,7 +1401,7 @@ async def async_main(
                 # try:
                 #     actor.load_modules()
                 # except ModuleNotFoundError as err:
-                #     _debug.pause_from_sync()
+                #     debug.pause_from_sync()
                 #     import pdbp; pdbp.set_trace()
                 #     raise
 
@@ -1433,7 +1433,7 @@ async def async_main(
                     # tranport address bind errors - normally it's
                     # something silly like the wrong socket-address
                     # passed via a config or CLI Bo
-                    entered_debug: bool = await _debug._maybe_enter_pm(
+                    entered_debug: bool = await debug._maybe_enter_pm(
                         oserr,
                     )
                     if not entered_debug:
@@ -1473,7 +1473,7 @@ async def async_main(
                         waddr = wrap_address(addr)
                         assert waddr.is_valid
                     except AssertionError:
-                        await _debug.pause()
+                        await debug.pause()
 
                     async with get_registry(addr) as reg_portal:
                         for accept_addr in accept_addrs:
@@ -1590,7 +1590,7 @@ async def async_main(
             # prevents any `infected_aio` actor from continuing
             # and any callbacks in the `ls` here WILL NOT be
             # called!!
-            # await _debug.pause(shield=True)
+            # await debug.pause(shield=True)
 
         ls.close()
 
@@ -1603,7 +1603,7 @@ async def async_main(
         #
         # if actor.name == 'brokerd.ib':
         #     with CancelScope(shield=True):
-        #         await _debug.breakpoint()
+        #         await debug.breakpoint()
 
         # Unregister actor from the registry-sys / registrar.
         if (
@@ -1792,7 +1792,7 @@ class Arbiter(Actor):
         waddr: Address = wrap_address(addr)
         if not waddr.is_valid:
             # should never be 0-dynamic-os-alloc
-            await _debug.pause()
+            await debug.pause()
 
         self._registry[uid] = addr
 

tractor/_spawn.py

@@ -34,7 +34,7 @@ from typing import (
 import trio
 from trio import TaskStatus
 
-from .devx._debug import (
+from .devx.debug import (
     maybe_wait_for_debugger,
     acquire_debug_lock,
 )

tractor/_state.py

@@ -48,8 +48,9 @@ _current_actor: Actor|None = None  # type: ignore # noqa
 _last_actor_terminated: Actor|None = None
 
 # TODO: mk this a `msgspec.Struct`!
+# -[ ] type out all fields obvi!
+# -[ ] (eventually) mk wire-ready for monitoring?
 _runtime_vars: dict[str, Any] = {
-    '_debug_mode': False,
     # root of actor-process tree info
     '_is_root': False,  # bool
     '_root_mailbox': (None, None),  # tuple[str|None, str|None]
@@ -61,10 +62,14 @@ _runtime_vars: dict[str, Any] = {
     # registrar info
     '_registry_addrs': [],
 
-    '_is_infected_aio': False,
+    # `debug_mode: bool` settings
-
+    '_debug_mode': False,  # bool
+    'repl_fixture': False,  # |AbstractContextManager[bool]
     # for `tractor.pause_from_sync()` & `breakpoint()` support
     'use_greenback': False,
+
+    # infected-`asyncio`-mode: `trio` running as guest.
+    '_is_infected_aio': False,
 }
 
 

tractor/_streaming.py

@@ -426,8 +426,8 @@ class MsgStream(trio.abc.Channel):
             self._closed = re
 
         # if caught_eoc:
-        #     # from .devx import _debug
+        #     # from .devx import debug
-        #     # await _debug.pause()
+        #     # await debug.pause()
         #     with trio.CancelScope(shield=True):
         #         await rx_chan.aclose()
 

tractor/_supervise.py

@@ -31,7 +31,7 @@ import warnings
 import trio
 
 
-from .devx._debug import maybe_wait_for_debugger
+from .devx.debug import maybe_wait_for_debugger
 from ._addr import (
     UnwrappedAddress,
     mk_uuid,

tractor/_testing/__init__.py

@@ -26,7 +26,7 @@ import os
 import pathlib
 
 import tractor
-from tractor.devx._debug import (
+from tractor.devx.debug import (
     BoxedMaybeException,
 )
 from .pytest import (
@@ -37,6 +37,9 @@ from .fault_simulation import (
 )
 
 
+# TODO, use dulwhich for this instead?
+# -> we're going to likely need it (or something similar)
+#   for supporting hot-coad reload feats eventually anyway!
 def repodir() -> pathlib.Path:
     '''
     Return the abspath to the repo directory.

tractor/devx/__init__.py

@@ -20,7 +20,7 @@ Runtime "developer experience" utils and addons to aid our
 and working with/on the actor runtime.
 
 """
-from ._debug import (
+from .debug import (
     maybe_wait_for_debugger as maybe_wait_for_debugger,
     acquire_debug_lock as acquire_debug_lock,
     breakpoint as breakpoint,

tractor/devx/_frame_stack.py

@@ -20,13 +20,18 @@ as it pertains to improving the grok-ability of our runtime!
 
 '''
 from __future__ import annotations
+from contextlib import (
+    _GeneratorContextManager,
+    _AsyncGeneratorContextManager,
+)
 from functools import partial
 import inspect
+import textwrap
 from types import (
     FrameType,
     FunctionType,
     MethodType,
-    # CodeType,
+    CodeType,
 )
 from typing import (
     Any,
@@ -34,6 +39,9 @@ from typing import (
     Type,
 )
 
+import pdbp
+from tractor.log import get_logger
+import trio
 from tractor.msg import (
     pretty_struct,
     NamespacePath,
@@ -41,6 +49,8 @@ from tractor.msg import (
 import wrapt
 
 
+log = get_logger(__name__)
+
 # TODO: yeah, i don't love this and we should prolly just
 # write a decorator that actually keeps a stupid ref to the func
 # obj..
@@ -301,3 +311,70 @@ def api_frame(
 #     error_set: set[BaseException],
 # ) -> TracebackType:
 #     ...
+
+
+def hide_runtime_frames() -> dict[FunctionType, CodeType]:
+    '''
+    Hide call-stack frames for various std-lib and `trio`-API primitives
+    such that the tracebacks presented from our runtime are as minimized
+    as possible, particularly from inside a `PdbREPL`.
+
+    '''
+    # XXX HACKZONE XXX
+    #  hide exit stack frames on nurseries and cancel-scopes!
+    # |_ so avoid seeing it when the `pdbp` REPL is first engaged from
+    #    inside a `trio.open_nursery()` scope (with no line after it
+    #    in before the block end??).
+    #
+    # TODO: FINALLY got this workin originally with
+    #  `@pdbp.hideframe` around the `wrapper()` def embedded inside
+    #  `_ki_protection_decoratior()`.. which is in the module:
+    #  /home/goodboy/.virtualenvs/tractor311/lib/python3.11/site-packages/trio/_core/_ki.py
+    #
+    # -[ ] make an issue and patch for `trio` core? maybe linked
+    #    to the long outstanding `pdb` one below?
+    #   |_ it's funny that there's frame hiding throughout `._run.py`
+    #      but not where it matters on the below exit funcs..
+    #
+    # -[ ] provide a patchset for the lonstanding
+    #   |_ https://github.com/python-trio/trio/issues/1155
+    #
+    # -[ ] make a linked issue to ^ and propose allowing all the
+    #     `._core._run` code to have their `__tracebackhide__` value
+    #     configurable by a `RunVar` to allow getting scheduler frames
+    #     if desired through configuration?
+    #
+    # -[ ] maybe dig into the core `pdb` issue why the extra frame is shown
+    #      at all?
+    #
+    funcs: list[FunctionType] = [
+        trio._core._run.NurseryManager.__aexit__,
+        trio._core._run.CancelScope.__exit__,
+         _GeneratorContextManager.__exit__,
+         _AsyncGeneratorContextManager.__aexit__,
+         _AsyncGeneratorContextManager.__aenter__,
+         trio.Event.wait,
+    ]
+    func_list_str: str = textwrap.indent(
+        "\n".join(f.__qualname__ for f in funcs),
+        prefix=' |_ ',
+    )
+    log.devx(
+        'Hiding the following runtime frames by default:\n'
+        f'{func_list_str}\n'
+    )
+
+    codes: dict[FunctionType, CodeType] = {}
+    for ref in funcs:
+        # stash a pre-modified version of each ref's code-obj
+        # so it can be reverted later if needed.
+        codes[ref] = ref.__code__
+        pdbp.hideframe(ref)
+    #
+    # pdbp.hideframe(trio._core._run.NurseryManager.__aexit__)
+    # pdbp.hideframe(trio._core._run.CancelScope.__exit__)
+    # pdbp.hideframe(_GeneratorContextManager.__exit__)
+    # pdbp.hideframe(_AsyncGeneratorContextManager.__aexit__)
+    # pdbp.hideframe(_AsyncGeneratorContextManager.__aenter__)
+    # pdbp.hideframe(trio.Event.wait)
+    return codes

tractor/devx/_stackscope.py

@@ -49,7 +49,7 @@ from tractor import (
     _state,
     log as logmod,
 )
-from tractor.devx import _debug
+from tractor.devx import debug
 
 log = logmod.get_logger(__name__)
 
@@ -82,7 +82,7 @@ def dump_task_tree() -> None:
     if (
         current_sigint_handler
         is not
-        _debug.DebugStatus._trio_handler
+        debug.DebugStatus._trio_handler
     ):
         sigint_handler_report: str = (
             'The default `trio` SIGINT handler was replaced?!'
@@ -238,7 +238,8 @@ def enable_stack_on_sig(
         import stackscope
     except ImportError:
         log.error(
-            '`stackscope` not installed for use in debug mode!'
+            '`stackscope` not installed for use in debug mode!\n'
+            '`Ignoring {enable_stack_on_sig!r} call!\n'
         )
         return None
 

tractor/devx/debug/__init__.py

@@ -0,0 +1,100 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+Multi-actor debugging for da peeps!
+
+'''
+from __future__ import annotations
+from tractor.log import get_logger
+from ._repl import (
+    PdbREPL as PdbREPL,
+    mk_pdb as mk_pdb,
+    TractorConfig as TractorConfig,
+)
+from ._tty_lock import (
+    DebugStatus as DebugStatus,
+    DebugStateError as DebugStateError,
+)
+from ._trace import (
+    Lock as Lock,
+    _pause_msg as _pause_msg,
+    _repl_fail_msg as _repl_fail_msg,
+    _set_trace as _set_trace,
+    _sync_pause_from_builtin as _sync_pause_from_builtin,
+    breakpoint as breakpoint,
+    maybe_init_greenback as maybe_init_greenback,
+    maybe_import_greenback as maybe_import_greenback,
+    pause as pause,
+    pause_from_sync as pause_from_sync,
+)
+from ._post_mortem import (
+    BoxedMaybeException as BoxedMaybeException,
+    maybe_open_crash_handler as maybe_open_crash_handler,
+    open_crash_handler as open_crash_handler,
+    post_mortem as post_mortem,
+    _crash_msg as _crash_msg,
+    _maybe_enter_pm as _maybe_enter_pm,
+)
+from ._sync import (
+    maybe_wait_for_debugger as maybe_wait_for_debugger,
+    acquire_debug_lock as acquire_debug_lock,
+)
+from ._sigint import (
+    sigint_shield as sigint_shield,
+    _ctlc_ignore_header as _ctlc_ignore_header
+)
+
+log = get_logger(__name__)
+
+# ----------------
+# XXX PKG TODO XXX
+# ----------------
+# refine the internal impl and APIs!
+#
+# -[ ] rework `._pause()` and it's branch-cases for root vs.
+#     subactor:
+#  -[ ] `._pause_from_root()` + `_pause_from_subactor()`?
+#  -[ ]  do the de-factor based on bg-thread usage in
+#    `.pause_from_sync()` & `_pause_from_bg_root_thread()`.
+#  -[ ] drop `debug_func == None` case which is confusing af..
+#  -[ ]  factor out `_enter_repl_sync()` into a util func for calling
+#    the `_set_trace()` / `_post_mortem()` APIs?
+#
+# -[ ] figure out if we need `acquire_debug_lock()` and/or re-implement
+#    it as part of the `.pause_from_sync()` rework per above?
+#
+# -[ ] pair the `._pause_from_subactor()` impl with a "debug nursery"
+#   that's dynamically allocated inside the `._rpc` task thus
+#   avoiding the `._service_n.start()` usage for the IPC request?
+#  -[ ] see the TODO inside `._rpc._errors_relayed_via_ipc()`
+#
+# -[ ] impl a `open_debug_request()` which encaps all
+#   `request_root_stdio_lock()` task scheduling deats
+#   + `DebugStatus` state mgmt; which should prolly be re-branded as
+#   a `DebugRequest` type anyway AND with suppoort for bg-thread
+#   (from root actor) usage?
+#
+# -[ ] handle the `xonsh` case for bg-root-threads in the SIGINT
+#     handler!
+#   -[ ] do we need to do the same for subactors?
+#   -[ ] make the failing tests finally pass XD
+#
+# -[ ] simplify `maybe_wait_for_debugger()` to be a root-task only
+#     API?
+#   -[ ] currently it's implemented as that so might as well make it
+#     formal?

tractor/devx/debug/_post_mortem.py

@@ -0,0 +1,412 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+Post-mortem debugging APIs and surrounding machinery for both
+sync and async contexts.
+
+Generally we maintain the same semantics a `pdb.post.mortem()` but
+with actor-tree-wide sync/cooperation around any (sub)actor's use of
+the root's TTY.
+
+'''
+from __future__ import annotations
+import bdb
+from contextlib import (
+    AbstractContextManager,
+    contextmanager as cm,
+    nullcontext,
+)
+from functools import (
+    partial,
+)
+import inspect
+import sys
+import traceback
+from typing import (
+    Callable,
+    Sequence,
+    Type,
+    TYPE_CHECKING,
+)
+from types import (
+    TracebackType,
+    FrameType,
+)
+
+from msgspec import Struct
+import trio
+from tractor._exceptions import (
+    NoRuntime,
+)
+from tractor import _state
+from tractor._state import (
+    current_actor,
+    debug_mode,
+)
+from tractor.log import get_logger
+from tractor._exceptions import (
+    is_multi_cancelled,
+)
+from ._trace import (
+    _pause,
+)
+from ._tty_lock import (
+    DebugStatus,
+)
+from ._repl import (
+    PdbREPL,
+    mk_pdb,
+    TractorConfig as TractorConfig,
+)
+
+if TYPE_CHECKING:
+    from trio.lowlevel import Task
+    from tractor._runtime import (
+        Actor,
+    )
+
+_crash_msg: str = (
+    'Opening a pdb REPL in crashed actor'
+)
+
+log = get_logger(__package__)
+
+
+class BoxedMaybeException(Struct):
+    '''
+    Box a maybe-exception for post-crash introspection usage
+    from the body of a `open_crash_handler()` scope.
+
+    '''
+    value: BaseException|None = None
+
+    # handler can suppress crashes dynamically
+    raise_on_exit: bool|Sequence[Type[BaseException]] = True
+
+    def pformat(self) -> str:
+        '''
+        Repr the boxed `.value` error in more-than-string
+        repr form.
+
+        '''
+        if not self.value:
+            return f'<{type(self).__name__}( .value=None )>'
+
+        return (
+            f'<{type(self.value).__name__}(\n'
+            f' |_.value = {self.value}\n'
+            f')>\n'
+        )
+
+    __repr__ = pformat
+
+
+def _post_mortem(
+    repl: PdbREPL,  # normally passed by `_pause()`
+
+    # XXX all `partial`-ed in by `post_mortem()` below!
+    tb: TracebackType,
+    api_frame: FrameType,
+
+    shield: bool = False,
+    hide_tb: bool = True,
+
+    # maybe pre/post REPL entry
+    repl_fixture: (
+        AbstractContextManager[bool]
+        |None
+    ) = None,
+
+    boxed_maybe_exc: BoxedMaybeException|None = None,
+
+) -> None:
+    '''
+    Enter the ``pdbpp`` port mortem entrypoint using our custom
+    debugger instance.
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    # maybe enter any user fixture
+    enter_repl: bool = DebugStatus.maybe_enter_repl_fixture(
+        repl=repl,
+        repl_fixture=repl_fixture,
+        boxed_maybe_exc=boxed_maybe_exc,
+    )
+    try:
+        if not enter_repl:
+            # XXX, trigger `.release()` below immediately!
+            return
+        try:
+            actor: Actor = current_actor()
+            actor_repr: str = str(actor.uid)
+            # ^TODO, instead a nice runtime-info + maddr + uid?
+            # -[ ] impl a `Actor.__repr()__`??
+            #  |_ <task>:<thread> @ <actor>
+
+        except NoRuntime:
+            actor_repr: str = '<no-actor-runtime?>'
+
+        try:
+            task_repr: Task = trio.lowlevel.current_task()
+        except RuntimeError:
+            task_repr: str = '<unknown-Task>'
+
+        # TODO: print the actor supervion tree up to the root
+        # here! Bo
+        log.pdb(
+            f'{_crash_msg}\n'
+            f'x>(\n'
+            f' |_ {task_repr} @ {actor_repr}\n'
+
+        )
+
+        # XXX NOTE(s) on `pdbp.xpm()` version..
+        #
+        # - seems to lose the up-stack tb-info?
+        # - currently we're (only) replacing this from `pdbp.xpm()`
+        #   to add the `end=''` to the print XD
+        #
+        print(traceback.format_exc(), end='')
+        caller_frame: FrameType = api_frame.f_back
+
+        # NOTE, see the impl details of these in the lib to
+        # understand usage:
+        # - `pdbp.post_mortem()`
+        # - `pdbp.xps()`
+        # - `bdb.interaction()`
+        repl.reset()
+        repl.interaction(
+            frame=caller_frame,
+            # frame=None,
+            traceback=tb,
+        )
+    finally:
+        # XXX NOTE XXX: this is abs required to avoid hangs!
+        #
+        # Since we presume the post-mortem was enaged to
+        # a task-ending error, we MUST release the local REPL request
+        # so that not other local task nor the root remains blocked!
+        DebugStatus.release()
+
+
+async def post_mortem(
+    *,
+    tb: TracebackType|None = None,
+    api_frame: FrameType|None = None,
+    hide_tb: bool = False,
+
+    # TODO: support shield here just like in `pause()`?
+    # shield: bool = False,
+
+    **_pause_kwargs,
+
+) -> None:
+    '''
+    Our builtin async equivalient of `pdb.post_mortem()` which can be
+    used inside exception handlers.
+
+    It's also used for the crash handler when `debug_mode == True` ;)
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    tb: TracebackType = tb or sys.exc_info()[2]
+
+    # TODO: do upward stack scan for highest @api_frame and
+    # use its parent frame as the expected user-app code
+    # interact point.
+    api_frame: FrameType = api_frame or inspect.currentframe()
+
+    # TODO, move to submod `._pausing` or ._api? _trace
+    await _pause(
+        debug_func=partial(
+            _post_mortem,
+            api_frame=api_frame,
+            tb=tb,
+        ),
+        hide_tb=hide_tb,
+        **_pause_kwargs
+    )
+
+
+async def _maybe_enter_pm(
+    err: BaseException,
+    *,
+    tb: TracebackType|None = None,
+    api_frame: FrameType|None = None,
+    hide_tb: bool = False,
+
+    # only enter debugger REPL when returns `True`
+    debug_filter: Callable[
+        [BaseException|BaseExceptionGroup],
+        bool,
+    ] = lambda err: not is_multi_cancelled(err),
+    **_pause_kws,
+):
+    if (
+        debug_mode()
+
+        # NOTE: don't enter debug mode recursively after quitting pdb
+        # Iow, don't re-enter the repl if the `quit` command was issued
+        # by the user.
+        and not isinstance(err, bdb.BdbQuit)
+
+        # XXX: if the error is the likely result of runtime-wide
+        # cancellation, we don't want to enter the debugger since
+        # there's races between when the parent actor has killed all
+        # comms and when the child tries to contact said parent to
+        # acquire the tty lock.
+
+        # Really we just want to mostly avoid catching KBIs here so there
+        # might be a simpler check we can do?
+        and
+        debug_filter(err)
+    ):
+        api_frame: FrameType = api_frame or inspect.currentframe()
+        tb: TracebackType = tb or sys.exc_info()[2]
+        await post_mortem(
+            api_frame=api_frame,
+            tb=tb,
+            **_pause_kws,
+        )
+        return True
+
+    else:
+        return False
+
+
+# TODO: better naming and what additionals?
+# - [ ] optional runtime plugging?
+# - [ ] detection for sync vs. async code?
+# - [ ] specialized REPL entry when in distributed mode?
+# -[x] hide tb by def
+# - [x] allow ignoring kbi Bo
+@cm
+def open_crash_handler(
+    catch: set[BaseException] = {
+        BaseException,
+    },
+    ignore: set[BaseException] = {
+        KeyboardInterrupt,
+        trio.Cancelled,
+    },
+    hide_tb: bool = True,
+
+    repl_fixture: (
+        AbstractContextManager[bool]  # pre/post REPL entry
+        |None
+    ) = None,
+    raise_on_exit: bool|Sequence[Type[BaseException]] = True,
+):
+    '''
+    Generic "post mortem" crash handler using `pdbp` REPL debugger.
+
+    We expose this as a CLI framework addon to both `click` and
+    `typer` users so they can quickly wrap cmd endpoints which get
+    automatically wrapped to use the runtime's `debug_mode: bool`
+    AND `pdbp.pm()` around any code that is PRE-runtime entry
+    - any sync code which runs BEFORE the main call to
+      `trio.run()`.
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    # TODO, yield a `outcome.Error`-like boxed type?
+    # -[~] use `outcome.Value/Error` X-> frozen!
+    # -[x] write our own..?
+    # -[ ] consider just wtv is used by `pytest.raises()`?
+    #
+    boxed_maybe_exc = BoxedMaybeException(
+        raise_on_exit=raise_on_exit,
+    )
+    err: BaseException
+    try:
+        yield boxed_maybe_exc
+    except tuple(catch) as err:
+        boxed_maybe_exc.value = err
+        if (
+            type(err) not in ignore
+            and
+            not is_multi_cancelled(
+                err,
+                ignore_nested=ignore
+            )
+        ):
+            try:
+                # use our re-impl-ed version of `pdbp.xpm()`
+                _post_mortem(
+                    repl=mk_pdb(),
+                    tb=sys.exc_info()[2],
+                    api_frame=inspect.currentframe().f_back,
+                    hide_tb=hide_tb,
+
+                    repl_fixture=repl_fixture,
+                    boxed_maybe_exc=boxed_maybe_exc,
+                )
+            except bdb.BdbQuit:
+                __tracebackhide__: bool = False
+                raise err
+
+        if (
+            raise_on_exit is True
+            or (
+                raise_on_exit is not False
+                and (
+                    set(raise_on_exit)
+                    and
+                    type(err) in raise_on_exit
+                )
+            )
+            and
+            boxed_maybe_exc.raise_on_exit == raise_on_exit
+        ):
+            raise err
+
+
+@cm
+def maybe_open_crash_handler(
+    pdb: bool|None = None,
+    hide_tb: bool = True,
+
+    **kwargs,
+):
+    '''
+    Same as `open_crash_handler()` but with bool input flag
+    to allow conditional handling.
+
+    Normally this is used with CLI endpoints such that if the --pdb
+    flag is passed the pdb REPL is engaed on any crashes B)
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    if pdb is None:
+        pdb: bool = _state.is_debug_mode()
+
+    rtctx = nullcontext(
+        enter_result=BoxedMaybeException()
+    )
+    if pdb:
+        rtctx = open_crash_handler(
+            hide_tb=hide_tb,
+            **kwargs,
+        )
+
+    with rtctx as boxed_maybe_exc:
+        yield boxed_maybe_exc

tractor/devx/debug/_repl.py

@@ -0,0 +1,207 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+`pdpp.Pdb` extentions/customization and other delegate usage.
+
+'''
+from functools import (
+    cached_property,
+)
+import os
+
+import pdbp
+from tractor._state import (
+    is_root_process,
+)
+
+from ._tty_lock import (
+    Lock,
+    DebugStatus,
+)
+
+
+class TractorConfig(pdbp.DefaultConfig):
+    '''
+    Custom `pdbp` config which tries to use the best tradeoff
+    between pretty and minimal.
+
+    '''
+    use_pygments: bool = True
+    sticky_by_default: bool = False
+    enable_hidden_frames: bool = True
+
+    # much thanks @mdmintz for the hot tip!
+    # fixes line spacing issue when resizing terminal B)
+    truncate_long_lines: bool = False
+
+    # ------ - ------
+    # our own custom config vars mostly
+    # for syncing with the actor tree's singleton
+    # TTY `Lock`.
+
+
+class PdbREPL(pdbp.Pdb):
+    '''
+    Add teardown hooks and local state describing any
+    ongoing TTY `Lock` request dialog.
+
+    '''
+    # override the pdbp config with our coolio one
+    # NOTE: this is only loaded when no `~/.pdbrc` exists
+    # so we should prolly pass it into the .__init__() instead?
+    # i dunno, see the `DefaultFactory` and `pdb.Pdb` impls.
+    DefaultConfig = TractorConfig
+
+    status = DebugStatus
+
+    # NOTE: see details in stdlib's `bdb.py`
+    # def user_exception(self, frame, exc_info):
+    #     '''
+    #     Called when we stop on an exception.
+    #     '''
+    #     log.warning(
+    #         'Exception during REPL sesh\n\n'
+    #         f'{frame}\n\n'
+    #         f'{exc_info}\n\n'
+    #     )
+
+    # NOTE: this actually hooks but i don't see anyway to detect
+    # if an error was caught.. this is why currently we just always
+    # call `DebugStatus.release` inside `_post_mortem()`.
+    # def preloop(self):
+    #     print('IN PRELOOP')
+    #     super().preloop()
+
+    # TODO: cleaner re-wrapping of all this?
+    # -[ ] figure out how to disallow recursive .set_trace() entry
+    #     since that'll cause deadlock for us.
+    # -[ ] maybe a `@cm` to call `super().<same_meth_name>()`?
+    # -[ ] look at hooking into the `pp` hook specially with our
+    #     own set of pretty-printers?
+    #    * `.pretty_struct.Struct.pformat()`
+    #    * `.pformat(MsgType.pld)`
+    #    * `.pformat(Error.tb_str)`?
+    #    * .. maybe more?
+    #
+    def set_continue(self):
+        try:
+            super().set_continue()
+        finally:
+            # NOTE: for subactors the stdio lock is released via the
+            # allocated RPC locker task, so for root we have to do it
+            # manually.
+            if (
+                is_root_process()
+                and
+                Lock._debug_lock.locked()
+                and
+                DebugStatus.is_main_trio_thread()
+            ):
+                # Lock.release(raise_on_thread=False)
+                Lock.release()
+
+            # XXX AFTER `Lock.release()` for root local repl usage
+            DebugStatus.release()
+
+    def set_quit(self):
+        try:
+            super().set_quit()
+        finally:
+            if (
+                is_root_process()
+                and
+                Lock._debug_lock.locked()
+                and
+                DebugStatus.is_main_trio_thread()
+            ):
+                # Lock.release(raise_on_thread=False)
+                Lock.release()
+
+            # XXX after `Lock.release()` for root local repl usage
+            DebugStatus.release()
+
+    # XXX NOTE: we only override this because apparently the stdlib pdb
+    # bois likes to touch the SIGINT handler as much as i like to touch
+    # my d$%&.
+    def _cmdloop(self):
+        self.cmdloop()
+
+    @cached_property
+    def shname(self) -> str | None:
+        '''
+        Attempt to return the login shell name with a special check for
+        the infamous `xonsh` since it seems to have some issues much
+        different from std shells when it comes to flushing the prompt?
+
+        '''
+        # SUPER HACKY and only really works if `xonsh` is not used
+        # before spawning further sub-shells..
+        shpath = os.getenv('SHELL', None)
+
+        if shpath:
+            if (
+                os.getenv('XONSH_LOGIN', default=False)
+                or 'xonsh' in shpath
+            ):
+                return 'xonsh'
+
+            return os.path.basename(shpath)
+
+        return None
+
+
+def mk_pdb() -> PdbREPL:
+    '''
+    Deliver a new `PdbREPL`: a multi-process safe `pdbp.Pdb`-variant
+    using the magic of `tractor`'s SC-safe IPC.
+
+    B)
+
+    Our `pdb.Pdb` subtype accomplishes multi-process safe debugging
+    by:
+
+    - mutexing access to the root process' std-streams (& thus parent
+      process TTY) via an IPC managed `Lock` singleton per
+      actor-process tree.
+
+    - temporarily overriding any subactor's SIGINT handler to shield
+      during live REPL sessions in sub-actors such that cancellation
+      is never (mistakenly) triggered by a ctrl-c and instead only by
+      explicit runtime API requests or after the
+      `pdb.Pdb.interaction()` call has returned.
+
+    FURTHER, the `pdbp.Pdb` instance is configured to be `trio`
+    "compatible" from a SIGINT handling perspective; we mask out
+    the default `pdb` handler and instead apply `trio`s default
+    which mostly addresses all issues described in:
+
+     - https://github.com/python-trio/trio/issues/1155
+
+    The instance returned from this factory should always be
+    preferred over the default `pdb[p].set_trace()` whenever using
+    a `pdb` REPL inside a `trio` based runtime.
+
+    '''
+    pdb = PdbREPL()
+
+    # XXX: These are the important flags mentioned in
+    # https://github.com/python-trio/trio/issues/1155
+    # which resolve the traceback spews to console.
+    pdb.allow_kbdint = True
+    pdb.nosigint = True
+    return pdb

tractor/devx/debug/_sigint.py

@@ -0,0 +1,333 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+A custom SIGINT handler which mainly shields actor (task)
+cancellation during REPL interaction.
+
+'''
+from __future__ import annotations
+from typing import (
+    TYPE_CHECKING,
+)
+import trio
+from tractor.log import get_logger
+from tractor._state import (
+    current_actor,
+    is_root_process,
+)
+from ._repl import (
+    PdbREPL,
+)
+from ._tty_lock import (
+    any_connected_locker_child,
+    DebugStatus,
+    Lock,
+)
+
+if TYPE_CHECKING:
+    from tractor.ipc import (
+        Channel,
+    )
+    from tractor._runtime import (
+        Actor,
+    )
+
+log = get_logger(__name__)
+
+_ctlc_ignore_header: str = (
+    'Ignoring SIGINT while debug REPL in use'
+)
+
+
+def sigint_shield(
+    signum: int,
+    frame: 'frame',  # type: ignore # noqa
+    *args,
+
+) -> None:
+    '''
+    Specialized, debugger-aware SIGINT handler.
+
+    In childred we always ignore/shield for SIGINT to avoid
+    deadlocks since cancellation should always be managed by the
+    supervising parent actor. The root actor-proces is always
+    cancelled on ctrl-c.
+
+    '''
+    __tracebackhide__: bool = True
+    actor: Actor = current_actor()
+
+    def do_cancel():
+        # If we haven't tried to cancel the runtime then do that instead
+        # of raising a KBI (which may non-gracefully destroy
+        # a ``trio.run()``).
+        if not actor._cancel_called:
+            actor.cancel_soon()
+
+        # If the runtime is already cancelled it likely means the user
+        # hit ctrl-c again because teardown didn't fully take place in
+        # which case we do the "hard" raising of a local KBI.
+        else:
+            raise KeyboardInterrupt
+
+    # only set in the actor actually running the REPL
+    repl: PdbREPL|None = DebugStatus.repl
+
+    # TODO: maybe we should flatten out all these cases using
+    # a match/case?
+    #
+    # root actor branch that reports whether or not a child
+    # has locked debugger.
+    if is_root_process():
+        # log.warning(
+        log.devx(
+            'Handling SIGINT in root actor\n'
+            f'{Lock.repr()}'
+            f'{DebugStatus.repr()}\n'
+        )
+        # try to see if the supposed (sub)actor in debug still
+        # has an active connection to *this* actor, and if not
+        # it's likely they aren't using the TTY lock / debugger
+        # and we should propagate SIGINT normally.
+        any_connected: bool = any_connected_locker_child()
+
+        problem = (
+            f'root {actor.uid} handling SIGINT\n'
+            f'any_connected: {any_connected}\n\n'
+
+            f'{Lock.repr()}\n'
+        )
+
+        if (
+            (ctx := Lock.ctx_in_debug)
+            and
+            (uid_in_debug := ctx.chan.uid) # "someone" is (ostensibly) using debug `Lock`
+        ):
+            name_in_debug: str = uid_in_debug[0]
+            assert not repl
+            # if not repl:  # but it's NOT us, the root actor.
+            # sanity: since no repl ref is set, we def shouldn't
+            # be the lock owner!
+            assert name_in_debug != 'root'
+
+            # IDEAL CASE: child has REPL as expected
+            if any_connected:  # there are subactors we can contact
+                # XXX: only if there is an existing connection to the
+                # (sub-)actor in debug do we ignore SIGINT in this
+                # parent! Otherwise we may hang waiting for an actor
+                # which has already terminated to unlock.
+                #
+                # NOTE: don't emit this with `.pdb()` level in
+                # root without a higher level.
+                log.runtime(
+                    _ctlc_ignore_header
+                    +
+                    f' by child '
+                    f'{uid_in_debug}\n'
+                )
+                problem = None
+
+            else:
+                problem += (
+                    '\n'
+                    f'A `pdb` REPL is SUPPOSEDLY in use by child {uid_in_debug}\n'
+                    f'BUT, no child actors are IPC contactable!?!?\n'
+                )
+
+        # IDEAL CASE: root has REPL as expected
+        else:
+            # root actor still has this SIGINT handler active without
+            # an actor using the `Lock` (a bug state) ??
+            # => so immediately cancel any stale lock cs and revert
+            # the handler!
+            if not DebugStatus.repl:
+                # TODO: WHEN should we revert back to ``trio``
+                # handler if this one is stale?
+                # -[ ] maybe after a counts work of ctl-c mashes?
+                # -[ ] use a state var like `stale_handler: bool`?
+                problem += (
+                    'No subactor is using a `pdb` REPL according `Lock.ctx_in_debug`?\n'
+                    'BUT, the root should be using it, WHY this handler ??\n\n'
+                    'So either..\n'
+                    '- some root-thread is using it but has no `.repl` set?, OR\n'
+                    '- something else weird is going on outside the runtime!?\n'
+                )
+            else:
+                # NOTE: since we emit this msg on ctl-c, we should
+                # also always re-print the prompt the tail block!
+                log.pdb(
+                    _ctlc_ignore_header
+                    +
+                    f' by root actor..\n'
+                    f'{DebugStatus.repl_task}\n'
+                    f' |_{repl}\n'
+                )
+                problem = None
+
+        # XXX if one is set it means we ARE NOT operating an ideal
+        # case where a child subactor or us (the root) has the
+        # lock without any other detected problems.
+        if problem:
+
+            # detect, report and maybe clear a stale lock request
+            # cancel scope.
+            lock_cs: trio.CancelScope = Lock.get_locking_task_cs()
+            maybe_stale_lock_cs: bool = (
+                lock_cs is not None
+                and not lock_cs.cancel_called
+            )
+            if maybe_stale_lock_cs:
+                problem += (
+                    '\n'
+                    'Stale `Lock.ctx_in_debug._scope: CancelScope` detected?\n'
+                    f'{Lock.ctx_in_debug}\n\n'
+
+                    '-> Calling ctx._scope.cancel()!\n'
+                )
+                lock_cs.cancel()
+
+            # TODO: wen do we actually want/need this, see above.
+            # DebugStatus.unshield_sigint()
+            log.warning(problem)
+
+    # child actor that has locked the debugger
+    elif not is_root_process():
+        log.debug(
+            f'Subactor {actor.uid} handling SIGINT\n\n'
+            f'{Lock.repr()}\n'
+        )
+
+        rent_chan: Channel = actor._parent_chan
+        if (
+            rent_chan is None
+            or
+            not rent_chan.connected()
+        ):
+            log.warning(
+                'This sub-actor thinks it is debugging '
+                'but it has no connection to its parent ??\n'
+                f'{actor.uid}\n'
+                'Allowing SIGINT propagation..'
+            )
+            DebugStatus.unshield_sigint()
+
+        repl_task: str|None = DebugStatus.repl_task
+        req_task: str|None = DebugStatus.req_task
+        if (
+            repl_task
+            and
+            repl
+        ):
+            log.pdb(
+                _ctlc_ignore_header
+                +
+                f' by local task\n\n'
+                f'{repl_task}\n'
+                f' |_{repl}\n'
+            )
+        elif req_task:
+            log.debug(
+                _ctlc_ignore_header
+                +
+                f' by local request-task and either,\n'
+                f'- someone else is already REPL-in and has the `Lock`, or\n'
+                f'- some other local task already is replin?\n\n'
+                f'{req_task}\n'
+            )
+
+        # TODO can we remove this now?
+        # -[ ] does this path ever get hit any more?
+        else:
+            msg: str = (
+                'SIGINT shield handler still active BUT, \n\n'
+            )
+            if repl_task is None:
+                msg += (
+                    '- No local task claims to be in debug?\n'
+                )
+
+            if repl is None:
+                msg += (
+                    '- No local REPL is currently active?\n'
+                )
+
+            if req_task is None:
+                msg += (
+                    '- No debug request task is active?\n'
+                )
+
+            log.warning(
+                msg
+                +
+                'Reverting handler to `trio` default!\n'
+            )
+            DebugStatus.unshield_sigint()
+
+            # XXX ensure that the reverted-to-handler actually is
+            # able to rx what should have been **this** KBI ;)
+            do_cancel()
+
+        # TODO: how to handle the case of an intermediary-child actor
+        # that **is not** marked in debug mode? See oustanding issue:
+        # https://github.com/goodboy/tractor/issues/320
+        # elif debug_mode():
+
+    # maybe redraw/print last REPL output to console since
+    # we want to alert the user that more input is expect since
+    # nothing has been done dur to ignoring sigint.
+    if (
+        DebugStatus.repl  # only when current actor has a REPL engaged
+    ):
+        flush_status: str = (
+            'Flushing stdout to ensure new prompt line!\n'
+        )
+
+        # XXX: yah, mega hack, but how else do we catch this madness XD
+        if (
+            repl.shname == 'xonsh'
+        ):
+            flush_status += (
+                '-> ALSO re-flushing due to `xonsh`..\n'
+            )
+            repl.stdout.write(repl.prompt)
+
+        # log.warning(
+        log.devx(
+            flush_status
+        )
+        repl.stdout.flush()
+
+        # TODO: better console UX to match the current "mode":
+        # -[ ] for example if in sticky mode where if there is output
+        #   detected as written to the tty we redraw this part underneath
+        #   and erase the past draw of this same bit above?
+        # repl.sticky = True
+        # repl._print_if_sticky()
+
+        # also see these links for an approach from `ptk`:
+        # https://github.com/goodboy/tractor/issues/130#issuecomment-663752040
+        # https://github.com/prompt-toolkit/python-prompt-toolkit/blob/c2c6af8a0308f9e5d7c0e28cb8a02963fe0ce07a/prompt_toolkit/patch_stdout.py
+    else:
+        log.devx(
+        # log.warning(
+            'Not flushing stdout since not needed?\n'
+            f'|_{repl}\n'
+        )
+
+    # XXX only for tracing this handler
+    log.devx('exiting SIGINT')

tractor/devx/debug/_sync.py

@@ -0,0 +1,220 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+Debugger synchronization APIs to ensure orderly access and
+non-TTY-clobbering graceful teardown.
+
+
+'''
+from __future__ import annotations
+from contextlib import (
+    asynccontextmanager as acm,
+)
+from functools import (
+    partial,
+)
+from typing import (
+    AsyncGenerator,
+    Callable,
+)
+
+from tractor.log import get_logger
+import trio
+from trio.lowlevel import (
+    current_task,
+    Task,
+)
+from tractor._context import Context
+from tractor._state import (
+    current_actor,
+    debug_mode,
+    is_root_process,
+)
+from ._repl import (
+    TractorConfig as TractorConfig,
+)
+from ._tty_lock import (
+    Lock,
+    request_root_stdio_lock,
+    any_connected_locker_child,
+)
+from ._sigint import (
+    sigint_shield as sigint_shield,
+    _ctlc_ignore_header as _ctlc_ignore_header
+)
+
+log = get_logger(__package__)
+
+
+async def maybe_wait_for_debugger(
+    poll_steps: int = 2,
+    poll_delay: float = 0.1,
+    child_in_debug: bool = False,
+
+    header_msg: str = '',
+    _ll: str = 'devx',
+
+) -> bool:  # was locked and we polled?
+
+    if (
+        not debug_mode()
+        and
+        not child_in_debug
+    ):
+        return False
+
+    logmeth: Callable = getattr(log, _ll)
+
+    msg: str = header_msg
+    if (
+        is_root_process()
+    ):
+        # If we error in the root but the debugger is
+        # engaged we don't want to prematurely kill (and
+        # thus clobber access to) the local tty since it
+        # will make the pdb repl unusable.
+        # Instead try to wait for pdb to be released before
+        # tearing down.
+        ctx_in_debug: Context|None = Lock.ctx_in_debug
+        in_debug: tuple[str, str]|None = (
+            ctx_in_debug.chan.uid
+            if ctx_in_debug
+            else None
+        )
+        if in_debug == current_actor().uid:
+            log.debug(
+                msg
+                +
+                'Root already owns the TTY LOCK'
+            )
+            return True
+
+        elif in_debug:
+            msg += (
+                f'Debug `Lock` in use by subactor\n|\n|_{in_debug}\n'
+            )
+            # TODO: could this make things more deterministic?
+            # wait to see if a sub-actor task will be
+            # scheduled and grab the tty lock on the next
+            # tick?
+            # XXX => but it doesn't seem to work..
+            # await trio.testing.wait_all_tasks_blocked(cushion=0)
+        else:
+            logmeth(
+                msg
+                +
+                'Root immediately acquired debug TTY LOCK'
+            )
+            return False
+
+        for istep in range(poll_steps):
+            if (
+                Lock.req_handler_finished is not None
+                and not Lock.req_handler_finished.is_set()
+                and in_debug is not None
+            ):
+                # caller_frame_info: str = pformat_caller_frame()
+                logmeth(
+                    msg
+                    +
+                    '\n^^ Root is waiting on tty lock release.. ^^\n'
+                    # f'{caller_frame_info}\n'
+                )
+
+                if not any_connected_locker_child():
+                    Lock.get_locking_task_cs().cancel()
+
+                with trio.CancelScope(shield=True):
+                    await Lock.req_handler_finished.wait()
+
+                log.devx(
+                    f'Subactor released debug lock\n'
+                    f'|_{in_debug}\n'
+                )
+                break
+
+            # is no subactor locking debugger currently?
+            if (
+                in_debug is None
+                and (
+                    Lock.req_handler_finished is None
+                    or Lock.req_handler_finished.is_set()
+                )
+            ):
+                logmeth(
+                    msg
+                    +
+                    'Root acquired tty lock!'
+                )
+                break
+
+            else:
+                logmeth(
+                    'Root polling for debug:\n'
+                    f'poll step: {istep}\n'
+                    f'poll delya: {poll_delay}\n\n'
+                    f'{Lock.repr()}\n'
+                )
+                with trio.CancelScope(shield=True):
+                    await trio.sleep(poll_delay)
+                    continue
+
+        return True
+
+    # else:
+    #     # TODO: non-root call for #320?
+    #     this_uid: tuple[str, str] = current_actor().uid
+    #     async with acquire_debug_lock(
+    #         subactor_uid=this_uid,
+    #     ):
+    #         pass
+    return False
+
+
+@acm
+async def acquire_debug_lock(
+    subactor_uid: tuple[str, str],
+) -> AsyncGenerator[
+    trio.CancelScope|None,
+    tuple,
+]:
+    '''
+    Request to acquire the TTY `Lock` in the root actor, release on
+    exit.
+
+    This helper is for actor's who don't actually need to acquired
+    the debugger but want to wait until the lock is free in the
+    process-tree root such that they don't clobber an ongoing pdb
+    REPL session in some peer or child!
+
+    '''
+    if not debug_mode():
+        yield None
+        return
+
+    task: Task = current_task()
+    async with trio.open_nursery() as n:
+        ctx: Context = await n.start(
+            partial(
+                request_root_stdio_lock,
+                actor_uid=subactor_uid,
+                task_uid=(task.name, id(task)),
+            )
+        )
+        yield ctx
+        ctx.cancel()

tractor/ipc/_server.py

@@ -43,7 +43,7 @@ from trio import (
     SocketListener,
 )
 
-# from ..devx import _debug
+# from ..devx import debug
 from .._exceptions import (
     TransportClosed,
 )
@@ -318,7 +318,7 @@ async def handle_stream_from_peer(
     server._no_more_peers = trio.Event()  # unset by making new
 
     # TODO, debug_mode tooling for when hackin this lower layer?
-    # with _debug.maybe_open_crash_handler(
+    # with debug.maybe_open_crash_handler(
     #     pdb=True,
     # ) as boxerr:
 
@@ -504,8 +504,8 @@ async def handle_stream_from_peer(
                 and
                 _state.is_debug_mode()
             ):
-                from ..devx import _debug
+                from ..devx import debug
-                pdb_lock = _debug.Lock
+                pdb_lock = debug.Lock
                 pdb_lock._blocked.add(uid)
 
                 # TODO: NEEEDS TO BE TESTED!
@@ -540,7 +540,7 @@ async def handle_stream_from_peer(
                                 f'last disconnected child uid: {uid}\n'
                                 f'locking child uid: {pdb_user_uid}\n'
                             )
-                            await _debug.maybe_wait_for_debugger(
+                            await debug.maybe_wait_for_debugger(
                                 child_in_debug=True
                             )
 

tractor/msg/_ops.py

@@ -608,7 +608,7 @@ async def drain_to_final_msg(
             #
             # -[ ] make sure pause points work here for REPLing
             #   the runtime itself; i.e. ensure there's no hangs!
-            # |_from tractor.devx._debug import pause
+            # |_from tractor.devx.debug import pause
             #   await pause()
 
         # NOTE: we get here if the far end was

tractor/to_asyncio.py

@@ -49,7 +49,7 @@ from tractor._state import (
     _runtime_vars,
 )
 from tractor._context import Unresolved
-from tractor.devx import _debug
+from tractor.devx import debug
 from tractor.log import (
     get_logger,
     StackLevelAdapter,
@@ -479,12 +479,12 @@ def _run_asyncio_task(
     if (
         debug_mode()
         and
-        (greenback := _debug.maybe_import_greenback(
+        (greenback := debug.maybe_import_greenback(
             force_reload=True,
             raise_not_found=False,
         ))
     ):
-        log.info(
+        log.devx(
             f'Bestowing `greenback` portal for `asyncio`-task\n'
             f'{task}\n'
         )
@@ -841,7 +841,7 @@ async def translate_aio_errors(
     except BaseException as _trio_err:
         trio_err = chan._trio_err = _trio_err
         # await tractor.pause(shield=True)  # workx!
-        entered: bool = await _debug._maybe_enter_pm(
+        entered: bool = await debug._maybe_enter_pm(
             trio_err,
             api_frame=inspect.currentframe(),
         )
@@ -1406,7 +1406,7 @@ def run_as_asyncio_guest(
             )
             # XXX make it obvi we know this isn't supported yet!
             assert 0
-            # await _debug.maybe_init_greenback(
+            # await debug.maybe_init_greenback(
             #     force_reload=True,
             # )
 

tractor/trionics/__init__.py

@@ -31,4 +31,5 @@ from ._broadcast import (
 )
 from ._beg import (
     collapse_eg as collapse_eg,
+    maybe_collapse_eg as maybe_collapse_eg,
 )