Reorg `.devx.debug` into sub-mods!

Which cleans out the pkg-mod to just the expected exports with (its
longstanding todo comment list) and thus a separation-of-concerns
and smaller mod-file sizes via the following new sub-mods:
- `._trace` for the `.pause()`/`breakpoint()`/`pdb.set_trace()`-style
  APIs including all sync-caller variants.
- `._post_mortem` to contain our async `.post_mortem()` and all other
  public crash handling APIs for use from sync callers.
- `._sync` for the high-level syncing helper-routines used throughout the
  runtime to avoid multi-proc TTY use collisions.

And also,
- remove `hide_runtime_frames()` since moved to `.devx._frame_stack`.

tractor/devx/debug/_post_mortem.py

@@ -0,0 +1,411 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+Post-mortem debugging APIs and surrounding machinery for both
+sync and async contexts.
+
+Generally we maintain the same semantics a `pdb.post.mortem()` but
+with actor-tree-wide sync/cooperation around any (sub)actor's use of
+the root's TTY.
+
+'''
+from __future__ import annotations
+import bdb
+from contextlib import (
+    AbstractContextManager,
+    contextmanager as cm,
+    nullcontext,
+)
+from functools import (
+    partial,
+)
+import inspect
+import sys
+import traceback
+from typing import (
+    Callable,
+    Sequence,
+    Type,
+    TYPE_CHECKING,
+)
+from types import (
+    TracebackType,
+    FrameType,
+)
+
+from msgspec import Struct
+import trio
+from tractor._exceptions import (
+    NoRuntime,
+)
+from tractor import _state
+from tractor._state import (
+    current_actor,
+    debug_mode,
+)
+from tractor.log import get_logger
+from tractor._exceptions import (
+    is_multi_cancelled,
+)
+from ._trace import (
+    _pause,
+    _maybe_open_repl_fixture,
+)
+from ._tty_lock import (
+    DebugStatus,
+)
+from ._repl import (
+    PdbREPL,
+    mk_pdb,
+    TractorConfig as TractorConfig,
+)
+
+if TYPE_CHECKING:
+    from trio.lowlevel import Task
+    from tractor._runtime import (
+        Actor,
+    )
+
+_crash_msg: str = (
+    'Opening a pdb REPL in crashed actor'
+)
+
+log = get_logger(__package__)
+
+
+class BoxedMaybeException(Struct):
+    '''
+    Box a maybe-exception for post-crash introspection usage
+    from the body of a `open_crash_handler()` scope.
+
+    '''
+    value: BaseException|None = None
+
+    # handler can suppress crashes dynamically
+    raise_on_exit: bool|Sequence[Type[BaseException]] = True
+
+    def pformat(self) -> str:
+        '''
+        Repr the boxed `.value` error in more-than-string
+        repr form.
+
+        '''
+        if not self.value:
+            return f'<{type(self).__name__}( .value=None )>\n'
+
+        return (
+            f'<{type(self.value).__name__}(\n'
+            f' |_.value = {self.value}\n'
+            f')>\n'
+        )
+
+    __repr__ = pformat
+
+
+def _post_mortem(
+    repl: PdbREPL,  # normally passed by `_pause()`
+
+    # XXX all `partial`-ed in by `post_mortem()` below!
+    tb: TracebackType,
+    api_frame: FrameType,
+
+    shield: bool = False,
+    hide_tb: bool = True,
+
+    # maybe pre/post REPL entry
+    repl_fixture: (
+        AbstractContextManager[bool]
+        |None
+    ) = None,
+
+    boxed_maybe_exc: BoxedMaybeException|None = None,
+
+) -> None:
+    '''
+    Enter the ``pdbpp`` port mortem entrypoint using our custom
+    debugger instance.
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    with _maybe_open_repl_fixture(
+        repl_fixture=repl_fixture,
+        boxed_maybe_exc=boxed_maybe_exc,
+    ) as enter_repl:
+        if not enter_repl:
+            return
+
+        try:
+            actor: Actor = current_actor()
+            actor_repr: str = str(actor.uid)
+            # ^TODO, instead a nice runtime-info + maddr + uid?
+            # -[ ] impl a `Actor.__repr()__`??
+            #  |_ <task>:<thread> @ <actor>
+
+        except NoRuntime:
+            actor_repr: str = '<no-actor-runtime?>'
+
+        try:
+            task_repr: Task = trio.lowlevel.current_task()
+        except RuntimeError:
+            task_repr: str = '<unknown-Task>'
+
+        # TODO: print the actor supervion tree up to the root
+        # here! Bo
+        log.pdb(
+            f'{_crash_msg}\n'
+            f'x>(\n'
+            f' |_ {task_repr} @ {actor_repr}\n'
+
+        )
+
+        # XXX NOTE(s) on `pdbp.xpm()` version..
+        #
+        # - seems to lose the up-stack tb-info?
+        # - currently we're (only) replacing this from `pdbp.xpm()`
+        #   to add the `end=''` to the print XD
+        #
+        print(traceback.format_exc(), end='')
+        caller_frame: FrameType = api_frame.f_back
+
+        # NOTE, see the impl details of these in the lib to
+        # understand usage:
+        # - `pdbp.post_mortem()`
+        # - `pdbp.xps()`
+        # - `bdb.interaction()`
+        repl.reset()
+        repl.interaction(
+            frame=caller_frame,
+            # frame=None,
+            traceback=tb,
+        )
+
+        # XXX NOTE XXX: this is abs required to avoid hangs!
+        #
+        # Since we presume the post-mortem was enaged to
+        # a task-ending error, we MUST release the local REPL request
+        # so that not other local task nor the root remains blocked!
+        DebugStatus.release()
+
+
+async def post_mortem(
+    *,
+    tb: TracebackType|None = None,
+    api_frame: FrameType|None = None,
+    hide_tb: bool = False,
+
+    # TODO: support shield here just like in `pause()`?
+    # shield: bool = False,
+
+    **_pause_kwargs,
+
+) -> None:
+    '''
+    Our builtin async equivalient of `pdb.post_mortem()` which can be
+    used inside exception handlers.
+
+    It's also used for the crash handler when `debug_mode == True` ;)
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    tb: TracebackType = tb or sys.exc_info()[2]
+
+    # TODO: do upward stack scan for highest @api_frame and
+    # use its parent frame as the expected user-app code
+    # interact point.
+    api_frame: FrameType = api_frame or inspect.currentframe()
+
+    # TODO, move to submod `._pausing` or ._api? _trace
+    await _pause(
+        debug_func=partial(
+            _post_mortem,
+            api_frame=api_frame,
+            tb=tb,
+        ),
+        hide_tb=hide_tb,
+        **_pause_kwargs
+    )
+
+
+async def _maybe_enter_pm(
+    err: BaseException,
+    *,
+    tb: TracebackType|None = None,
+    api_frame: FrameType|None = None,
+    hide_tb: bool = False,
+
+    # only enter debugger REPL when returns `True`
+    debug_filter: Callable[
+        [BaseException|BaseExceptionGroup],
+        bool,
+    ] = lambda err: not is_multi_cancelled(err),
+    **_pause_kws,
+
+):
+    if (
+        debug_mode()
+
+        # NOTE: don't enter debug mode recursively after quitting pdb
+        # Iow, don't re-enter the repl if the `quit` command was issued
+        # by the user.
+        and not isinstance(err, bdb.BdbQuit)
+
+        # XXX: if the error is the likely result of runtime-wide
+        # cancellation, we don't want to enter the debugger since
+        # there's races between when the parent actor has killed all
+        # comms and when the child tries to contact said parent to
+        # acquire the tty lock.
+
+        # Really we just want to mostly avoid catching KBIs here so there
+        # might be a simpler check we can do?
+        and
+        debug_filter(err)
+    ):
+        api_frame: FrameType = api_frame or inspect.currentframe()
+        tb: TracebackType = tb or sys.exc_info()[2]
+        await post_mortem(
+            api_frame=api_frame,
+            tb=tb,
+            **_pause_kws,
+        )
+        return True
+
+    else:
+        return False
+
+
+# TODO: better naming and what additionals?
+# - [ ] optional runtime plugging?
+# - [ ] detection for sync vs. async code?
+# - [ ] specialized REPL entry when in distributed mode?
+# -[x] hide tb by def
+# - [x] allow ignoring kbi Bo
+@cm
+def open_crash_handler(
+    catch: set[BaseException] = {
+        BaseException,
+    },
+    ignore: set[BaseException] = {
+        KeyboardInterrupt,
+        trio.Cancelled,
+    },
+    hide_tb: bool = True,
+
+    repl_fixture: (
+        AbstractContextManager[bool]  # pre/post REPL entry
+        |None
+    ) = None,
+    raise_on_exit: bool|Sequence[Type[BaseException]] = True,
+):
+    '''
+    Generic "post mortem" crash handler using `pdbp` REPL debugger.
+
+    We expose this as a CLI framework addon to both `click` and
+    `typer` users so they can quickly wrap cmd endpoints which get
+    automatically wrapped to use the runtime's `debug_mode: bool`
+    AND `pdbp.pm()` around any code that is PRE-runtime entry
+    - any sync code which runs BEFORE the main call to
+      `trio.run()`.
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    # TODO, yield a `outcome.Error`-like boxed type?
+    # -[~] use `outcome.Value/Error` X-> frozen!
+    # -[x] write our own..?
+    # -[ ] consider just wtv is used by `pytest.raises()`?
+    #
+    boxed_maybe_exc = BoxedMaybeException(
+        raise_on_exit=raise_on_exit,
+    )
+    err: BaseException
+    try:
+        yield boxed_maybe_exc
+    except tuple(catch) as err:
+        boxed_maybe_exc.value = err
+        if (
+            type(err) not in ignore
+            and
+            not is_multi_cancelled(
+                err,
+                ignore_nested=ignore
+            )
+        ):
+            try:
+                # use our re-impl-ed version of `pdbp.xpm()`
+                _post_mortem(
+                    repl=mk_pdb(),
+                    tb=sys.exc_info()[2],
+                    api_frame=inspect.currentframe().f_back,
+                    hide_tb=hide_tb,
+
+                    repl_fixture=repl_fixture,
+                    boxed_maybe_exc=boxed_maybe_exc,
+                )
+            except bdb.BdbQuit:
+                __tracebackhide__: bool = False
+                raise err
+
+        if (
+            raise_on_exit is True
+            or (
+                raise_on_exit is not False
+                and (
+                    set(raise_on_exit)
+                    and
+                    type(err) in raise_on_exit
+                )
+            )
+            and
+            boxed_maybe_exc.raise_on_exit == raise_on_exit
+        ):
+            raise err
+
+
+@cm
+def maybe_open_crash_handler(
+    pdb: bool|None = None,
+    hide_tb: bool = True,
+
+    **kwargs,
+):
+    '''
+    Same as `open_crash_handler()` but with bool input flag
+    to allow conditional handling.
+
+    Normally this is used with CLI endpoints such that if the --pdb
+    flag is passed the pdb REPL is engaed on any crashes B)
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    if pdb is None:
+        pdb: bool = _state.is_debug_mode()
+
+    rtctx = nullcontext(
+        enter_result=BoxedMaybeException()
+    )
+    if pdb:
+        rtctx = open_crash_handler(
+            hide_tb=hide_tb,
+            **kwargs,
+        )
+
+    with rtctx as boxed_maybe_exc:
+        yield boxed_maybe_exc

tractor/devx/debug/_sync.py

@@ -0,0 +1,220 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+Debugger synchronization APIs to ensure orderly access and
+non-TTY-clobbering graceful teardown.
+
+
+'''
+from __future__ import annotations
+from contextlib import (
+    asynccontextmanager as acm,
+)
+from functools import (
+    partial,
+)
+from typing import (
+    AsyncGenerator,
+    Callable,
+)
+
+from tractor.log import get_logger
+import trio
+from trio.lowlevel import (
+    current_task,
+    Task,
+)
+from tractor._context import Context
+from tractor._state import (
+    current_actor,
+    debug_mode,
+    is_root_process,
+)
+from ._repl import (
+    TractorConfig as TractorConfig,
+)
+from ._tty_lock import (
+    Lock,
+    request_root_stdio_lock,
+    any_connected_locker_child,
+)
+from ._sigint import (
+    sigint_shield as sigint_shield,
+    _ctlc_ignore_header as _ctlc_ignore_header
+)
+
+log = get_logger(__package__)
+
+
+async def maybe_wait_for_debugger(
+    poll_steps: int = 2,
+    poll_delay: float = 0.1,
+    child_in_debug: bool = False,
+
+    header_msg: str = '',
+    _ll: str = 'devx',
+
+) -> bool:  # was locked and we polled?
+
+    if (
+        not debug_mode()
+        and
+        not child_in_debug
+    ):
+        return False
+
+    logmeth: Callable = getattr(log, _ll)
+
+    msg: str = header_msg
+    if (
+        is_root_process()
+    ):
+        # If we error in the root but the debugger is
+        # engaged we don't want to prematurely kill (and
+        # thus clobber access to) the local tty since it
+        # will make the pdb repl unusable.
+        # Instead try to wait for pdb to be released before
+        # tearing down.
+        ctx_in_debug: Context|None = Lock.ctx_in_debug
+        in_debug: tuple[str, str]|None = (
+            ctx_in_debug.chan.uid
+            if ctx_in_debug
+            else None
+        )
+        if in_debug == current_actor().uid:
+            log.debug(
+                msg
+                +
+                'Root already owns the TTY LOCK'
+            )
+            return True
+
+        elif in_debug:
+            msg += (
+                f'Debug `Lock` in use by subactor\n|\n|_{in_debug}\n'
+            )
+            # TODO: could this make things more deterministic?
+            # wait to see if a sub-actor task will be
+            # scheduled and grab the tty lock on the next
+            # tick?
+            # XXX => but it doesn't seem to work..
+            # await trio.testing.wait_all_tasks_blocked(cushion=0)
+        else:
+            logmeth(
+                msg
+                +
+                'Root immediately acquired debug TTY LOCK'
+            )
+            return False
+
+        for istep in range(poll_steps):
+            if (
+                Lock.req_handler_finished is not None
+                and not Lock.req_handler_finished.is_set()
+                and in_debug is not None
+            ):
+                # caller_frame_info: str = pformat_caller_frame()
+                logmeth(
+                    msg
+                    +
+                    '\n^^ Root is waiting on tty lock release.. ^^\n'
+                    # f'{caller_frame_info}\n'
+                )
+
+                if not any_connected_locker_child():
+                    Lock.get_locking_task_cs().cancel()
+
+                with trio.CancelScope(shield=True):
+                    await Lock.req_handler_finished.wait()
+
+                log.devx(
+                    f'Subactor released debug lock\n'
+                    f'|_{in_debug}\n'
+                )
+                break
+
+            # is no subactor locking debugger currently?
+            if (
+                in_debug is None
+                and (
+                    Lock.req_handler_finished is None
+                    or Lock.req_handler_finished.is_set()
+                )
+            ):
+                logmeth(
+                    msg
+                    +
+                    'Root acquired tty lock!'
+                )
+                break
+
+            else:
+                logmeth(
+                    'Root polling for debug:\n'
+                    f'poll step: {istep}\n'
+                    f'poll delya: {poll_delay}\n\n'
+                    f'{Lock.repr()}\n'
+                )
+                with trio.CancelScope(shield=True):
+                    await trio.sleep(poll_delay)
+                    continue
+
+        return True
+
+    # else:
+    #     # TODO: non-root call for #320?
+    #     this_uid: tuple[str, str] = current_actor().uid
+    #     async with acquire_debug_lock(
+    #         subactor_uid=this_uid,
+    #     ):
+    #         pass
+    return False
+
+
+@acm
+async def acquire_debug_lock(
+    subactor_uid: tuple[str, str],
+) -> AsyncGenerator[
+    trio.CancelScope|None,
+    tuple,
+]:
+    '''
+    Request to acquire the TTY `Lock` in the root actor, release on
+    exit.
+
+    This helper is for actor's who don't actually need to acquired
+    the debugger but want to wait until the lock is free in the
+    process-tree root such that they don't clobber an ongoing pdb
+    REPL session in some peer or child!
+
+    '''
+    if not debug_mode():
+        yield None
+        return
+
+    task: Task = current_task()
+    async with trio.open_nursery() as n:
+        ctx: Context = await n.start(
+            partial(
+                request_root_stdio_lock,
+                actor_uid=subactor_uid,
+                task_uid=(task.name, id(task)),
+            )
+        )
+        yield ctx
+        ctx.cancel()