Even smarter `RemoteActorError.pformat()`-ing

Related to the prior patch, re the new `with_type_header: bool`:
- in the `with_type_header == True` use case make sure we keep the first
  `._message: str` line non-indented since it'll show just after the
  header-line's type path with ':'.
- when `False` drop the `)>` `repr()`-instance style as well so that we
  just get the ascii boxed traceback as though it's the error
  message-`str` not the `repr()` of the error obj.

Other,
- hide `pack_from_raise()` call frame since it'll show in debug mode
  crash handling..
- mk `MsgTypeError.from_decode()` explicitly accept and proxy an
  optional `ipc_msg` and change `msgdict` to also be optional, only
  reading out the `**extra_msgdata` when provided.
- expose a `_mk_msg_type_err(src_err_msg: Error|None = None,)` for
  callers who which to inject a `._ipc_msg: Msgtype` to the MTE.
  |_ add a note how we can't use it due to a causality-dilemma when pld
     validating `Started` on the send side..

tests/test_debugger.py

@@ -146,9 +146,10 @@ def in_prompt_msg(
     log/REPL output for a given `pdb` interact point.
 
     '''
+    __tracebackhide__: bool = False
+
     for part in parts:
         if part not in prompt:
-
             if pause_on_false:
                 import pdbp
                 pdbp.set_trace()
@@ -167,6 +168,7 @@ def assert_before(
     **kwargs,
 
 ) -> None:
+    __tracebackhide__: bool = False
 
     # as in before the prompt end
     before: str = str(child.before.decode())
@@ -219,7 +221,10 @@ def ctlc(
     ],
     ids=lambda item: f'{item[0]} -> {item[1]}',
 )
-def test_root_actor_error(spawn, user_in_out):
+def test_root_actor_error(
+    spawn,
+    user_in_out,
+):
     '''
     Demonstrate crash handler entering pdb from basic error in root actor.
 
@@ -465,8 +470,12 @@ def test_subactor_breakpoint(
     child.expect(PROMPT)
 
     before = str(child.before.decode())
-    assert "RemoteActorError: ('breakpoint_forever'" in before
-    assert 'bdb.BdbQuit' in before
+    assert in_prompt_msg(
+        before,
+        ['RemoteActorError:',
+         "('breakpoint_forever'",
+         'bdb.BdbQuit',]
+    )
 
     if ctlc:
         do_ctlc(child)
@@ -478,8 +487,12 @@ def test_subactor_breakpoint(
     child.expect(pexpect.EOF)
 
     before = str(child.before.decode())
-    assert "RemoteActorError: ('breakpoint_forever'" in before
-    assert 'bdb.BdbQuit' in before
+    assert in_prompt_msg(
+        before,
+        ['RemoteActorError:',
+         "('breakpoint_forever'",
+         'bdb.BdbQuit',]
+    )
 
 
 @has_nested_actors
@@ -747,8 +760,9 @@ def test_multi_daemon_subactors(
             # boxed error raised in root task
             # "Attaching to pdb in crashed actor: ('root'",
             _crash_msg,
-            "('root'",
-            "_exceptions.RemoteActorError: ('name_error'",
+            "('root'",  # should attach in root
+            "_exceptions.RemoteActorError:",  # with an embedded RAE for..
+            "('name_error'",  # the src subactor which raised
         ]
     )
 
@@ -849,10 +863,11 @@ def test_multi_nested_subactors_error_through_nurseries(
     # https://github.com/goodboy/tractor/issues/320
     # ctlc: bool,
 ):
-    """Verify deeply nested actors that error trigger debugger entries
+    '''
+    Verify deeply nested actors that error trigger debugger entries
     at each actor nurserly (level) all the way up the tree.
 
-    """
+    '''
     # NOTE: previously, inside this script was a bug where if the
     # parent errors before a 2-levels-lower actor has released the lock,
     # the parent tries to cancel it but it's stuck in the debugger?
@@ -872,22 +887,31 @@ def test_multi_nested_subactors_error_through_nurseries(
         except EOF:
             break
 
-    assert_before(child, [
-
-        # boxed source errors
+    assert_before(
+        child,
+        [ # boxed source errors
             "NameError: name 'doggypants' is not defined",
-        "tractor._exceptions.RemoteActorError: ('name_error'",
+            "tractor._exceptions.RemoteActorError:",
+            "('name_error'",
             "bdb.BdbQuit",
 
             # first level subtrees
-        "tractor._exceptions.RemoteActorError: ('spawner0'",
+            # "tractor._exceptions.RemoteActorError: ('spawner0'",
+            "src_uid=('spawner0'",
+
             # "tractor._exceptions.RemoteActorError: ('spawner1'",
 
             # propagation of errors up through nested subtrees
-        "tractor._exceptions.RemoteActorError: ('spawn_until_0'",
-        "tractor._exceptions.RemoteActorError: ('spawn_until_1'",
-        "tractor._exceptions.RemoteActorError: ('spawn_until_2'",
-    ])
+            # "tractor._exceptions.RemoteActorError: ('spawn_until_0'",
+            # "tractor._exceptions.RemoteActorError: ('spawn_until_1'",
+            # "tractor._exceptions.RemoteActorError: ('spawn_until_2'",
+            # ^-NOTE-^ old RAE repr, new one is below with a field
+            # showing the src actor's uid.
+            "src_uid=('spawn_until_0'",
+            "relay_uid=('spawn_until_1'",
+            "src_uid=('spawn_until_2'",
+        ]
+    )
 
 
 @pytest.mark.timeout(15)
@@ -1021,13 +1045,16 @@ def test_different_debug_mode_per_actor(
     # msg reported back from the debug mode actor is processed.
     # assert "tractor._exceptions.RemoteActorError: ('debugged_boi'" in before
 
-    assert "tractor._exceptions.RemoteActorError: ('crash_boi'" in before
-
     # the crash boi should not have made a debugger request but
     # instead crashed completely
-    assert "tractor._exceptions.RemoteActorError: ('crash_boi'" in before
-    assert "RuntimeError" in before
-
+    assert_before(
+        child,
+        [
+            "tractor._exceptions.RemoteActorError:",
+            "src_uid=('crash_boi'",
+            "RuntimeError",
+        ]
+    )
 
 
 def test_pause_from_sync(
@@ -1046,13 +1073,15 @@ def test_pause_from_sync(
     assert_before(
         child,
         [
-            '`greenback` portal opened!',
             # pre-prompt line
-            _pause_msg, "('root'",
+            _pause_msg,
+            "<Task '__main__.main'",
+            "('root'",
         ]
     )
     if ctlc:
         do_ctlc(child)
+
     child.sendline('c')
     child.expect(PROMPT)
 
@@ -1069,6 +1098,7 @@ def test_pause_from_sync(
 
     if ctlc:
         do_ctlc(child)
+
     child.sendline('c')
     child.expect(PROMPT)
     assert_before(
@@ -1078,6 +1108,7 @@ def test_pause_from_sync(
 
     if ctlc:
         do_ctlc(child)
+
     child.sendline('c')
     child.expect(PROMPT)
     # non-main thread case
@@ -1089,5 +1120,22 @@ def test_pause_from_sync(
 
     if ctlc:
         do_ctlc(child)
+
     child.sendline('c')
     child.expect(pexpect.EOF)
+
+
+# TODO!
+def test_correct_frames_below_hidden():
+    '''
+    Ensure that once a `tractor.pause()` enages, when the user
+    inputs a "next"/"n" command the actual next line steps
+    and that using a "step"/"s" into the next LOC, particuarly
+    `tractor` APIs, you can step down into that code.
+
+    '''
+    ...
+
+
+def test_cant_pause_from_paused_task():
+    ...

tractor/_entry.py

@@ -33,6 +33,7 @@ from .log import (
     get_logger,
 )
 from . import _state
+from .devx import _debug
 from .to_asyncio import run_as_asyncio_guest
 from ._runtime import (
     async_main,
@@ -96,7 +97,6 @@ def _mp_main(
 
 
 def _trio_main(
-
     actor: Actor,
     *,
     parent_addr: tuple[str, int] | None = None,
@@ -107,7 +107,9 @@ def _trio_main(
     Entry point for a `trio_run_in_process` subactor.
 
     '''
-    __tracebackhide__: bool = True
+    # __tracebackhide__: bool = True
+    _debug.hide_runtime_frames()
+
     _state._current_actor = actor
     trio_main = partial(
         async_main,
@@ -146,7 +148,6 @@ def _trio_main(
             +
             actor_info
         )
-
     finally:
         log.info(
             'Subactor terminated\n'

tractor/_exceptions.py

@@ -35,7 +35,6 @@ import trio
 from msgspec import (
     defstruct,
     msgpack,
-    Raw,
     structs,
     ValidationError,
 )
@@ -44,11 +43,12 @@ from tractor._state import current_actor
 from tractor.log import get_logger
 from tractor.msg import (
     Error,
+    PayloadMsg,
     MsgType,
-    Stop,
-    types as msgtypes,
     MsgCodec,
     MsgDec,
+    Stop,
+    types as msgtypes,
 )
 from tractor.msg.pretty_struct import (
     iter_fields,
@@ -156,6 +156,7 @@ def pack_from_raise(
     `Error`-msg using `pack_error()` to extract the tb info.
 
     '''
+    __tracebackhide__: bool = True
     try:
         raise local_err
     except type(local_err) as local_err:
@@ -525,10 +526,26 @@ class RemoteActorError(Exception):
             if not with_type_header:
                 body = '\n' + body
         else:
-            body: str = textwrap.indent(
-                self._message,
+            first: str = ''
+            message: str = self._message
+
+            # split off the first line so it isn't indented
+            # the same like the "boxed content".
+            if not with_type_header:
+                lines: list[str] = message.splitlines()
+                first = lines[0]
+                message = ''.join(lines[1:])
+
+            body: str = (
+                first
+                +
+                textwrap.indent(
+                    message,
                     prefix='  ',
-            ) + '\n'
+                )
+                +
+                '\n'
+            )
 
         if with_type_header:
             tail: str = ')>'
@@ -734,25 +751,38 @@ class MsgTypeError(
     def from_decode(
         cls,
         message: str,
-        msgdict: dict,
+
+        ipc_msg: PayloadMsg|None = None,
+        msgdict: dict|None = None,
 
     ) -> MsgTypeError:
-        return cls(
-            message=message,
-            boxed_type=cls,
+        '''
+        Constuctor for easy creation from (presumably) catching
+        the backend interchange lib's underlying validation error
+        and passing context-specific meta-data to `_mk_msg_type_err()`
+        (which is normally the caller of this).
 
-            # NOTE: original "vanilla decode" of the msg-bytes
-            # is placed inside a value readable from
-            # `.msgdata['_msg_dict']`
-            _msg_dict=msgdict,
-
-            # expand and pack all RAE compat fields
-            # into the `._extra_msgdata` aux `dict`.
-            **{
+        '''
+        # if provided, expand and pack all RAE compat fields into the
+        # `._extra_msgdata` auxillary data `dict` internal to
+        # `RemoteActorError`.
+        extra_msgdata: dict = {}
+        if msgdict:
+            extra_msgdata: dict = {
                 k: v
                 for k, v in msgdict.items()
                 if k in _ipcmsg_keys
-            },
+            }
+            # NOTE: original "vanilla decode" of the msg-bytes
+            # is placed inside a value readable from
+            # `.msgdata['_msg_dict']`
+            extra_msgdata['_msg_dict'] = msgdict
+
+        return cls(
+            message=message,
+            boxed_type=cls,
+            ipc_msg=ipc_msg,
+            **extra_msgdata,
         )
 
 
@@ -1076,7 +1106,7 @@ _raise_from_no_key_in_msg = _raise_from_unexpected_msg
 
 
 def _mk_msg_type_err(
-    msg: Any|bytes|Raw,
+    msg: Any|bytes|MsgType,
     codec: MsgCodec|MsgDec,
 
     message: str|None = None,
@@ -1085,6 +1115,7 @@ def _mk_msg_type_err(
     src_validation_error: ValidationError|None = None,
     src_type_error: TypeError|None = None,
     is_invalid_payload: bool = False,
+    src_err_msg: Error|None = None,
 
     **mte_kwargs,
 
@@ -1159,9 +1190,10 @@ def _mk_msg_type_err(
             # only the payload being wrong?
             # -[ ] maybe the better design is to break this construct
             #   logic into a separate explicit helper raiser-func?
-            msg_dict: dict = {}
+            msg_dict = None
 
         else:
+            msg: bytes
             # decode the msg-bytes using the std msgpack
             # interchange-prot (i.e. without any
             # `msgspec.Struct` handling) so that we can
@@ -1206,6 +1238,14 @@ def _mk_msg_type_err(
         msgtyperr = MsgTypeError.from_decode(
             message=message,
             msgdict=msg_dict,
+
+            # NOTE: for the send-side `.started()` pld-validate
+            # case we actually set the `._ipc_msg` AFTER we return
+            # from here inside `Context.started()` since we actually
+            # want to emulate the `Error` from the mte we build here
+            # Bo
+            # so by default in that case this is set to `None`
+            ipc_msg=src_err_msg,
         )
         msgtyperr.__cause__ = src_validation_error
         return msgtyperr

tractor/_root.py

@@ -91,12 +91,16 @@ async def open_root_actor(
     # and that this call creates it.
     ensure_registry: bool = False,
 
+    hide_tb: bool = True,
+
 ) -> Actor:
     '''
     Runtime init entry point for ``tractor``.
 
     '''
-    __tracebackhide__ = True
+    __tracebackhide__: bool = hide_tb
+    _debug.hide_runtime_frames()
+
     # TODO: stick this in a `@cm` defined in `devx._debug`?
     #
     # Override the global debugger hook to make it play nice with
@@ -125,7 +129,7 @@ async def open_root_actor(
         # usage by a clobbered TTY's stdstreams!
         def block_bps(*args, **kwargs):
             raise RuntimeError(
-                'Trying to use `breakpoint()` eh?\n'
+                'Trying to use `breakpoint()` eh?\n\n'
                 'Welp, `tractor` blocks `breakpoint()` built-in calls by default!\n'
                 'If you need to use it please install `greenback` and set '
                 '`debug_mode=True` when opening the runtime '
@@ -133,7 +137,9 @@ async def open_root_actor(
             )
 
         sys.breakpointhook = block_bps
-        # os.environ['PYTHONBREAKPOINT'] = None
+        # lol ok,
+        # https://docs.python.org/3/library/sys.html#sys.breakpointhook
+        os.environ['PYTHONBREAKPOINT'] = "0"
 
     # attempt to retreive ``trio``'s sigint handler and stash it
     # on our debugger lock state.
@@ -203,6 +209,7 @@ async def open_root_actor(
         ):
             loglevel = 'PDB'
 
+
     elif debug_mode:
         raise RuntimeError(
             "Debug mode is only supported for the `trio` backend!"

tractor/_runtime.py

@@ -1140,7 +1140,6 @@ class Actor:
             requester_type,
             req_chan,
             log_meth,
-
         ) = (
             req_chan.uid,
             'peer',
@@ -1173,7 +1172,11 @@ class Actor:
             # with the root actor in this tree
             debug_req = _debug.DebugStatus
             lock_req_ctx: Context = debug_req.req_ctx
-            if lock_req_ctx is not None:
+            if (
+                lock_req_ctx
+                and
+                lock_req_ctx.has_outcome
+            ):
                 msg += (
                     '-> Cancelling active debugger request..\n'
                     f'|_{_debug.Lock.repr()}\n\n'

tractor/_spawn.py

@@ -43,6 +43,7 @@ from tractor._state import (
     is_main_process,
     is_root_process,
     debug_mode,
+    _runtime_vars,
 )
 from tractor.log import get_logger
 from tractor._portal import Portal
@@ -299,7 +300,6 @@ async def hard_kill(
 
 
 async def soft_kill(
-
     proc: ProcessType,
     wait_func: Callable[
         [ProcessType],
@@ -329,6 +329,18 @@ async def soft_kill(
         await wait_func(proc)
 
     except trio.Cancelled:
+        with trio.CancelScope(shield=True):
+            await maybe_wait_for_debugger(
+                child_in_debug=_runtime_vars.get(
+                    '_debug_mode', False
+                ),
+                header_msg=(
+                    'Delaying `soft_kill()` subproc reaper while debugger locked..\n'
+                ),
+                # TODO: need a diff value then default?
+                # poll_steps=9999999,
+            )
+
         # if cancelled during a soft wait, cancel the child
         # actor before entering the hard reap sequence
         # below. This means we try to do a graceful teardown

tractor/devx/_debug.py

@@ -48,9 +48,11 @@ from typing import (
     TYPE_CHECKING,
 )
 from types import (
+    FunctionType,
     FrameType,
     ModuleType,
     TracebackType,
+    CodeType,
 )
 
 from msgspec import Struct
@@ -90,6 +92,14 @@ if TYPE_CHECKING:
 
 log = get_logger(__name__)
 
+
+def hide_runtime_frames() -> dict[FunctionType, CodeType]:
+    '''
+    Hide call-stack frames for various std-lib and `trio`-API primitives
+    such that the tracebacks presented from our runtime are as minimized
+    as possible, particularly from inside a `PdbREPL`.
+
+    '''
     # XXX HACKZONE XXX
     #  hide exit stack frames on nurseries and cancel-scopes!
     # |_ so avoid seeing it when the `pdbp` REPL is first engaged from
@@ -117,16 +127,37 @@ log = get_logger(__name__)
     # -[ ] maybe dig into the core `pdb` issue why the extra frame is shown
     #      at all?
     #
-pdbp.hideframe(trio._core._run.NurseryManager.__aexit__)
-pdbp.hideframe(trio._core._run.CancelScope.__exit__)
-pdbp.hideframe(_GeneratorContextManager.__exit__)
-pdbp.hideframe(_AsyncGeneratorContextManager.__aexit__)
-pdbp.hideframe(trio.Event.wait)
-
-__all__ = [
-    'breakpoint',
-    'post_mortem',
+    funcs: list[FunctionType] = [
+        trio._core._run.NurseryManager.__aexit__,
+        trio._core._run.CancelScope.__exit__,
+         _GeneratorContextManager.__exit__,
+         _AsyncGeneratorContextManager.__aexit__,
+         _AsyncGeneratorContextManager.__aenter__,
+         trio.Event.wait,
     ]
+    func_list_str: str = textwrap.indent(
+        "\n".join(f.__qualname__ for f in funcs),
+        prefix=' |_ ',
+    )
+    log.devx(
+        'Hiding the following runtime frames by default:\n'
+        f'{func_list_str}\n'
+    )
+
+    codes: dict[FunctionType, CodeType] = {}
+    for ref in funcs:
+        # stash a pre-modified version of each ref's code-obj
+        # so it can be reverted later if needed.
+        codes[ref] = ref.__code__
+        pdbp.hideframe(ref)
+    #
+    # pdbp.hideframe(trio._core._run.NurseryManager.__aexit__)
+    # pdbp.hideframe(trio._core._run.CancelScope.__exit__)
+    # pdbp.hideframe(_GeneratorContextManager.__exit__)
+    # pdbp.hideframe(_AsyncGeneratorContextManager.__aexit__)
+    # pdbp.hideframe(_AsyncGeneratorContextManager.__aenter__)
+    # pdbp.hideframe(trio.Event.wait)
+    return codes
 
 
 class LockStatus(
@@ -1032,15 +1063,24 @@ async def request_root_stdio_lock(
 
             except (
                 BaseException,
-            ):
-                log.exception(
-                    'Failed during root TTY-lock dialog?\n'
-                    f'{req_ctx}\n'
+            ) as ctx_err:
+                message: str = (
+                    'Failed during debug request dialog with root actor?\n\n'
+                )
 
+                if req_ctx:
+                    message += (
+                        f'{req_ctx}\n'
                         f'Cancelling IPC ctx!\n'
                     )
                     await req_ctx.cancel()
-                raise
+
+                else:
+                    message += 'Failed during `Portal.open_context()` ?\n'
+
+                log.exception(message)
+                ctx_err.add_note(message)
+                raise ctx_err
 
 
     except (
@@ -1067,6 +1107,7 @@ async def request_root_stdio_lock(
         #   ctl-c out of the currently hanging task! 
         raise DebugRequestError(
             'Failed to lock stdio from subactor IPC ctx!\n\n'
+
             f'req_ctx: {DebugStatus.req_ctx}\n'
         ) from req_err
 
@@ -1777,7 +1818,7 @@ def _set_trace(
 ):
     __tracebackhide__: bool = hide_tb
     actor: tractor.Actor = actor or current_actor()
-    task: task or current_task()
+    task: trio.Task = task or current_task()
 
     # else:
     # TODO: maybe print the actor supervion tree up to the

tractor/log.py

@@ -53,17 +53,14 @@ LOG_FORMAT = (
 
 DATE_FORMAT = '%b %d %H:%M:%S'
 
-LEVELS: dict[str, int] = {
+# FYI, ERROR is 40
+CUSTOM_LEVELS: dict[str, int] = {
     'TRANSPORT': 5,
     'RUNTIME': 15,
     'CANCEL': 16,
-    'DEVX': 400,
+    'DEVX': 17,
     'PDB': 500,
 }
-# _custom_levels: set[str] = {
-#     lvlname.lower for lvlname in LEVELS.keys()
-# }
-
 STD_PALETTE = {
     'CRITICAL': 'red',
     'ERROR': 'red',
@@ -137,7 +134,7 @@ class StackLevelAdapter(LoggerAdapter):
         "Developer experience" sub-sys statuses.
 
         '''
-        return self.log(400, msg)
+        return self.log(17, msg)
 
     def log(
         self,
@@ -154,8 +151,7 @@ class StackLevelAdapter(LoggerAdapter):
         if self.isEnabledFor(level):
             stacklevel: int = 3
             if (
-                level in LEVELS.values()
-                # or level in _custom_levels
+                level in CUSTOM_LEVELS.values()
             ):
                 stacklevel: int = 4
 
@@ -202,7 +198,8 @@ class StackLevelAdapter(LoggerAdapter):
         )
 
 
-# TODO IDEA:
+# TODO IDEAs:
+# -[ ] move to `.devx.pformat`?
 # -[ ] do per task-name and actor-name color coding
 # -[ ] unique color per task-id and actor-uuid
 def pformat_task_uid(
@@ -309,7 +306,7 @@ def get_logger(
     logger = StackLevelAdapter(log, ActorContextInfo())
 
     # additional levels
-    for name, val in LEVELS.items():
+    for name, val in CUSTOM_LEVELS.items():
         logging.addLevelName(val, name)
 
         # ensure customs levels exist as methods
@@ -377,7 +374,7 @@ def at_least_level(
 
     '''
     if isinstance(level, str):
-        level: int = LEVELS[level.upper()]
+        level: int = CUSTOM_LEVELS[level.upper()]
 
     if log.getEffectiveLevel() <= level:
         return True

tractor/msg/_codec.py

@@ -162,7 +162,10 @@ class MsgDec(Struct):
     # TODO: would get moved into `FieldSpec.__str__()` right?
     @property
     def spec_str(self) -> str:
-        return pformat_msgspec(codec=self)
+        return pformat_msgspec(
+            codec=self,
+            join_char='|',
+        )
 
     pld_spec_str = spec_str
 
@@ -211,7 +214,7 @@ def mk_msgspec_table(
         msgtypes = [msgspec]
 
     msgt_table: dict[str, MsgType] = {
-        msgt: str(msgt)
+        msgt: str(msgt.__name__)
         for msgt in msgtypes
     }
     if msg: