Log tbs from non-RAE `._invoke()`-RPC-task errors

`RemoteActorError`s show this by default in their `.__repr__()`, and we
obvi capture and embed the src traceback in an `Error` msg prior to
transit, but for logging it's also handy to see the tb of any set
`Context._remote_error` on console especially when trying to decipher
remote error details at their origin actor. Also improve the log message
description using `ctx.repr_state` and show any `ctx.outcome`.

tractor/_context.py

@@ -58,6 +58,7 @@ from typing import (
 import warnings
 # ------ - ------
 import trio
+from trio.lowlevel import Task
 # ------ - ------
 from ._exceptions import (
     ContextCancelled,
@@ -121,7 +122,7 @@ class Unresolved:
 @dataclass
 class Context:
     '''
-    An inter-actor, SC transitive, `trio.Task` communication context.
+    An inter-actor, SC transitive, `Task` communication context.
 
     NB: This class should **never be instatiated directly**, it is allocated
     by the runtime in 2 ways:
@@ -134,7 +135,7 @@ class Context:
 
     Allows maintaining task or protocol specific state between
     2 cancel-scope-linked, communicating and parallel executing
-    `trio.Task`s. Contexts are allocated on each side of any task
+    `Task`s. Contexts are allocated on each side of any task
     RPC-linked msg dialog, i.e. for every request to a remote
     actor from a `Portal`. On the "callee" side a context is
     always allocated inside `._rpc._invoke()`.
@@ -214,7 +215,7 @@ class Context:
     # which is exactly the primitive that allows for
     # cross-actor-task-supervision and thus SC.
     _scope: trio.CancelScope|None = None
-    _task: trio.lowlevel.Task|None = None
+    _task: Task|None = None
 
     # TODO: cs around result waiting so we can cancel any
     # permanently blocking `._rx_chan.receive()` call in
@@ -258,14 +259,14 @@ class Context:
     # a call to `.cancel()` which triggers `ContextCancelled`.
     _cancel_msg: str|dict|None = None
 
-    # NOTE: this state var used by the runtime to determine if the
+    # NOTE: this state-var is used by the runtime to determine if the
     # `pdbp` REPL is allowed to engage on contexts terminated via
     # a `ContextCancelled` due to a call to `.cancel()` triggering
     # "graceful closure" on either side:
     # - `._runtime._invoke()` will check this flag before engaging
     #   the crash handler REPL in such cases where the "callee"
     #   raises the cancellation,
-    # - `.devx._debug.lock_tty_for_child()` will set it to `False` if
+    # - `.devx._debug.lock_stdio_for_peer()` will set it to `False` if
     #   the global tty-lock has been configured to filter out some
     #   actors from being able to acquire the debugger lock.
     _enter_debugger_on_cancel: bool = True
@@ -861,7 +862,7 @@ class Context:
     ) -> None:
         '''
         Cancel this inter-actor IPC context by requestng the
-        remote side's cancel-scope-linked `trio.Task` by calling
+        remote side's cancel-scope-linked `Task` by calling
         `._scope.cancel()` and delivering an `ContextCancelled`
         ack msg in reponse.
 
@@ -1030,7 +1031,7 @@ class Context:
         # XXX NOTE XXX: `ContextCancelled`/`StreamOverrun` absorption
         # for "graceful cancellation" case:
         #
-        # Whenever a "side" of a context (a `trio.Task` running in
+        # Whenever a "side" of a context (a `Task` running in
         # an actor) **is** the side which requested ctx
         # cancellation (likekly via ``Context.cancel()``), we
         # **don't** want to re-raise any eventually received
@@ -1089,7 +1090,8 @@ class Context:
             else:
                 log.warning(
                     'Local error already set for ctx?\n'
-                    f'{self._local_error}\n'
+                    f'{self._local_error}\n\n'
+                    f'{self}'
                 )
 
             return remote_error
@@ -2117,8 +2119,9 @@ async def open_context_from_portal(
         # the `ContextCancelled` "self cancellation absorbed" case
         # handled in the block above ^^^ !!
         # await _debug.pause()
-        log.cancel(
+        # log.cancel(
-            'Context terminated due to\n\n'
+        log.exception(
+            f'{ctx.side}-side of `Context` terminated with '
             f'.outcome => {ctx.repr_outcome()}\n'
         )
 
@@ -2319,7 +2322,7 @@ async def open_context_from_portal(
                     # type_only=True,
                 )
                 log.cancel(
-                    f'Context terminated due to local scope error:\n\n'
+                    f'Context terminated due to local {ctx.side!r}-side error:\n\n'
                     f'{ctx.chan.uid} => {outcome_str}\n'
                 )
 
@@ -2385,15 +2388,25 @@ def mk_context(
 
 # TODO: use the new type-parameters to annotate this in 3.13?
 # -[ ] https://peps.python.org/pep-0718/#unknown-types
+# -[ ] allow for `pld_spec` input(s) ideally breaking down,
+# |_ `start: ParameterSpec`,
+# |_ `started: TypeAlias`,
+# |_ `yields: TypeAlias`,
+# |_ `return: TypeAlias`,
+# |_ `invalid_policy: str|Callable` ?
+# -[ ] prolly implement the `@acm` wrapper using
+#     a `contextlib.ContextDecorator`?
+#
 def context(
     func: Callable,
 ) -> Callable:
     '''
-    Mark an (async) function as an SC-supervised, inter-`Actor`,
+    Mark an async function as an SC-supervised, inter-`Actor`, RPC
-    child-`trio.Task`, IPC endpoint otherwise known more
+    scheduled child-side `Task`, IPC endpoint otherwise
-    colloquially as a (RPC) "context".
+    known more colloquially as a (RPC) "context".
 
-    Functions annotated the fundamental IPC endpoint type offered by `tractor`.
+    Functions annotated the fundamental IPC endpoint type offered by
+    `tractor`.
 
     '''
     # TODO: apply whatever solution ``mypy`` ends up picking for this:

tractor/_root.py

@@ -80,6 +80,7 @@ async def open_root_actor(
     # enables the multi-process debugger support
     debug_mode: bool = False,
     maybe_enable_greenback: bool = False,  # `.pause_from_sync()/breakpoint()` support
+    enable_stack_on_sig: bool = False,
 
     # internal logging
     loglevel: str|None = None,
@@ -119,7 +120,7 @@ async def open_root_actor(
         )
     ):
         os.environ['PYTHONBREAKPOINT'] = (
-            'tractor.devx._debug.pause_from_sync'
+            'tractor.devx._debug._sync_pause_from_builtin'
         )
         _state._runtime_vars['use_greenback'] = True
 
@@ -220,7 +221,11 @@ async def open_root_actor(
     assert _log
 
     # TODO: factor this into `.devx._stackscope`!!
-    if debug_mode:
+    if (
+        debug_mode
+        and
+        enable_stack_on_sig
+    ):
         try:
             logger.info('Enabling `stackscope` traces on SIGUSR1')
             from .devx import enable_stack_on_sig

tractor/_rpc.py

@@ -26,6 +26,7 @@ from contextlib import (
 from functools import partial
 import inspect
 from pprint import pformat
+import traceback
 from typing import (
     Any,
     Callable,
@@ -47,6 +48,7 @@ from ._context import (
 )
 from ._exceptions import (
     ContextCancelled,
+    RemoteActorError,
     ModuleNotExposed,
     MsgTypeError,
     TransportClosed,
@@ -198,7 +200,8 @@ async def _invoke_non_context(
                 raise ipc_err
             else:
                 log.exception(
-                    f'Failed to respond to runtime RPC request for\n\n'
+                    f'Failed to ack runtime RPC request\n\n'
+                    f'{func} x=> {ctx.chan}\n\n'
                     f'{ack}\n'
                 )
 
@@ -415,7 +418,6 @@ async def _errors_relayed_via_ipc(
 
 
 async def _invoke(
-
     actor: Actor,
     cid: str,
     chan: Channel,
@@ -691,10 +693,6 @@ async def _invoke(
                         boxed_type=trio.Cancelled,
                         canceller=canceller,
                     )
-                    # does this matter other then for
-                    # consistentcy/testing? |_ no user code should be
-                    # in this scope at this point..
-                    # ctx._local_error = ctxc
                     raise ctxc
 
         # XXX: do we ever trigger this block any more?
@@ -715,6 +713,11 @@ async def _invoke(
             # always set this (child) side's exception as the
             # local error on the context
             ctx._local_error: BaseException = scope_error
+            # ^-TODO-^ question,
+            # does this matter other then for
+            # consistentcy/testing?
+            # |_ no user code should be in this scope at this point
+            #    AND we already set this in the block below?
 
             # if a remote error was set then likely the
             # exception group was raised due to that, so
@@ -741,22 +744,32 @@ async def _invoke(
 
             logmeth: Callable = log.runtime
             merr: Exception|None = ctx.maybe_error
-            descr_str: str = 'with final result `{repr(ctx.outcome)}`'
+            message: str = 'IPC context terminated '
-            message: str = (
+            descr_str: str = (
-                f'IPC context terminated {descr_str}\n\n'
+                f'after having {ctx.repr_state!r}\n'
             )
             if merr:
-                descr_str: str = (
+
-                    f'with ctx having {ctx.repr_state!r}\n'
+                logmeth: Callable = log.error
-                    f'{ctx.repr_outcome()}\n'
-                )
                 if isinstance(merr, ContextCancelled):
                     logmeth: Callable = log.runtime
-                else:
-                    logmeth: Callable = log.error
-                    message += f'\n{merr!r}\n'
 
-            logmeth(message)
+                if not isinstance(merr, RemoteActorError):
+                    tb_str: str = ''.join(traceback.format_exception(merr))
+                    descr_str += (
+                        f'\n{merr!r}\n'  # needed?
+                        f'{tb_str}\n'
+                    )
+                else:
+                    descr_str += f'\n{merr!r}\n'
+            else:
+                descr_str += f'\nand final result {ctx.outcome!r}\n'
+
+            logmeth(
+                message
+                +
+                descr_str
+            )
 
 
 async def try_ship_error_to_remote(

tractor/log.py

@@ -57,8 +57,8 @@ DATE_FORMAT = '%b %d %H:%M:%S'
 CUSTOM_LEVELS: dict[str, int] = {
     'TRANSPORT': 5,
     'RUNTIME': 15,
-    'CANCEL': 16,
     'DEVX': 17,
+    'CANCEL': 18,
     'PDB': 500,
 }
 STD_PALETTE = {
@@ -111,7 +111,7 @@ class StackLevelAdapter(LoggerAdapter):
 
         '''
         return self.log(
-            level=16,
+            level=22,
             msg=msg,
             # stacklevel=4,
         )