Compare commits

...

8 Commits

Author SHA1 Message Date
Tyler Goodlet e4ec6b7b0c Even smarter `RemoteActorError.pformat()`-ing
Related to the prior patch, re the new `with_type_header: bool`:
- in the `with_type_header == True` use case make sure we keep the first
  `._message: str` line non-indented since it'll show just after the
  header-line's type path with ':'.
- when `False` drop the `)>` `repr()`-instance style as well so that we
  just get the ascii boxed traceback as though it's the error
  message-`str` not the `repr()` of the error obj.

Other,
- hide `pack_from_raise()` call frame since it'll show in debug mode
  crash handling..
- mk `MsgTypeError.from_decode()` explicitly accept and proxy an
  optional `ipc_msg` and change `msgdict` to also be optional, only
  reading out the `**extra_msgdata` when provided.
- expose a `_mk_msg_type_err(src_err_msg: Error|None = None,)` for
  callers who which to inject a `._ipc_msg: Msgtype` to the MTE.
  |_ add a note how we can't use it due to a causality-dilemma when pld
     validating `Started` on the send side..
2024-05-22 15:26:48 -04:00
Tyler Goodlet 9ce958cb4a Add debug check-n-wait inside `._spawn.soft_kill()`
And IFF the `await wait_func(proc)` is cancelled such that we avoid
clobbering some subactor that might be REPL-ing even though its parent
actor is in the midst of (gracefully) cancelling it.
2024-05-22 15:21:01 -04:00
Tyler Goodlet ce4d64ed2f Mk `MsgDec.spec_str` have a more compact ` 2024-05-22 15:18:45 -04:00
Tyler Goodlet c6f599b1be Call `.devx._debug.hide_runtime_frames()` by default
From both `open_root_actor()` and `._entry._trio_main()`.

Other `breakpoint()`-from-sync-func fixes:
- properly disable the default hook using `"0"` XD
- offer a `hide_tb: bool` from `open_root_actor()`.
- disable hiding the `._trio_main()` frame, bc pretty sure it doesn't
  help anyone (either way) when REPL-ing/tb-ing from a subactor..?
2024-05-22 15:16:29 -04:00
Tyler Goodlet 9eb74560ad Port `Actor._stream_handler()` to use `.has_outcome`, fix indent bug.. 2024-05-22 15:10:39 -04:00
Tyler Goodlet 702dfe47d5 Update debugger tests to expect new pformatting
Mostly the result of the `RemoteActorError.pformat()` and our
new `_pause/crash_msg: str`s which include the `trio.Task.__repr__()`
in the `log.pdb()` message.

Obvi use the `in_prompt_msg()` to accomplish where not used prior.

ToDo later:
-[ ] still some outstanding questions on how detailed inceptions
   should look, eg. in `test_multi_nested_subactors_error_through_nurseries()`
  |_maybe we should be more pedantic at checking `.src_uid` vs.
    `.relay_uid` fields?
-[ ] staged a placeholder test for verifying correct call-stack frame on
   crash handler REPL entry.
-[ ] also need a test to verify that you can't pause from an already paused actor task
   such as can happen if you try to step through runtime code that has
   a recurrent entry to `._debug.pause()`.
2024-05-22 15:01:31 -04:00
Tyler Goodlet d15e73557a Move runtime frame hiding into helper func
Call it `hide_runtime_frames()` and stick all the lines from the top of
the `._debug` mod in there along with a little `log.devx()` emission on
what gets hidden by default ;)

Other,
- fix ref-error where internal-error handler might trigger despite the
  debug `req_ctx` not yet having init-ed, such that we don't try to
  cancel or log about it when it never was fully created/initialize..
- fix assignment typo iniside `_set_trace()` for `task`.. lel
2024-05-22 14:56:54 -04:00
Tyler Goodlet 74d4b5280a Woops, make `log.devx()` level less `.error()` 2024-05-22 14:56:18 -04:00
9 changed files with 287 additions and 135 deletions

View File

@ -146,9 +146,10 @@ def in_prompt_msg(
log/REPL output for a given `pdb` interact point.
'''
__tracebackhide__: bool = False
for part in parts:
if part not in prompt:
if pause_on_false:
import pdbp
pdbp.set_trace()
@ -167,6 +168,7 @@ def assert_before(
**kwargs,
) -> None:
__tracebackhide__: bool = False
# as in before the prompt end
before: str = str(child.before.decode())
@ -219,7 +221,10 @@ def ctlc(
],
ids=lambda item: f'{item[0]} -> {item[1]}',
)
def test_root_actor_error(spawn, user_in_out):
def test_root_actor_error(
spawn,
user_in_out,
):
'''
Demonstrate crash handler entering pdb from basic error in root actor.
@ -465,8 +470,12 @@ def test_subactor_breakpoint(
child.expect(PROMPT)
before = str(child.before.decode())
assert "RemoteActorError: ('breakpoint_forever'" in before
assert 'bdb.BdbQuit' in before
assert in_prompt_msg(
before,
['RemoteActorError:',
"('breakpoint_forever'",
'bdb.BdbQuit',]
)
if ctlc:
do_ctlc(child)
@ -478,8 +487,12 @@ def test_subactor_breakpoint(
child.expect(pexpect.EOF)
before = str(child.before.decode())
assert "RemoteActorError: ('breakpoint_forever'" in before
assert 'bdb.BdbQuit' in before
assert in_prompt_msg(
before,
['RemoteActorError:',
"('breakpoint_forever'",
'bdb.BdbQuit',]
)
@has_nested_actors
@ -747,8 +760,9 @@ def test_multi_daemon_subactors(
# boxed error raised in root task
# "Attaching to pdb in crashed actor: ('root'",
_crash_msg,
"('root'",
"_exceptions.RemoteActorError: ('name_error'",
"('root'", # should attach in root
"_exceptions.RemoteActorError:", # with an embedded RAE for..
"('name_error'", # the src subactor which raised
]
)
@ -849,10 +863,11 @@ def test_multi_nested_subactors_error_through_nurseries(
# https://github.com/goodboy/tractor/issues/320
# ctlc: bool,
):
"""Verify deeply nested actors that error trigger debugger entries
'''
Verify deeply nested actors that error trigger debugger entries
at each actor nurserly (level) all the way up the tree.
"""
'''
# NOTE: previously, inside this script was a bug where if the
# parent errors before a 2-levels-lower actor has released the lock,
# the parent tries to cancel it but it's stuck in the debugger?
@ -872,22 +887,31 @@ def test_multi_nested_subactors_error_through_nurseries(
except EOF:
break
assert_before(child, [
# boxed source errors
assert_before(
child,
[ # boxed source errors
"NameError: name 'doggypants' is not defined",
"tractor._exceptions.RemoteActorError: ('name_error'",
"tractor._exceptions.RemoteActorError:",
"('name_error'",
"bdb.BdbQuit",
# first level subtrees
"tractor._exceptions.RemoteActorError: ('spawner0'",
# "tractor._exceptions.RemoteActorError: ('spawner0'",
"src_uid=('spawner0'",
# "tractor._exceptions.RemoteActorError: ('spawner1'",
# propagation of errors up through nested subtrees
"tractor._exceptions.RemoteActorError: ('spawn_until_0'",
"tractor._exceptions.RemoteActorError: ('spawn_until_1'",
"tractor._exceptions.RemoteActorError: ('spawn_until_2'",
])
# "tractor._exceptions.RemoteActorError: ('spawn_until_0'",
# "tractor._exceptions.RemoteActorError: ('spawn_until_1'",
# "tractor._exceptions.RemoteActorError: ('spawn_until_2'",
# ^-NOTE-^ old RAE repr, new one is below with a field
# showing the src actor's uid.
"src_uid=('spawn_until_0'",
"relay_uid=('spawn_until_1'",
"src_uid=('spawn_until_2'",
]
)
@pytest.mark.timeout(15)
@ -1021,13 +1045,16 @@ def test_different_debug_mode_per_actor(
# msg reported back from the debug mode actor is processed.
# assert "tractor._exceptions.RemoteActorError: ('debugged_boi'" in before
assert "tractor._exceptions.RemoteActorError: ('crash_boi'" in before
# the crash boi should not have made a debugger request but
# instead crashed completely
assert "tractor._exceptions.RemoteActorError: ('crash_boi'" in before
assert "RuntimeError" in before
assert_before(
child,
[
"tractor._exceptions.RemoteActorError:",
"src_uid=('crash_boi'",
"RuntimeError",
]
)
def test_pause_from_sync(
@ -1046,13 +1073,15 @@ def test_pause_from_sync(
assert_before(
child,
[
'`greenback` portal opened!',
# pre-prompt line
_pause_msg, "('root'",
_pause_msg,
"<Task '__main__.main'",
"('root'",
]
)
if ctlc:
do_ctlc(child)
child.sendline('c')
child.expect(PROMPT)
@ -1069,6 +1098,7 @@ def test_pause_from_sync(
if ctlc:
do_ctlc(child)
child.sendline('c')
child.expect(PROMPT)
assert_before(
@ -1078,6 +1108,7 @@ def test_pause_from_sync(
if ctlc:
do_ctlc(child)
child.sendline('c')
child.expect(PROMPT)
# non-main thread case
@ -1089,5 +1120,22 @@ def test_pause_from_sync(
if ctlc:
do_ctlc(child)
child.sendline('c')
child.expect(pexpect.EOF)
# TODO!
def test_correct_frames_below_hidden():
'''
Ensure that once a `tractor.pause()` enages, when the user
inputs a "next"/"n" command the actual next line steps
and that using a "step"/"s" into the next LOC, particuarly
`tractor` APIs, you can step down into that code.
'''
...
def test_cant_pause_from_paused_task():
...

View File

@ -33,6 +33,7 @@ from .log import (
get_logger,
)
from . import _state
from .devx import _debug
from .to_asyncio import run_as_asyncio_guest
from ._runtime import (
async_main,
@ -96,7 +97,6 @@ def _mp_main(
def _trio_main(
actor: Actor,
*,
parent_addr: tuple[str, int] | None = None,
@ -107,7 +107,9 @@ def _trio_main(
Entry point for a `trio_run_in_process` subactor.
'''
__tracebackhide__: bool = True
# __tracebackhide__: bool = True
_debug.hide_runtime_frames()
_state._current_actor = actor
trio_main = partial(
async_main,
@ -146,7 +148,6 @@ def _trio_main(
+
actor_info
)
finally:
log.info(
'Subactor terminated\n'

View File

@ -35,7 +35,6 @@ import trio
from msgspec import (
defstruct,
msgpack,
Raw,
structs,
ValidationError,
)
@ -44,11 +43,12 @@ from tractor._state import current_actor
from tractor.log import get_logger
from tractor.msg import (
Error,
PayloadMsg,
MsgType,
Stop,
types as msgtypes,
MsgCodec,
MsgDec,
Stop,
types as msgtypes,
)
from tractor.msg.pretty_struct import (
iter_fields,
@ -156,6 +156,7 @@ def pack_from_raise(
`Error`-msg using `pack_error()` to extract the tb info.
'''
__tracebackhide__: bool = True
try:
raise local_err
except type(local_err) as local_err:
@ -525,10 +526,26 @@ class RemoteActorError(Exception):
if not with_type_header:
body = '\n' + body
else:
body: str = textwrap.indent(
self._message,
first: str = ''
message: str = self._message
# split off the first line so it isn't indented
# the same like the "boxed content".
if not with_type_header:
lines: list[str] = message.splitlines()
first = lines[0]
message = ''.join(lines[1:])
body: str = (
first
+
textwrap.indent(
message,
prefix=' ',
) + '\n'
)
+
'\n'
)
if with_type_header:
tail: str = ')>'
@ -734,25 +751,38 @@ class MsgTypeError(
def from_decode(
cls,
message: str,
msgdict: dict,
ipc_msg: PayloadMsg|None = None,
msgdict: dict|None = None,
) -> MsgTypeError:
return cls(
message=message,
boxed_type=cls,
'''
Constuctor for easy creation from (presumably) catching
the backend interchange lib's underlying validation error
and passing context-specific meta-data to `_mk_msg_type_err()`
(which is normally the caller of this).
# NOTE: original "vanilla decode" of the msg-bytes
# is placed inside a value readable from
# `.msgdata['_msg_dict']`
_msg_dict=msgdict,
# expand and pack all RAE compat fields
# into the `._extra_msgdata` aux `dict`.
**{
'''
# if provided, expand and pack all RAE compat fields into the
# `._extra_msgdata` auxillary data `dict` internal to
# `RemoteActorError`.
extra_msgdata: dict = {}
if msgdict:
extra_msgdata: dict = {
k: v
for k, v in msgdict.items()
if k in _ipcmsg_keys
},
}
# NOTE: original "vanilla decode" of the msg-bytes
# is placed inside a value readable from
# `.msgdata['_msg_dict']`
extra_msgdata['_msg_dict'] = msgdict
return cls(
message=message,
boxed_type=cls,
ipc_msg=ipc_msg,
**extra_msgdata,
)
@ -1076,7 +1106,7 @@ _raise_from_no_key_in_msg = _raise_from_unexpected_msg
def _mk_msg_type_err(
msg: Any|bytes|Raw,
msg: Any|bytes|MsgType,
codec: MsgCodec|MsgDec,
message: str|None = None,
@ -1085,6 +1115,7 @@ def _mk_msg_type_err(
src_validation_error: ValidationError|None = None,
src_type_error: TypeError|None = None,
is_invalid_payload: bool = False,
src_err_msg: Error|None = None,
**mte_kwargs,
@ -1159,9 +1190,10 @@ def _mk_msg_type_err(
# only the payload being wrong?
# -[ ] maybe the better design is to break this construct
# logic into a separate explicit helper raiser-func?
msg_dict: dict = {}
msg_dict = None
else:
msg: bytes
# decode the msg-bytes using the std msgpack
# interchange-prot (i.e. without any
# `msgspec.Struct` handling) so that we can
@ -1206,6 +1238,14 @@ def _mk_msg_type_err(
msgtyperr = MsgTypeError.from_decode(
message=message,
msgdict=msg_dict,
# NOTE: for the send-side `.started()` pld-validate
# case we actually set the `._ipc_msg` AFTER we return
# from here inside `Context.started()` since we actually
# want to emulate the `Error` from the mte we build here
# Bo
# so by default in that case this is set to `None`
ipc_msg=src_err_msg,
)
msgtyperr.__cause__ = src_validation_error
return msgtyperr

View File

@ -91,12 +91,16 @@ async def open_root_actor(
# and that this call creates it.
ensure_registry: bool = False,
hide_tb: bool = True,
) -> Actor:
'''
Runtime init entry point for ``tractor``.
'''
__tracebackhide__ = True
__tracebackhide__: bool = hide_tb
_debug.hide_runtime_frames()
# TODO: stick this in a `@cm` defined in `devx._debug`?
#
# Override the global debugger hook to make it play nice with
@ -125,7 +129,7 @@ async def open_root_actor(
# usage by a clobbered TTY's stdstreams!
def block_bps(*args, **kwargs):
raise RuntimeError(
'Trying to use `breakpoint()` eh?\n'
'Trying to use `breakpoint()` eh?\n\n'
'Welp, `tractor` blocks `breakpoint()` built-in calls by default!\n'
'If you need to use it please install `greenback` and set '
'`debug_mode=True` when opening the runtime '
@ -133,7 +137,9 @@ async def open_root_actor(
)
sys.breakpointhook = block_bps
# os.environ['PYTHONBREAKPOINT'] = None
# lol ok,
# https://docs.python.org/3/library/sys.html#sys.breakpointhook
os.environ['PYTHONBREAKPOINT'] = "0"
# attempt to retreive ``trio``'s sigint handler and stash it
# on our debugger lock state.
@ -203,6 +209,7 @@ async def open_root_actor(
):
loglevel = 'PDB'
elif debug_mode:
raise RuntimeError(
"Debug mode is only supported for the `trio` backend!"

View File

@ -1140,7 +1140,6 @@ class Actor:
requester_type,
req_chan,
log_meth,
) = (
req_chan.uid,
'peer',
@ -1173,7 +1172,11 @@ class Actor:
# with the root actor in this tree
debug_req = _debug.DebugStatus
lock_req_ctx: Context = debug_req.req_ctx
if lock_req_ctx is not None:
if (
lock_req_ctx
and
lock_req_ctx.has_outcome
):
msg += (
'-> Cancelling active debugger request..\n'
f'|_{_debug.Lock.repr()}\n\n'

View File

@ -43,6 +43,7 @@ from tractor._state import (
is_main_process,
is_root_process,
debug_mode,
_runtime_vars,
)
from tractor.log import get_logger
from tractor._portal import Portal
@ -299,7 +300,6 @@ async def hard_kill(
async def soft_kill(
proc: ProcessType,
wait_func: Callable[
[ProcessType],
@ -329,6 +329,18 @@ async def soft_kill(
await wait_func(proc)
except trio.Cancelled:
with trio.CancelScope(shield=True):
await maybe_wait_for_debugger(
child_in_debug=_runtime_vars.get(
'_debug_mode', False
),
header_msg=(
'Delaying `soft_kill()` subproc reaper while debugger locked..\n'
),
# TODO: need a diff value then default?
# poll_steps=9999999,
)
# if cancelled during a soft wait, cancel the child
# actor before entering the hard reap sequence
# below. This means we try to do a graceful teardown

View File

@ -48,9 +48,11 @@ from typing import (
TYPE_CHECKING,
)
from types import (
FunctionType,
FrameType,
ModuleType,
TracebackType,
CodeType,
)
from msgspec import Struct
@ -90,6 +92,14 @@ if TYPE_CHECKING:
log = get_logger(__name__)
def hide_runtime_frames() -> dict[FunctionType, CodeType]:
'''
Hide call-stack frames for various std-lib and `trio`-API primitives
such that the tracebacks presented from our runtime are as minimized
as possible, particularly from inside a `PdbREPL`.
'''
# XXX HACKZONE XXX
# hide exit stack frames on nurseries and cancel-scopes!
# |_ so avoid seeing it when the `pdbp` REPL is first engaged from
@ -117,16 +127,37 @@ log = get_logger(__name__)
# -[ ] maybe dig into the core `pdb` issue why the extra frame is shown
# at all?
#
pdbp.hideframe(trio._core._run.NurseryManager.__aexit__)
pdbp.hideframe(trio._core._run.CancelScope.__exit__)
pdbp.hideframe(_GeneratorContextManager.__exit__)
pdbp.hideframe(_AsyncGeneratorContextManager.__aexit__)
pdbp.hideframe(trio.Event.wait)
__all__ = [
'breakpoint',
'post_mortem',
funcs: list[FunctionType] = [
trio._core._run.NurseryManager.__aexit__,
trio._core._run.CancelScope.__exit__,
_GeneratorContextManager.__exit__,
_AsyncGeneratorContextManager.__aexit__,
_AsyncGeneratorContextManager.__aenter__,
trio.Event.wait,
]
func_list_str: str = textwrap.indent(
"\n".join(f.__qualname__ for f in funcs),
prefix=' |_ ',
)
log.devx(
'Hiding the following runtime frames by default:\n'
f'{func_list_str}\n'
)
codes: dict[FunctionType, CodeType] = {}
for ref in funcs:
# stash a pre-modified version of each ref's code-obj
# so it can be reverted later if needed.
codes[ref] = ref.__code__
pdbp.hideframe(ref)
#
# pdbp.hideframe(trio._core._run.NurseryManager.__aexit__)
# pdbp.hideframe(trio._core._run.CancelScope.__exit__)
# pdbp.hideframe(_GeneratorContextManager.__exit__)
# pdbp.hideframe(_AsyncGeneratorContextManager.__aexit__)
# pdbp.hideframe(_AsyncGeneratorContextManager.__aenter__)
# pdbp.hideframe(trio.Event.wait)
return codes
class LockStatus(
@ -1032,15 +1063,24 @@ async def request_root_stdio_lock(
except (
BaseException,
):
log.exception(
'Failed during root TTY-lock dialog?\n'
f'{req_ctx}\n'
) as ctx_err:
message: str = (
'Failed during debug request dialog with root actor?\n\n'
)
if req_ctx:
message += (
f'{req_ctx}\n'
f'Cancelling IPC ctx!\n'
)
await req_ctx.cancel()
raise
else:
message += 'Failed during `Portal.open_context()` ?\n'
log.exception(message)
ctx_err.add_note(message)
raise ctx_err
except (
@ -1067,6 +1107,7 @@ async def request_root_stdio_lock(
# ctl-c out of the currently hanging task!
raise DebugRequestError(
'Failed to lock stdio from subactor IPC ctx!\n\n'
f'req_ctx: {DebugStatus.req_ctx}\n'
) from req_err
@ -1777,7 +1818,7 @@ def _set_trace(
):
__tracebackhide__: bool = hide_tb
actor: tractor.Actor = actor or current_actor()
task: task or current_task()
task: trio.Task = task or current_task()
# else:
# TODO: maybe print the actor supervion tree up to the

View File

@ -53,17 +53,14 @@ LOG_FORMAT = (
DATE_FORMAT = '%b %d %H:%M:%S'
LEVELS: dict[str, int] = {
# FYI, ERROR is 40
CUSTOM_LEVELS: dict[str, int] = {
'TRANSPORT': 5,
'RUNTIME': 15,
'CANCEL': 16,
'DEVX': 400,
'DEVX': 17,
'PDB': 500,
}
# _custom_levels: set[str] = {
# lvlname.lower for lvlname in LEVELS.keys()
# }
STD_PALETTE = {
'CRITICAL': 'red',
'ERROR': 'red',
@ -137,7 +134,7 @@ class StackLevelAdapter(LoggerAdapter):
"Developer experience" sub-sys statuses.
'''
return self.log(400, msg)
return self.log(17, msg)
def log(
self,
@ -154,8 +151,7 @@ class StackLevelAdapter(LoggerAdapter):
if self.isEnabledFor(level):
stacklevel: int = 3
if (
level in LEVELS.values()
# or level in _custom_levels
level in CUSTOM_LEVELS.values()
):
stacklevel: int = 4
@ -202,7 +198,8 @@ class StackLevelAdapter(LoggerAdapter):
)
# TODO IDEA:
# TODO IDEAs:
# -[ ] move to `.devx.pformat`?
# -[ ] do per task-name and actor-name color coding
# -[ ] unique color per task-id and actor-uuid
def pformat_task_uid(
@ -309,7 +306,7 @@ def get_logger(
logger = StackLevelAdapter(log, ActorContextInfo())
# additional levels
for name, val in LEVELS.items():
for name, val in CUSTOM_LEVELS.items():
logging.addLevelName(val, name)
# ensure customs levels exist as methods
@ -377,7 +374,7 @@ def at_least_level(
'''
if isinstance(level, str):
level: int = LEVELS[level.upper()]
level: int = CUSTOM_LEVELS[level.upper()]
if log.getEffectiveLevel() <= level:
return True

View File

@ -162,7 +162,10 @@ class MsgDec(Struct):
# TODO: would get moved into `FieldSpec.__str__()` right?
@property
def spec_str(self) -> str:
return pformat_msgspec(codec=self)
return pformat_msgspec(
codec=self,
join_char='|',
)
pld_spec_str = spec_str
@ -211,7 +214,7 @@ def mk_msgspec_table(
msgtypes = [msgspec]
msgt_table: dict[str, MsgType] = {
msgt: str(msgt)
msgt: str(msgt.__name__)
for msgt in msgtypes
}
if msg: