Reorg `.devx.debug` into sub-mods!
Which cleans out the pkg-mod to just the expected exports with (its longstanding todo comment list) and thus a separation-of-concerns and smaller mod-file sizes via the following new sub-mods: - `._trace` for the `.pause()`/`breakpoint()`/`pdb.set_trace()`-style APIs including all sync-caller variants. - `._post_mortem` to contain our async `.post_mortem()` and all other public crash handling APIs for use from sync callers. - `._sync` for the high-level syncing helper-routines used throughout the runtime to avoid multi-proc TTY use collisions. And also, - remove `hide_runtime_frames()` since moved to `.devx._frame_stack`.repl_fixture
Tyler Goodlet
parent
69267ae656
commit
f1e9926b79
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,411 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Affero General Public License
|
||||
# as published by the Free Software Foundation, either version 3 of
|
||||
# the License, or (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public
|
||||
# License along with this program. If not, see
|
||||
# <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Post-mortem debugging APIs and surrounding machinery for both
|
||||
sync and async contexts.
|
||||
|
||||
Generally we maintain the same semantics a `pdb.post.mortem()` but
|
||||
with actor-tree-wide sync/cooperation around any (sub)actor's use of
|
||||
the root's TTY.
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
import bdb
|
||||
from contextlib import (
|
||||
AbstractContextManager,
|
||||
contextmanager as cm,
|
||||
nullcontext,
|
||||
)
|
||||
from functools import (
|
||||
partial,
|
||||
)
|
||||
import inspect
|
||||
import sys
|
||||
import traceback
|
||||
from typing import (
|
||||
Callable,
|
||||
Sequence,
|
||||
Type,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
from types import (
|
||||
TracebackType,
|
||||
FrameType,
|
||||
)
|
||||
|
||||
from msgspec import Struct
|
||||
import trio
|
||||
from tractor._exceptions import (
|
||||
NoRuntime,
|
||||
)
|
||||
from tractor import _state
|
||||
from tractor._state import (
|
||||
current_actor,
|
||||
debug_mode,
|
||||
)
|
||||
from tractor.log import get_logger
|
||||
from tractor._exceptions import (
|
||||
is_multi_cancelled,
|
||||
)
|
||||
from ._trace import (
|
||||
_pause,
|
||||
_maybe_open_repl_fixture,
|
||||
)
|
||||
from ._tty_lock import (
|
||||
DebugStatus,
|
||||
)
|
||||
from ._repl import (
|
||||
PdbREPL,
|
||||
mk_pdb,
|
||||
TractorConfig as TractorConfig,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from trio.lowlevel import Task
|
||||
from tractor._runtime import (
|
||||
Actor,
|
||||
)
|
||||
|
||||
_crash_msg: str = (
|
||||
'Opening a pdb REPL in crashed actor'
|
||||
)
|
||||
|
||||
log = get_logger(__package__)
|
||||
|
||||
|
||||
class BoxedMaybeException(Struct):
|
||||
'''
|
||||
Box a maybe-exception for post-crash introspection usage
|
||||
from the body of a `open_crash_handler()` scope.
|
||||
|
||||
'''
|
||||
value: BaseException|None = None
|
||||
|
||||
# handler can suppress crashes dynamically
|
||||
raise_on_exit: bool|Sequence[Type[BaseException]] = True
|
||||
|
||||
def pformat(self) -> str:
|
||||
'''
|
||||
Repr the boxed `.value` error in more-than-string
|
||||
repr form.
|
||||
|
||||
'''
|
||||
if not self.value:
|
||||
return f'<{type(self).__name__}( .value=None )>\n'
|
||||
|
||||
return (
|
||||
f'<{type(self.value).__name__}(\n'
|
||||
f' |_.value = {self.value}\n'
|
||||
f')>\n'
|
||||
)
|
||||
|
||||
__repr__ = pformat
|
||||
|
||||
|
||||
def _post_mortem(
|
||||
repl: PdbREPL, # normally passed by `_pause()`
|
||||
|
||||
# XXX all `partial`-ed in by `post_mortem()` below!
|
||||
tb: TracebackType,
|
||||
api_frame: FrameType,
|
||||
|
||||
shield: bool = False,
|
||||
hide_tb: bool = True,
|
||||
|
||||
# maybe pre/post REPL entry
|
||||
repl_fixture: (
|
||||
AbstractContextManager[bool]
|
||||
|None
|
||||
) = None,
|
||||
|
||||
boxed_maybe_exc: BoxedMaybeException|None = None,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Enter the ``pdbpp`` port mortem entrypoint using our custom
|
||||
debugger instance.
|
||||
|
||||
'''
|
||||
__tracebackhide__: bool = hide_tb
|
||||
|
||||
with _maybe_open_repl_fixture(
|
||||
repl_fixture=repl_fixture,
|
||||
boxed_maybe_exc=boxed_maybe_exc,
|
||||
) as enter_repl:
|
||||
if not enter_repl:
|
||||
return
|
||||
|
||||
try:
|
||||
actor: Actor = current_actor()
|
||||
actor_repr: str = str(actor.uid)
|
||||
# ^TODO, instead a nice runtime-info + maddr + uid?
|
||||
# -[ ] impl a `Actor.__repr()__`??
|
||||
# |_ <task>:<thread> @ <actor>
|
||||
|
||||
except NoRuntime:
|
||||
actor_repr: str = '<no-actor-runtime?>'
|
||||
|
||||
try:
|
||||
task_repr: Task = trio.lowlevel.current_task()
|
||||
except RuntimeError:
|
||||
task_repr: str = '<unknown-Task>'
|
||||
|
||||
# TODO: print the actor supervion tree up to the root
|
||||
# here! Bo
|
||||
log.pdb(
|
||||
f'{_crash_msg}\n'
|
||||
f'x>(\n'
|
||||
f' |_ {task_repr} @ {actor_repr}\n'
|
||||
|
||||
)
|
||||
|
||||
# XXX NOTE(s) on `pdbp.xpm()` version..
|
||||
#
|
||||
# - seems to lose the up-stack tb-info?
|
||||
# - currently we're (only) replacing this from `pdbp.xpm()`
|
||||
# to add the `end=''` to the print XD
|
||||
#
|
||||
print(traceback.format_exc(), end='')
|
||||
caller_frame: FrameType = api_frame.f_back
|
||||
|
||||
# NOTE, see the impl details of these in the lib to
|
||||
# understand usage:
|
||||
# - `pdbp.post_mortem()`
|
||||
# - `pdbp.xps()`
|
||||
# - `bdb.interaction()`
|
||||
repl.reset()
|
||||
repl.interaction(
|
||||
frame=caller_frame,
|
||||
# frame=None,
|
||||
traceback=tb,
|
||||
)
|
||||
|
||||
# XXX NOTE XXX: this is abs required to avoid hangs!
|
||||
#
|
||||
# Since we presume the post-mortem was enaged to
|
||||
# a task-ending error, we MUST release the local REPL request
|
||||
# so that not other local task nor the root remains blocked!
|
||||
DebugStatus.release()
|
||||
|
||||
|
||||
async def post_mortem(
|
||||
*,
|
||||
tb: TracebackType|None = None,
|
||||
api_frame: FrameType|None = None,
|
||||
hide_tb: bool = False,
|
||||
|
||||
# TODO: support shield here just like in `pause()`?
|
||||
# shield: bool = False,
|
||||
|
||||
**_pause_kwargs,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Our builtin async equivalient of `pdb.post_mortem()` which can be
|
||||
used inside exception handlers.
|
||||
|
||||
It's also used for the crash handler when `debug_mode == True` ;)
|
||||
|
||||
'''
|
||||
__tracebackhide__: bool = hide_tb
|
||||
|
||||
tb: TracebackType = tb or sys.exc_info()[2]
|
||||
|
||||
# TODO: do upward stack scan for highest @api_frame and
|
||||
# use its parent frame as the expected user-app code
|
||||
# interact point.
|
||||
api_frame: FrameType = api_frame or inspect.currentframe()
|
||||
|
||||
# TODO, move to submod `._pausing` or ._api? _trace
|
||||
await _pause(
|
||||
debug_func=partial(
|
||||
_post_mortem,
|
||||
api_frame=api_frame,
|
||||
tb=tb,
|
||||
),
|
||||
hide_tb=hide_tb,
|
||||
**_pause_kwargs
|
||||
)
|
||||
|
||||
|
||||
async def _maybe_enter_pm(
|
||||
err: BaseException,
|
||||
*,
|
||||
tb: TracebackType|None = None,
|
||||
api_frame: FrameType|None = None,
|
||||
hide_tb: bool = False,
|
||||
|
||||
# only enter debugger REPL when returns `True`
|
||||
debug_filter: Callable[
|
||||
[BaseException|BaseExceptionGroup],
|
||||
bool,
|
||||
] = lambda err: not is_multi_cancelled(err),
|
||||
**_pause_kws,
|
||||
|
||||
):
|
||||
if (
|
||||
debug_mode()
|
||||
|
||||
# NOTE: don't enter debug mode recursively after quitting pdb
|
||||
# Iow, don't re-enter the repl if the `quit` command was issued
|
||||
# by the user.
|
||||
and not isinstance(err, bdb.BdbQuit)
|
||||
|
||||
# XXX: if the error is the likely result of runtime-wide
|
||||
# cancellation, we don't want to enter the debugger since
|
||||
# there's races between when the parent actor has killed all
|
||||
# comms and when the child tries to contact said parent to
|
||||
# acquire the tty lock.
|
||||
|
||||
# Really we just want to mostly avoid catching KBIs here so there
|
||||
# might be a simpler check we can do?
|
||||
and
|
||||
debug_filter(err)
|
||||
):
|
||||
api_frame: FrameType = api_frame or inspect.currentframe()
|
||||
tb: TracebackType = tb or sys.exc_info()[2]
|
||||
await post_mortem(
|
||||
api_frame=api_frame,
|
||||
tb=tb,
|
||||
**_pause_kws,
|
||||
)
|
||||
return True
|
||||
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
# TODO: better naming and what additionals?
|
||||
# - [ ] optional runtime plugging?
|
||||
# - [ ] detection for sync vs. async code?
|
||||
# - [ ] specialized REPL entry when in distributed mode?
|
||||
# -[x] hide tb by def
|
||||
# - [x] allow ignoring kbi Bo
|
||||
@cm
|
||||
def open_crash_handler(
|
||||
catch: set[BaseException] = {
|
||||
BaseException,
|
||||
},
|
||||
ignore: set[BaseException] = {
|
||||
KeyboardInterrupt,
|
||||
trio.Cancelled,
|
||||
},
|
||||
hide_tb: bool = True,
|
||||
|
||||
repl_fixture: (
|
||||
AbstractContextManager[bool] # pre/post REPL entry
|
||||
|None
|
||||
) = None,
|
||||
raise_on_exit: bool|Sequence[Type[BaseException]] = True,
|
||||
):
|
||||
'''
|
||||
Generic "post mortem" crash handler using `pdbp` REPL debugger.
|
||||
|
||||
We expose this as a CLI framework addon to both `click` and
|
||||
`typer` users so they can quickly wrap cmd endpoints which get
|
||||
automatically wrapped to use the runtime's `debug_mode: bool`
|
||||
AND `pdbp.pm()` around any code that is PRE-runtime entry
|
||||
- any sync code which runs BEFORE the main call to
|
||||
`trio.run()`.
|
||||
|
||||
'''
|
||||
__tracebackhide__: bool = hide_tb
|
||||
|
||||
# TODO, yield a `outcome.Error`-like boxed type?
|
||||
# -[~] use `outcome.Value/Error` X-> frozen!
|
||||
# -[x] write our own..?
|
||||
# -[ ] consider just wtv is used by `pytest.raises()`?
|
||||
#
|
||||
boxed_maybe_exc = BoxedMaybeException(
|
||||
raise_on_exit=raise_on_exit,
|
||||
)
|
||||
err: BaseException
|
||||
try:
|
||||
yield boxed_maybe_exc
|
||||
except tuple(catch) as err:
|
||||
boxed_maybe_exc.value = err
|
||||
if (
|
||||
type(err) not in ignore
|
||||
and
|
||||
not is_multi_cancelled(
|
||||
err,
|
||||
ignore_nested=ignore
|
||||
)
|
||||
):
|
||||
try:
|
||||
# use our re-impl-ed version of `pdbp.xpm()`
|
||||
_post_mortem(
|
||||
repl=mk_pdb(),
|
||||
tb=sys.exc_info()[2],
|
||||
api_frame=inspect.currentframe().f_back,
|
||||
hide_tb=hide_tb,
|
||||
|
||||
repl_fixture=repl_fixture,
|
||||
boxed_maybe_exc=boxed_maybe_exc,
|
||||
)
|
||||
except bdb.BdbQuit:
|
||||
__tracebackhide__: bool = False
|
||||
raise err
|
||||
|
||||
if (
|
||||
raise_on_exit is True
|
||||
or (
|
||||
raise_on_exit is not False
|
||||
and (
|
||||
set(raise_on_exit)
|
||||
and
|
||||
type(err) in raise_on_exit
|
||||
)
|
||||
)
|
||||
and
|
||||
boxed_maybe_exc.raise_on_exit == raise_on_exit
|
||||
):
|
||||
raise err
|
||||
|
||||
|
||||
@cm
|
||||
def maybe_open_crash_handler(
|
||||
pdb: bool|None = None,
|
||||
hide_tb: bool = True,
|
||||
|
||||
**kwargs,
|
||||
):
|
||||
'''
|
||||
Same as `open_crash_handler()` but with bool input flag
|
||||
to allow conditional handling.
|
||||
|
||||
Normally this is used with CLI endpoints such that if the --pdb
|
||||
flag is passed the pdb REPL is engaed on any crashes B)
|
||||
|
||||
'''
|
||||
__tracebackhide__: bool = hide_tb
|
||||
|
||||
if pdb is None:
|
||||
pdb: bool = _state.is_debug_mode()
|
||||
|
||||
rtctx = nullcontext(
|
||||
enter_result=BoxedMaybeException()
|
||||
)
|
||||
if pdb:
|
||||
rtctx = open_crash_handler(
|
||||
hide_tb=hide_tb,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
with rtctx as boxed_maybe_exc:
|
||||
yield boxed_maybe_exc
|
||||
|
|
@ -0,0 +1,220 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Affero General Public License
|
||||
# as published by the Free Software Foundation, either version 3 of
|
||||
# the License, or (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public
|
||||
# License along with this program. If not, see
|
||||
# <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Debugger synchronization APIs to ensure orderly access and
|
||||
non-TTY-clobbering graceful teardown.
|
||||
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from functools import (
|
||||
partial,
|
||||
)
|
||||
from typing import (
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
)
|
||||
|
||||
from tractor.log import get_logger
|
||||
import trio
|
||||
from trio.lowlevel import (
|
||||
current_task,
|
||||
Task,
|
||||
)
|
||||
from tractor._context import Context
|
||||
from tractor._state import (
|
||||
current_actor,
|
||||
debug_mode,
|
||||
is_root_process,
|
||||
)
|
||||
from ._repl import (
|
||||
TractorConfig as TractorConfig,
|
||||
)
|
||||
from ._tty_lock import (
|
||||
Lock,
|
||||
request_root_stdio_lock,
|
||||
any_connected_locker_child,
|
||||
)
|
||||
from ._sigint import (
|
||||
sigint_shield as sigint_shield,
|
||||
_ctlc_ignore_header as _ctlc_ignore_header
|
||||
)
|
||||
|
||||
log = get_logger(__package__)
|
||||
|
||||
|
||||
async def maybe_wait_for_debugger(
|
||||
poll_steps: int = 2,
|
||||
poll_delay: float = 0.1,
|
||||
child_in_debug: bool = False,
|
||||
|
||||
header_msg: str = '',
|
||||
_ll: str = 'devx',
|
||||
|
||||
) -> bool: # was locked and we polled?
|
||||
|
||||
if (
|
||||
not debug_mode()
|
||||
and
|
||||
not child_in_debug
|
||||
):
|
||||
return False
|
||||
|
||||
logmeth: Callable = getattr(log, _ll)
|
||||
|
||||
msg: str = header_msg
|
||||
if (
|
||||
is_root_process()
|
||||
):
|
||||
# If we error in the root but the debugger is
|
||||
# engaged we don't want to prematurely kill (and
|
||||
# thus clobber access to) the local tty since it
|
||||
# will make the pdb repl unusable.
|
||||
# Instead try to wait for pdb to be released before
|
||||
# tearing down.
|
||||
ctx_in_debug: Context|None = Lock.ctx_in_debug
|
||||
in_debug: tuple[str, str]|None = (
|
||||
ctx_in_debug.chan.uid
|
||||
if ctx_in_debug
|
||||
else None
|
||||
)
|
||||
if in_debug == current_actor().uid:
|
||||
log.debug(
|
||||
msg
|
||||
+
|
||||
'Root already owns the TTY LOCK'
|
||||
)
|
||||
return True
|
||||
|
||||
elif in_debug:
|
||||
msg += (
|
||||
f'Debug `Lock` in use by subactor\n|\n|_{in_debug}\n'
|
||||
)
|
||||
# TODO: could this make things more deterministic?
|
||||
# wait to see if a sub-actor task will be
|
||||
# scheduled and grab the tty lock on the next
|
||||
# tick?
|
||||
# XXX => but it doesn't seem to work..
|
||||
# await trio.testing.wait_all_tasks_blocked(cushion=0)
|
||||
else:
|
||||
logmeth(
|
||||
msg
|
||||
+
|
||||
'Root immediately acquired debug TTY LOCK'
|
||||
)
|
||||
return False
|
||||
|
||||
for istep in range(poll_steps):
|
||||
if (
|
||||
Lock.req_handler_finished is not None
|
||||
and not Lock.req_handler_finished.is_set()
|
||||
and in_debug is not None
|
||||
):
|
||||
# caller_frame_info: str = pformat_caller_frame()
|
||||
logmeth(
|
||||
msg
|
||||
+
|
||||
'\n^^ Root is waiting on tty lock release.. ^^\n'
|
||||
# f'{caller_frame_info}\n'
|
||||
)
|
||||
|
||||
if not any_connected_locker_child():
|
||||
Lock.get_locking_task_cs().cancel()
|
||||
|
||||
with trio.CancelScope(shield=True):
|
||||
await Lock.req_handler_finished.wait()
|
||||
|
||||
log.devx(
|
||||
f'Subactor released debug lock\n'
|
||||
f'|_{in_debug}\n'
|
||||
)
|
||||
break
|
||||
|
||||
# is no subactor locking debugger currently?
|
||||
if (
|
||||
in_debug is None
|
||||
and (
|
||||
Lock.req_handler_finished is None
|
||||
or Lock.req_handler_finished.is_set()
|
||||
)
|
||||
):
|
||||
logmeth(
|
||||
msg
|
||||
+
|
||||
'Root acquired tty lock!'
|
||||
)
|
||||
break
|
||||
|
||||
else:
|
||||
logmeth(
|
||||
'Root polling for debug:\n'
|
||||
f'poll step: {istep}\n'
|
||||
f'poll delya: {poll_delay}\n\n'
|
||||
f'{Lock.repr()}\n'
|
||||
)
|
||||
with trio.CancelScope(shield=True):
|
||||
await trio.sleep(poll_delay)
|
||||
continue
|
||||
|
||||
return True
|
||||
|
||||
# else:
|
||||
# # TODO: non-root call for #320?
|
||||
# this_uid: tuple[str, str] = current_actor().uid
|
||||
# async with acquire_debug_lock(
|
||||
# subactor_uid=this_uid,
|
||||
# ):
|
||||
# pass
|
||||
return False
|
||||
|
||||
|
||||
@acm
|
||||
async def acquire_debug_lock(
|
||||
subactor_uid: tuple[str, str],
|
||||
) -> AsyncGenerator[
|
||||
trio.CancelScope|None,
|
||||
tuple,
|
||||
]:
|
||||
'''
|
||||
Request to acquire the TTY `Lock` in the root actor, release on
|
||||
exit.
|
||||
|
||||
This helper is for actor's who don't actually need to acquired
|
||||
the debugger but want to wait until the lock is free in the
|
||||
process-tree root such that they don't clobber an ongoing pdb
|
||||
REPL session in some peer or child!
|
||||
|
||||
'''
|
||||
if not debug_mode():
|
||||
yield None
|
||||
return
|
||||
|
||||
task: Task = current_task()
|
||||
async with trio.open_nursery() as n:
|
||||
ctx: Context = await n.start(
|
||||
partial(
|
||||
request_root_stdio_lock,
|
||||
actor_uid=subactor_uid,
|
||||
task_uid=(task.name, id(task)),
|
||||
)
|
||||
)
|
||||
yield ctx
|
||||
ctx.cancel()
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue