Add `--enable-stackscope` pytest plugin flag
New `--enable-stackscope` CLI flag installs a SIGUSR1 →
trio-task-tree-dump handler in pytest itself + every
spawned subactor for live stack visibility during hang
investigations. Lighter than `--tpdb` (no pdb machinery
/ tty-lock contention) — pure stack-only triage.
Plumbing:
- `_testing.pytest.pytest_addoption()` adds the flag.
- `_testing.pytest.pytest_configure()` (when flag set):
* exports `TRACTOR_ENABLE_STACKSCOPE=1` so fork-children
inherit it via environ,
* installs the handler in pytest itself via
`enable_stack_on_sig()`.
- `runtime._runtime.Actor.async_main()` extends the
existing `_debug_mode` gate to ALSO fire when
`TRACTOR_ENABLE_STACKSCOPE` is in env — so subactors
install the same handler at runtime startup.
Capture-bypass tee in `dump_task_tree()`:
Pytest's default `--capture=fd` swallows `log.devx()`
output, making SIGUSR1 dumps invisible right when you
need them. Render the dump once to a `full_dump` str,
then unconditionally tee to:
- `/tmp/tractor-stackscope-<pid>.log` (append-mode,
always written) — guaranteed-readable artifact even
under CI / `nohup` / no-tty. `tail -f` to follow.
- `/dev/tty` (best-effort) — pytest never captures the
tty; ignored if device is missing.
Other,
- squelch the benign `RuntimeWarning` ("coroutine method
'asend'/'athrow' was never awaited") from
`stackscope._glue`'s import-time async-gen type
introspection so `--enable-stackscope` setup stays
quiet.
- log msg in the `_runtime` ImportError branch now
mentions `--enable-stackscope` alongside debug-mode.
Usage,
pytest --enable-stackscope -k <hang-test>
# in another shell, find the pid + signal:
kill -USR1 <pytest-or-subactor-pid>
# tail the artifact:
tail -f /tmp/tractor-stackscope-<pid>.log
(this patch was generated in some part by [`claude-code`][claude-code-gh])
[claude-code-gh]: https://github.com/anthropics/claude-code
subint_forkserver_backend
parent
383b0fdd75
commit
5418f2dc3c
|
|
@ -213,6 +213,21 @@ def pytest_addoption(
|
|||
),
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--enable-stackscope",
|
||||
action="store_true",
|
||||
dest='tractor_enable_stackscope',
|
||||
default=False,
|
||||
help=(
|
||||
'Install `stackscope` SIGUSR1 handler in pytest + '
|
||||
'every spawned subactor for live trio task-tree '
|
||||
'dumps during hang investigations. Lighter than '
|
||||
'`--tpdb` (no pdb machinery / tty-lock contention) '
|
||||
'— use when you only need stack visibility. To '
|
||||
'capture: `kill -USR1 <pytest-or-subactor-pid>`.'
|
||||
),
|
||||
)
|
||||
|
||||
# provide which IPC transport protocols opting-in test suites
|
||||
# should accumulatively run against.
|
||||
parser.addoption(
|
||||
|
|
@ -253,6 +268,37 @@ def pytest_configure(
|
|||
'in `ai/conc-anal/subint_sigint_starvation_issue.md`).'
|
||||
)
|
||||
|
||||
# `--enable-stackscope`: install SIGUSR1 → trio task-tree
|
||||
# dump in pytest itself + propagate to every subactor via
|
||||
# an env var that fork-children inherit and the runtime
|
||||
# gate honors. Lighter than `--tpdb` (no pdb machinery) —
|
||||
# purely for hang-investigation stack visibility.
|
||||
if getattr(
|
||||
config.option, 'tractor_enable_stackscope', False
|
||||
):
|
||||
import os
|
||||
# Env var inherited via fork → subactor's runtime
|
||||
# picks it up at `Actor.async_main` startup. See the
|
||||
# gate in `tractor.runtime._runtime` matching this
|
||||
# var name.
|
||||
os.environ['TRACTOR_ENABLE_STACKSCOPE'] = '1'
|
||||
|
||||
# Install in pytest itself so `kill -USR1 <pytest>`
|
||||
# dumps the parent trio task-tree (which is where
|
||||
# most Mode-A-class hangs park).
|
||||
try:
|
||||
from tractor.devx._stackscope import (
|
||||
enable_stack_on_sig,
|
||||
)
|
||||
enable_stack_on_sig()
|
||||
except ImportError:
|
||||
import warnings
|
||||
warnings.warn(
|
||||
'`stackscope` not installed — '
|
||||
'--enable-stackscope is a no-op. '
|
||||
'Install via the `devx` dep group.'
|
||||
)
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(
|
||||
config: pytest.Config,
|
||||
|
|
|
|||
|
|
@ -66,7 +66,20 @@ def dump_task_tree() -> None:
|
|||
Do a classic `stackscope.extract()` task-tree dump to console at
|
||||
`.devx()` level.
|
||||
|
||||
Also unconditionally tee the rendered tree to two
|
||||
capture-bypassing sinks so SIGUSR1 dumps remain visible
|
||||
when the parent process has captured stdio (e.g. pytest's
|
||||
default `--capture=fd`):
|
||||
|
||||
- `/tmp/tractor-stackscope-<pid>.log` (append-mode, always
|
||||
written) — guaranteed-readable artifact even under CI
|
||||
/ `nohup` / no-tty conditions. `tail -f` to follow.
|
||||
- `/dev/tty` if a controlling terminal is attached —
|
||||
best-effort, ignored if the device is missing or write
|
||||
fails. pytest never captures the tty.
|
||||
|
||||
'''
|
||||
import os
|
||||
import stackscope
|
||||
tree_str: str = str(
|
||||
stackscope.extract(
|
||||
|
|
@ -96,7 +109,7 @@ def dump_task_tree() -> None:
|
|||
# |_{Supervisor/Scope
|
||||
# |_[Storage/Memory/IPC-Stream/Data-Struct
|
||||
|
||||
log.devx(
|
||||
full_dump: str = (
|
||||
f'Dumping `stackscope` tree for actor\n'
|
||||
f'(>: {actor.uid!r}\n'
|
||||
f' |_{mp.current_process()}\n'
|
||||
|
|
@ -105,33 +118,35 @@ def dump_task_tree() -> None:
|
|||
f'\n'
|
||||
f'{sigint_handler_report}\n'
|
||||
f'signal.getsignal(SIGINT) -> {current_sigint_handler!r}\n'
|
||||
# f'\n'
|
||||
# start-of-trace-tree delimiter (mostly for testing)
|
||||
# f'------ {actor.uid!r} ------\n'
|
||||
f'\n'
|
||||
f'------ start-of-{actor.uid!r} ------\n'
|
||||
f'|\n'
|
||||
f'{tree_str}'
|
||||
# end-of-trace-tree delimiter (mostly for testing)
|
||||
f'|\n'
|
||||
f'|_____ end-of-{actor.uid!r} ______\n'
|
||||
)
|
||||
# TODO: can remove this right?
|
||||
# -[ ] was original code from author
|
||||
#
|
||||
# print(
|
||||
# 'DUMPING FROM PRINT\n'
|
||||
# +
|
||||
# content
|
||||
# )
|
||||
# import logging
|
||||
# try:
|
||||
# with open("/dev/tty", "w") as tty:
|
||||
# tty.write(tree_str)
|
||||
# except BaseException:
|
||||
# logging.getLogger(
|
||||
# "task_tree"
|
||||
# ).exception("Error printing task tree")
|
||||
log.devx(full_dump)
|
||||
|
||||
# NOTE, capture-bypass sinks. Pytest's default
|
||||
# `--capture=fd` swallows `log.devx()` above; the
|
||||
# following two writes guarantee the dump reaches the
|
||||
# human even when stdio is captured.
|
||||
fpath: str = f'/tmp/tractor-stackscope-{os.getpid()}.log'
|
||||
try:
|
||||
with open(fpath, 'a') as f:
|
||||
f.write(full_dump + '\n')
|
||||
except OSError:
|
||||
log.exception(
|
||||
f'Failed to tee stackscope dump to {fpath!r}'
|
||||
)
|
||||
|
||||
try:
|
||||
with open('/dev/tty', 'w') as tty:
|
||||
tty.write(full_dump + '\n')
|
||||
except OSError:
|
||||
# no controlling tty (CI / nohup / detached) —
|
||||
# silently fall through; the file sink covers it.
|
||||
pass
|
||||
|
||||
_handler_lock = RLock()
|
||||
_tree_dumped: bool = False
|
||||
|
|
@ -233,7 +248,20 @@ def enable_stack_on_sig(
|
|||
|
||||
'''
|
||||
try:
|
||||
import stackscope
|
||||
# NOTE, `stackscope._glue` does intentional async-gen type
|
||||
# introspection at import-time which trips
|
||||
# `RuntimeWarning: coroutine method 'asend'/'athrow' was
|
||||
# never awaited`. Benign — they only want the wrapper
|
||||
# type — but visible to users. Squelch the import-only
|
||||
# warning so SIGUSR1 setup stays quiet.
|
||||
import warnings
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings(
|
||||
'ignore',
|
||||
category=RuntimeWarning,
|
||||
message=r"coroutine method '(asend|athrow)' .* was never awaited",
|
||||
)
|
||||
import stackscope
|
||||
except ImportError:
|
||||
log.warning(
|
||||
'The `stackscope` lib is not installed!\n'
|
||||
|
|
|
|||
|
|
@ -932,7 +932,20 @@ class Actor:
|
|||
# => update process-wide globals
|
||||
# TODO! -[ ] another `Struct` for rtvs..
|
||||
rvs: dict[str, Any] = spawnspec._runtime_vars
|
||||
if rvs['_debug_mode']:
|
||||
|
||||
# `stackscope` SIGUSR1 handler: install when EITHER
|
||||
# `_debug_mode=True` (full multi-actor pdb support
|
||||
# path) OR the `TRACTOR_ENABLE_STACKSCOPE` env var
|
||||
# is set (lighter test-time hang-debug path; see
|
||||
# `tractor._testing.pytest`'s `--enable-stackscope`
|
||||
# CLI flag — env var propagates via fork-inherited
|
||||
# environ).
|
||||
import os
|
||||
if (
|
||||
rvs['_debug_mode']
|
||||
or
|
||||
os.environ.get('TRACTOR_ENABLE_STACKSCOPE')
|
||||
):
|
||||
from ..devx import (
|
||||
enable_stack_on_sig,
|
||||
maybe_init_greenback,
|
||||
|
|
@ -948,7 +961,8 @@ class Actor:
|
|||
|
||||
except ImportError:
|
||||
log.warning(
|
||||
'`stackscope` not installed for use in debug mode!'
|
||||
'`stackscope` not installed for use in '
|
||||
'debug mode / `--enable-stackscope`!'
|
||||
)
|
||||
|
||||
if rvs.get('use_greenback', False):
|
||||
|
|
|
|||
Loading…
Reference in New Issue