Add `--enable-stackscope` pytest plugin flag

New `--enable-stackscope` CLI flag installs a SIGUSR1 →
trio-task-tree-dump handler in pytest itself + every
spawned subactor for live stack visibility during hang
investigations. Lighter than `--tpdb` (no pdb machinery
/ tty-lock contention) — pure stack-only triage.

Plumbing:
- `_testing.pytest.pytest_addoption()` adds the flag.
- `_testing.pytest.pytest_configure()` (when flag set):
  * exports `TRACTOR_ENABLE_STACKSCOPE=1` so fork-children
    inherit it via environ,
  * installs the handler in pytest itself via
    `enable_stack_on_sig()`.
- `runtime._runtime.Actor.async_main()` extends the
  existing `_debug_mode` gate to ALSO fire when
  `TRACTOR_ENABLE_STACKSCOPE` is in env — so subactors
  install the same handler at runtime startup.

Capture-bypass tee in `dump_task_tree()`:
Pytest's default `--capture=fd` swallows `log.devx()`
output, making SIGUSR1 dumps invisible right when you
need them. Render the dump once to a `full_dump` str,
then unconditionally tee to:

- `/tmp/tractor-stackscope-<pid>.log` (append-mode,
  always written) — guaranteed-readable artifact even
  under CI / `nohup` / no-tty. `tail -f` to follow.
- `/dev/tty` (best-effort) — pytest never captures the
  tty; ignored if device is missing.

Other,
- squelch the benign `RuntimeWarning` ("coroutine method
  'asend'/'athrow' was never awaited") from
  `stackscope._glue`'s import-time async-gen type
  introspection so `--enable-stackscope` setup stays
  quiet.
- log msg in the `_runtime` ImportError branch now
  mentions `--enable-stackscope` alongside debug-mode.

Usage,
  pytest --enable-stackscope -k <hang-test>
  # in another shell, find the pid + signal:
  kill -USR1 <pytest-or-subactor-pid>
  # tail the artifact:
  tail -f /tmp/tractor-stackscope-<pid>.log

(this patch was generated in some part by [`claude-code`][claude-code-gh])
[claude-code-gh]: https://github.com/anthropics/claude-code
subint_forkserver_backend
Gud Boi 2026-04-29 10:32:23 -04:00
parent 383b0fdd75
commit 5418f2dc3c
3 changed files with 112 additions and 24 deletions

View File

@ -213,6 +213,21 @@ def pytest_addoption(
), ),
) )
parser.addoption(
"--enable-stackscope",
action="store_true",
dest='tractor_enable_stackscope',
default=False,
help=(
'Install `stackscope` SIGUSR1 handler in pytest + '
'every spawned subactor for live trio task-tree '
'dumps during hang investigations. Lighter than '
'`--tpdb` (no pdb machinery / tty-lock contention) '
'— use when you only need stack visibility. To '
'capture: `kill -USR1 <pytest-or-subactor-pid>`.'
),
)
# provide which IPC transport protocols opting-in test suites # provide which IPC transport protocols opting-in test suites
# should accumulatively run against. # should accumulatively run against.
parser.addoption( parser.addoption(
@ -253,6 +268,37 @@ def pytest_configure(
'in `ai/conc-anal/subint_sigint_starvation_issue.md`).' 'in `ai/conc-anal/subint_sigint_starvation_issue.md`).'
) )
# `--enable-stackscope`: install SIGUSR1 → trio task-tree
# dump in pytest itself + propagate to every subactor via
# an env var that fork-children inherit and the runtime
# gate honors. Lighter than `--tpdb` (no pdb machinery) —
# purely for hang-investigation stack visibility.
if getattr(
config.option, 'tractor_enable_stackscope', False
):
import os
# Env var inherited via fork → subactor's runtime
# picks it up at `Actor.async_main` startup. See the
# gate in `tractor.runtime._runtime` matching this
# var name.
os.environ['TRACTOR_ENABLE_STACKSCOPE'] = '1'
# Install in pytest itself so `kill -USR1 <pytest>`
# dumps the parent trio task-tree (which is where
# most Mode-A-class hangs park).
try:
from tractor.devx._stackscope import (
enable_stack_on_sig,
)
enable_stack_on_sig()
except ImportError:
import warnings
warnings.warn(
'`stackscope` not installed — '
'--enable-stackscope is a no-op. '
'Install via the `devx` dep group.'
)
def pytest_collection_modifyitems( def pytest_collection_modifyitems(
config: pytest.Config, config: pytest.Config,

View File

@ -66,7 +66,20 @@ def dump_task_tree() -> None:
Do a classic `stackscope.extract()` task-tree dump to console at Do a classic `stackscope.extract()` task-tree dump to console at
`.devx()` level. `.devx()` level.
Also unconditionally tee the rendered tree to two
capture-bypassing sinks so SIGUSR1 dumps remain visible
when the parent process has captured stdio (e.g. pytest's
default `--capture=fd`):
- `/tmp/tractor-stackscope-<pid>.log` (append-mode, always
written) guaranteed-readable artifact even under CI
/ `nohup` / no-tty conditions. `tail -f` to follow.
- `/dev/tty` if a controlling terminal is attached
best-effort, ignored if the device is missing or write
fails. pytest never captures the tty.
''' '''
import os
import stackscope import stackscope
tree_str: str = str( tree_str: str = str(
stackscope.extract( stackscope.extract(
@ -96,7 +109,7 @@ def dump_task_tree() -> None:
# |_{Supervisor/Scope # |_{Supervisor/Scope
# |_[Storage/Memory/IPC-Stream/Data-Struct # |_[Storage/Memory/IPC-Stream/Data-Struct
log.devx( full_dump: str = (
f'Dumping `stackscope` tree for actor\n' f'Dumping `stackscope` tree for actor\n'
f'(>: {actor.uid!r}\n' f'(>: {actor.uid!r}\n'
f' |_{mp.current_process()}\n' f' |_{mp.current_process()}\n'
@ -105,33 +118,35 @@ def dump_task_tree() -> None:
f'\n' f'\n'
f'{sigint_handler_report}\n' f'{sigint_handler_report}\n'
f'signal.getsignal(SIGINT) -> {current_sigint_handler!r}\n' f'signal.getsignal(SIGINT) -> {current_sigint_handler!r}\n'
# f'\n'
# start-of-trace-tree delimiter (mostly for testing)
# f'------ {actor.uid!r} ------\n'
f'\n' f'\n'
f'------ start-of-{actor.uid!r} ------\n' f'------ start-of-{actor.uid!r} ------\n'
f'|\n' f'|\n'
f'{tree_str}' f'{tree_str}'
# end-of-trace-tree delimiter (mostly for testing)
f'|\n' f'|\n'
f'|_____ end-of-{actor.uid!r} ______\n' f'|_____ end-of-{actor.uid!r} ______\n'
) )
# TODO: can remove this right? log.devx(full_dump)
# -[ ] was original code from author
# # NOTE, capture-bypass sinks. Pytest's default
# print( # `--capture=fd` swallows `log.devx()` above; the
# 'DUMPING FROM PRINT\n' # following two writes guarantee the dump reaches the
# + # human even when stdio is captured.
# content fpath: str = f'/tmp/tractor-stackscope-{os.getpid()}.log'
# ) try:
# import logging with open(fpath, 'a') as f:
# try: f.write(full_dump + '\n')
# with open("/dev/tty", "w") as tty: except OSError:
# tty.write(tree_str) log.exception(
# except BaseException: f'Failed to tee stackscope dump to {fpath!r}'
# logging.getLogger( )
# "task_tree"
# ).exception("Error printing task tree") try:
with open('/dev/tty', 'w') as tty:
tty.write(full_dump + '\n')
except OSError:
# no controlling tty (CI / nohup / detached) —
# silently fall through; the file sink covers it.
pass
_handler_lock = RLock() _handler_lock = RLock()
_tree_dumped: bool = False _tree_dumped: bool = False
@ -233,7 +248,20 @@ def enable_stack_on_sig(
''' '''
try: try:
import stackscope # NOTE, `stackscope._glue` does intentional async-gen type
# introspection at import-time which trips
# `RuntimeWarning: coroutine method 'asend'/'athrow' was
# never awaited`. Benign — they only want the wrapper
# type — but visible to users. Squelch the import-only
# warning so SIGUSR1 setup stays quiet.
import warnings
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore',
category=RuntimeWarning,
message=r"coroutine method '(asend|athrow)' .* was never awaited",
)
import stackscope
except ImportError: except ImportError:
log.warning( log.warning(
'The `stackscope` lib is not installed!\n' 'The `stackscope` lib is not installed!\n'

View File

@ -932,7 +932,20 @@ class Actor:
# => update process-wide globals # => update process-wide globals
# TODO! -[ ] another `Struct` for rtvs.. # TODO! -[ ] another `Struct` for rtvs..
rvs: dict[str, Any] = spawnspec._runtime_vars rvs: dict[str, Any] = spawnspec._runtime_vars
if rvs['_debug_mode']:
# `stackscope` SIGUSR1 handler: install when EITHER
# `_debug_mode=True` (full multi-actor pdb support
# path) OR the `TRACTOR_ENABLE_STACKSCOPE` env var
# is set (lighter test-time hang-debug path; see
# `tractor._testing.pytest`'s `--enable-stackscope`
# CLI flag — env var propagates via fork-inherited
# environ).
import os
if (
rvs['_debug_mode']
or
os.environ.get('TRACTOR_ENABLE_STACKSCOPE')
):
from ..devx import ( from ..devx import (
enable_stack_on_sig, enable_stack_on_sig,
maybe_init_greenback, maybe_init_greenback,
@ -948,7 +961,8 @@ class Actor:
except ImportError: except ImportError:
log.warning( log.warning(
'`stackscope` not installed for use in debug mode!' '`stackscope` not installed for use in '
'debug mode / `--enable-stackscope`!'
) )
if rvs.get('use_greenback', False): if rvs.get('use_greenback', False):