Add `--enable-stackscope` pytest plugin flag
New `--enable-stackscope` CLI flag installs a SIGUSR1 →
trio-task-tree-dump handler in pytest itself + every
spawned subactor for live stack visibility during hang
investigations. Lighter than `--tpdb` (no pdb machinery
/ tty-lock contention) — pure stack-only triage.
Plumbing:
- `_testing.pytest.pytest_addoption()` adds the flag.
- `_testing.pytest.pytest_configure()` (when flag set):
* exports `TRACTOR_ENABLE_STACKSCOPE=1` so fork-children
inherit it via environ,
* installs the handler in pytest itself via
`enable_stack_on_sig()`.
- `runtime._runtime.Actor.async_main()` extends the
existing `_debug_mode` gate to ALSO fire when
`TRACTOR_ENABLE_STACKSCOPE` is in env — so subactors
install the same handler at runtime startup.
Capture-bypass tee in `dump_task_tree()`:
Pytest's default `--capture=fd` swallows `log.devx()`
output, making SIGUSR1 dumps invisible right when you
need them. Render the dump once to a `full_dump` str,
then unconditionally tee to:
- `/tmp/tractor-stackscope-<pid>.log` (append-mode,
always written) — guaranteed-readable artifact even
under CI / `nohup` / no-tty. `tail -f` to follow.
- `/dev/tty` (best-effort) — pytest never captures the
tty; ignored if device is missing.
Other,
- squelch the benign `RuntimeWarning` ("coroutine method
'asend'/'athrow' was never awaited") from
`stackscope._glue`'s import-time async-gen type
introspection so `--enable-stackscope` setup stays
quiet.
- log msg in the `_runtime` ImportError branch now
mentions `--enable-stackscope` alongside debug-mode.
Usage,
pytest --enable-stackscope -k <hang-test>
# in another shell, find the pid + signal:
kill -USR1 <pytest-or-subactor-pid>
# tail the artifact:
tail -f /tmp/tractor-stackscope-<pid>.log
(this patch was generated in some part by [`claude-code`][claude-code-gh])
[claude-code-gh]: https://github.com/anthropics/claude-code
subint_forkserver_backend
parent
383b0fdd75
commit
5418f2dc3c
|
|
@ -213,6 +213,21 @@ def pytest_addoption(
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
parser.addoption(
|
||||||
|
"--enable-stackscope",
|
||||||
|
action="store_true",
|
||||||
|
dest='tractor_enable_stackscope',
|
||||||
|
default=False,
|
||||||
|
help=(
|
||||||
|
'Install `stackscope` SIGUSR1 handler in pytest + '
|
||||||
|
'every spawned subactor for live trio task-tree '
|
||||||
|
'dumps during hang investigations. Lighter than '
|
||||||
|
'`--tpdb` (no pdb machinery / tty-lock contention) '
|
||||||
|
'— use when you only need stack visibility. To '
|
||||||
|
'capture: `kill -USR1 <pytest-or-subactor-pid>`.'
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
# provide which IPC transport protocols opting-in test suites
|
# provide which IPC transport protocols opting-in test suites
|
||||||
# should accumulatively run against.
|
# should accumulatively run against.
|
||||||
parser.addoption(
|
parser.addoption(
|
||||||
|
|
@ -253,6 +268,37 @@ def pytest_configure(
|
||||||
'in `ai/conc-anal/subint_sigint_starvation_issue.md`).'
|
'in `ai/conc-anal/subint_sigint_starvation_issue.md`).'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# `--enable-stackscope`: install SIGUSR1 → trio task-tree
|
||||||
|
# dump in pytest itself + propagate to every subactor via
|
||||||
|
# an env var that fork-children inherit and the runtime
|
||||||
|
# gate honors. Lighter than `--tpdb` (no pdb machinery) —
|
||||||
|
# purely for hang-investigation stack visibility.
|
||||||
|
if getattr(
|
||||||
|
config.option, 'tractor_enable_stackscope', False
|
||||||
|
):
|
||||||
|
import os
|
||||||
|
# Env var inherited via fork → subactor's runtime
|
||||||
|
# picks it up at `Actor.async_main` startup. See the
|
||||||
|
# gate in `tractor.runtime._runtime` matching this
|
||||||
|
# var name.
|
||||||
|
os.environ['TRACTOR_ENABLE_STACKSCOPE'] = '1'
|
||||||
|
|
||||||
|
# Install in pytest itself so `kill -USR1 <pytest>`
|
||||||
|
# dumps the parent trio task-tree (which is where
|
||||||
|
# most Mode-A-class hangs park).
|
||||||
|
try:
|
||||||
|
from tractor.devx._stackscope import (
|
||||||
|
enable_stack_on_sig,
|
||||||
|
)
|
||||||
|
enable_stack_on_sig()
|
||||||
|
except ImportError:
|
||||||
|
import warnings
|
||||||
|
warnings.warn(
|
||||||
|
'`stackscope` not installed — '
|
||||||
|
'--enable-stackscope is a no-op. '
|
||||||
|
'Install via the `devx` dep group.'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def pytest_collection_modifyitems(
|
def pytest_collection_modifyitems(
|
||||||
config: pytest.Config,
|
config: pytest.Config,
|
||||||
|
|
|
||||||
|
|
@ -66,7 +66,20 @@ def dump_task_tree() -> None:
|
||||||
Do a classic `stackscope.extract()` task-tree dump to console at
|
Do a classic `stackscope.extract()` task-tree dump to console at
|
||||||
`.devx()` level.
|
`.devx()` level.
|
||||||
|
|
||||||
|
Also unconditionally tee the rendered tree to two
|
||||||
|
capture-bypassing sinks so SIGUSR1 dumps remain visible
|
||||||
|
when the parent process has captured stdio (e.g. pytest's
|
||||||
|
default `--capture=fd`):
|
||||||
|
|
||||||
|
- `/tmp/tractor-stackscope-<pid>.log` (append-mode, always
|
||||||
|
written) — guaranteed-readable artifact even under CI
|
||||||
|
/ `nohup` / no-tty conditions. `tail -f` to follow.
|
||||||
|
- `/dev/tty` if a controlling terminal is attached —
|
||||||
|
best-effort, ignored if the device is missing or write
|
||||||
|
fails. pytest never captures the tty.
|
||||||
|
|
||||||
'''
|
'''
|
||||||
|
import os
|
||||||
import stackscope
|
import stackscope
|
||||||
tree_str: str = str(
|
tree_str: str = str(
|
||||||
stackscope.extract(
|
stackscope.extract(
|
||||||
|
|
@ -96,7 +109,7 @@ def dump_task_tree() -> None:
|
||||||
# |_{Supervisor/Scope
|
# |_{Supervisor/Scope
|
||||||
# |_[Storage/Memory/IPC-Stream/Data-Struct
|
# |_[Storage/Memory/IPC-Stream/Data-Struct
|
||||||
|
|
||||||
log.devx(
|
full_dump: str = (
|
||||||
f'Dumping `stackscope` tree for actor\n'
|
f'Dumping `stackscope` tree for actor\n'
|
||||||
f'(>: {actor.uid!r}\n'
|
f'(>: {actor.uid!r}\n'
|
||||||
f' |_{mp.current_process()}\n'
|
f' |_{mp.current_process()}\n'
|
||||||
|
|
@ -105,33 +118,35 @@ def dump_task_tree() -> None:
|
||||||
f'\n'
|
f'\n'
|
||||||
f'{sigint_handler_report}\n'
|
f'{sigint_handler_report}\n'
|
||||||
f'signal.getsignal(SIGINT) -> {current_sigint_handler!r}\n'
|
f'signal.getsignal(SIGINT) -> {current_sigint_handler!r}\n'
|
||||||
# f'\n'
|
|
||||||
# start-of-trace-tree delimiter (mostly for testing)
|
|
||||||
# f'------ {actor.uid!r} ------\n'
|
|
||||||
f'\n'
|
f'\n'
|
||||||
f'------ start-of-{actor.uid!r} ------\n'
|
f'------ start-of-{actor.uid!r} ------\n'
|
||||||
f'|\n'
|
f'|\n'
|
||||||
f'{tree_str}'
|
f'{tree_str}'
|
||||||
# end-of-trace-tree delimiter (mostly for testing)
|
|
||||||
f'|\n'
|
f'|\n'
|
||||||
f'|_____ end-of-{actor.uid!r} ______\n'
|
f'|_____ end-of-{actor.uid!r} ______\n'
|
||||||
)
|
)
|
||||||
# TODO: can remove this right?
|
log.devx(full_dump)
|
||||||
# -[ ] was original code from author
|
|
||||||
#
|
# NOTE, capture-bypass sinks. Pytest's default
|
||||||
# print(
|
# `--capture=fd` swallows `log.devx()` above; the
|
||||||
# 'DUMPING FROM PRINT\n'
|
# following two writes guarantee the dump reaches the
|
||||||
# +
|
# human even when stdio is captured.
|
||||||
# content
|
fpath: str = f'/tmp/tractor-stackscope-{os.getpid()}.log'
|
||||||
# )
|
try:
|
||||||
# import logging
|
with open(fpath, 'a') as f:
|
||||||
# try:
|
f.write(full_dump + '\n')
|
||||||
# with open("/dev/tty", "w") as tty:
|
except OSError:
|
||||||
# tty.write(tree_str)
|
log.exception(
|
||||||
# except BaseException:
|
f'Failed to tee stackscope dump to {fpath!r}'
|
||||||
# logging.getLogger(
|
)
|
||||||
# "task_tree"
|
|
||||||
# ).exception("Error printing task tree")
|
try:
|
||||||
|
with open('/dev/tty', 'w') as tty:
|
||||||
|
tty.write(full_dump + '\n')
|
||||||
|
except OSError:
|
||||||
|
# no controlling tty (CI / nohup / detached) —
|
||||||
|
# silently fall through; the file sink covers it.
|
||||||
|
pass
|
||||||
|
|
||||||
_handler_lock = RLock()
|
_handler_lock = RLock()
|
||||||
_tree_dumped: bool = False
|
_tree_dumped: bool = False
|
||||||
|
|
@ -233,6 +248,19 @@ def enable_stack_on_sig(
|
||||||
|
|
||||||
'''
|
'''
|
||||||
try:
|
try:
|
||||||
|
# NOTE, `stackscope._glue` does intentional async-gen type
|
||||||
|
# introspection at import-time which trips
|
||||||
|
# `RuntimeWarning: coroutine method 'asend'/'athrow' was
|
||||||
|
# never awaited`. Benign — they only want the wrapper
|
||||||
|
# type — but visible to users. Squelch the import-only
|
||||||
|
# warning so SIGUSR1 setup stays quiet.
|
||||||
|
import warnings
|
||||||
|
with warnings.catch_warnings():
|
||||||
|
warnings.filterwarnings(
|
||||||
|
'ignore',
|
||||||
|
category=RuntimeWarning,
|
||||||
|
message=r"coroutine method '(asend|athrow)' .* was never awaited",
|
||||||
|
)
|
||||||
import stackscope
|
import stackscope
|
||||||
except ImportError:
|
except ImportError:
|
||||||
log.warning(
|
log.warning(
|
||||||
|
|
|
||||||
|
|
@ -932,7 +932,20 @@ class Actor:
|
||||||
# => update process-wide globals
|
# => update process-wide globals
|
||||||
# TODO! -[ ] another `Struct` for rtvs..
|
# TODO! -[ ] another `Struct` for rtvs..
|
||||||
rvs: dict[str, Any] = spawnspec._runtime_vars
|
rvs: dict[str, Any] = spawnspec._runtime_vars
|
||||||
if rvs['_debug_mode']:
|
|
||||||
|
# `stackscope` SIGUSR1 handler: install when EITHER
|
||||||
|
# `_debug_mode=True` (full multi-actor pdb support
|
||||||
|
# path) OR the `TRACTOR_ENABLE_STACKSCOPE` env var
|
||||||
|
# is set (lighter test-time hang-debug path; see
|
||||||
|
# `tractor._testing.pytest`'s `--enable-stackscope`
|
||||||
|
# CLI flag — env var propagates via fork-inherited
|
||||||
|
# environ).
|
||||||
|
import os
|
||||||
|
if (
|
||||||
|
rvs['_debug_mode']
|
||||||
|
or
|
||||||
|
os.environ.get('TRACTOR_ENABLE_STACKSCOPE')
|
||||||
|
):
|
||||||
from ..devx import (
|
from ..devx import (
|
||||||
enable_stack_on_sig,
|
enable_stack_on_sig,
|
||||||
maybe_init_greenback,
|
maybe_init_greenback,
|
||||||
|
|
@ -948,7 +961,8 @@ class Actor:
|
||||||
|
|
||||||
except ImportError:
|
except ImportError:
|
||||||
log.warning(
|
log.warning(
|
||||||
'`stackscope` not installed for use in debug mode!'
|
'`stackscope` not installed for use in '
|
||||||
|
'debug mode / `--enable-stackscope`!'
|
||||||
)
|
)
|
||||||
|
|
||||||
if rvs.get('use_greenback', False):
|
if rvs.get('use_greenback', False):
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue