From 5418f2dc3c9960fb5036eea508ea32616d19d0d6 Mon Sep 17 00:00:00 2001 From: goodboy Date: Wed, 29 Apr 2026 10:32:23 -0400 Subject: [PATCH] Add `--enable-stackscope` pytest plugin flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New `--enable-stackscope` CLI flag installs a SIGUSR1 → trio-task-tree-dump handler in pytest itself + every spawned subactor for live stack visibility during hang investigations. Lighter than `--tpdb` (no pdb machinery / tty-lock contention) — pure stack-only triage. Plumbing: - `_testing.pytest.pytest_addoption()` adds the flag. - `_testing.pytest.pytest_configure()` (when flag set): * exports `TRACTOR_ENABLE_STACKSCOPE=1` so fork-children inherit it via environ, * installs the handler in pytest itself via `enable_stack_on_sig()`. - `runtime._runtime.Actor.async_main()` extends the existing `_debug_mode` gate to ALSO fire when `TRACTOR_ENABLE_STACKSCOPE` is in env — so subactors install the same handler at runtime startup. Capture-bypass tee in `dump_task_tree()`: Pytest's default `--capture=fd` swallows `log.devx()` output, making SIGUSR1 dumps invisible right when you need them. Render the dump once to a `full_dump` str, then unconditionally tee to: - `/tmp/tractor-stackscope-.log` (append-mode, always written) — guaranteed-readable artifact even under CI / `nohup` / no-tty. `tail -f` to follow. - `/dev/tty` (best-effort) — pytest never captures the tty; ignored if device is missing. Other, - squelch the benign `RuntimeWarning` ("coroutine method 'asend'/'athrow' was never awaited") from `stackscope._glue`'s import-time async-gen type introspection so `--enable-stackscope` setup stays quiet. - log msg in the `_runtime` ImportError branch now mentions `--enable-stackscope` alongside debug-mode. Usage, pytest --enable-stackscope -k # in another shell, find the pid + signal: kill -USR1 # tail the artifact: tail -f /tmp/tractor-stackscope-.log (this patch was generated in some part by [`claude-code`][claude-code-gh]) [claude-code-gh]: https://github.com/anthropics/claude-code --- tractor/_testing/pytest.py | 46 ++++++++++++++++++++++++ tractor/devx/_stackscope.py | 72 +++++++++++++++++++++++++------------ tractor/runtime/_runtime.py | 18 ++++++++-- 3 files changed, 112 insertions(+), 24 deletions(-) diff --git a/tractor/_testing/pytest.py b/tractor/_testing/pytest.py index fd78955c..66d2ccd9 100644 --- a/tractor/_testing/pytest.py +++ b/tractor/_testing/pytest.py @@ -213,6 +213,21 @@ def pytest_addoption( ), ) + parser.addoption( + "--enable-stackscope", + action="store_true", + dest='tractor_enable_stackscope', + default=False, + help=( + 'Install `stackscope` SIGUSR1 handler in pytest + ' + 'every spawned subactor for live trio task-tree ' + 'dumps during hang investigations. Lighter than ' + '`--tpdb` (no pdb machinery / tty-lock contention) ' + '— use when you only need stack visibility. To ' + 'capture: `kill -USR1 `.' + ), + ) + # provide which IPC transport protocols opting-in test suites # should accumulatively run against. parser.addoption( @@ -253,6 +268,37 @@ def pytest_configure( 'in `ai/conc-anal/subint_sigint_starvation_issue.md`).' ) + # `--enable-stackscope`: install SIGUSR1 → trio task-tree + # dump in pytest itself + propagate to every subactor via + # an env var that fork-children inherit and the runtime + # gate honors. Lighter than `--tpdb` (no pdb machinery) — + # purely for hang-investigation stack visibility. + if getattr( + config.option, 'tractor_enable_stackscope', False + ): + import os + # Env var inherited via fork → subactor's runtime + # picks it up at `Actor.async_main` startup. See the + # gate in `tractor.runtime._runtime` matching this + # var name. + os.environ['TRACTOR_ENABLE_STACKSCOPE'] = '1' + + # Install in pytest itself so `kill -USR1 ` + # dumps the parent trio task-tree (which is where + # most Mode-A-class hangs park). + try: + from tractor.devx._stackscope import ( + enable_stack_on_sig, + ) + enable_stack_on_sig() + except ImportError: + import warnings + warnings.warn( + '`stackscope` not installed — ' + '--enable-stackscope is a no-op. ' + 'Install via the `devx` dep group.' + ) + def pytest_collection_modifyitems( config: pytest.Config, diff --git a/tractor/devx/_stackscope.py b/tractor/devx/_stackscope.py index 6a9ecd48..3992858f 100644 --- a/tractor/devx/_stackscope.py +++ b/tractor/devx/_stackscope.py @@ -66,7 +66,20 @@ def dump_task_tree() -> None: Do a classic `stackscope.extract()` task-tree dump to console at `.devx()` level. + Also unconditionally tee the rendered tree to two + capture-bypassing sinks so SIGUSR1 dumps remain visible + when the parent process has captured stdio (e.g. pytest's + default `--capture=fd`): + + - `/tmp/tractor-stackscope-.log` (append-mode, always + written) — guaranteed-readable artifact even under CI + / `nohup` / no-tty conditions. `tail -f` to follow. + - `/dev/tty` if a controlling terminal is attached — + best-effort, ignored if the device is missing or write + fails. pytest never captures the tty. + ''' + import os import stackscope tree_str: str = str( stackscope.extract( @@ -96,7 +109,7 @@ def dump_task_tree() -> None: # |_{Supervisor/Scope # |_[Storage/Memory/IPC-Stream/Data-Struct - log.devx( + full_dump: str = ( f'Dumping `stackscope` tree for actor\n' f'(>: {actor.uid!r}\n' f' |_{mp.current_process()}\n' @@ -105,33 +118,35 @@ def dump_task_tree() -> None: f'\n' f'{sigint_handler_report}\n' f'signal.getsignal(SIGINT) -> {current_sigint_handler!r}\n' - # f'\n' - # start-of-trace-tree delimiter (mostly for testing) - # f'------ {actor.uid!r} ------\n' f'\n' f'------ start-of-{actor.uid!r} ------\n' f'|\n' f'{tree_str}' - # end-of-trace-tree delimiter (mostly for testing) f'|\n' f'|_____ end-of-{actor.uid!r} ______\n' ) - # TODO: can remove this right? - # -[ ] was original code from author - # - # print( - # 'DUMPING FROM PRINT\n' - # + - # content - # ) - # import logging - # try: - # with open("/dev/tty", "w") as tty: - # tty.write(tree_str) - # except BaseException: - # logging.getLogger( - # "task_tree" - # ).exception("Error printing task tree") + log.devx(full_dump) + + # NOTE, capture-bypass sinks. Pytest's default + # `--capture=fd` swallows `log.devx()` above; the + # following two writes guarantee the dump reaches the + # human even when stdio is captured. + fpath: str = f'/tmp/tractor-stackscope-{os.getpid()}.log' + try: + with open(fpath, 'a') as f: + f.write(full_dump + '\n') + except OSError: + log.exception( + f'Failed to tee stackscope dump to {fpath!r}' + ) + + try: + with open('/dev/tty', 'w') as tty: + tty.write(full_dump + '\n') + except OSError: + # no controlling tty (CI / nohup / detached) — + # silently fall through; the file sink covers it. + pass _handler_lock = RLock() _tree_dumped: bool = False @@ -233,7 +248,20 @@ def enable_stack_on_sig( ''' try: - import stackscope + # NOTE, `stackscope._glue` does intentional async-gen type + # introspection at import-time which trips + # `RuntimeWarning: coroutine method 'asend'/'athrow' was + # never awaited`. Benign — they only want the wrapper + # type — but visible to users. Squelch the import-only + # warning so SIGUSR1 setup stays quiet. + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings( + 'ignore', + category=RuntimeWarning, + message=r"coroutine method '(asend|athrow)' .* was never awaited", + ) + import stackscope except ImportError: log.warning( 'The `stackscope` lib is not installed!\n' diff --git a/tractor/runtime/_runtime.py b/tractor/runtime/_runtime.py index 8ab55df0..8b6780cd 100644 --- a/tractor/runtime/_runtime.py +++ b/tractor/runtime/_runtime.py @@ -932,7 +932,20 @@ class Actor: # => update process-wide globals # TODO! -[ ] another `Struct` for rtvs.. rvs: dict[str, Any] = spawnspec._runtime_vars - if rvs['_debug_mode']: + + # `stackscope` SIGUSR1 handler: install when EITHER + # `_debug_mode=True` (full multi-actor pdb support + # path) OR the `TRACTOR_ENABLE_STACKSCOPE` env var + # is set (lighter test-time hang-debug path; see + # `tractor._testing.pytest`'s `--enable-stackscope` + # CLI flag — env var propagates via fork-inherited + # environ). + import os + if ( + rvs['_debug_mode'] + or + os.environ.get('TRACTOR_ENABLE_STACKSCOPE') + ): from ..devx import ( enable_stack_on_sig, maybe_init_greenback, @@ -948,7 +961,8 @@ class Actor: except ImportError: log.warning( - '`stackscope` not installed for use in debug mode!' + '`stackscope` not installed for use in ' + 'debug mode / `--enable-stackscope`!' ) if rvs.get('use_greenback', False):