Add `hung-dump.xsh` hang-triage tool to `scripts`
Snapshot diagnostic state for a hung `pytest`/`tractor` process tree: for each pid (+ `pgrep -P` descendants) dump the `ps` forest, `/proc/<pid>/wchan` + `stack` (kernel-side blocked syscall), and `py-spy dump` (python-side stack). Salvaged from the retired `wkt/warnings_cleanup` branch (untracked, not upstream); homed here as standalone tooling. (this commit msg was generated in some part by [`claude-code`][claude-code-gh]) [claude-code-gh]: https://github.com/anthropics/claude-codewkt/rm_test_warnings
parent
d95cd94e20
commit
3335865d5d
|
|
@ -0,0 +1,89 @@
|
|||
"""
|
||||
`hung-dump`: snapshot diagnostic state for a hung
|
||||
`pytest`/`tractor` process tree.
|
||||
|
||||
For each pid (and all `pgrep -P` descendants), prints
|
||||
- `ps` forest header,
|
||||
- `/proc/<pid>/wchan` + `/proc/<pid>/stack` (kernel-side
|
||||
blocked-on syscall),
|
||||
- `sudo py-spy dump` (python-side stack).
|
||||
|
||||
Usage (xonsh):
|
||||
source-foreign xonsh ./scripts/hung-dump.xsh
|
||||
hung-dump <pid> [<pid> ...]
|
||||
|
||||
Or just:
|
||||
source ./scripts/hung-dump.xsh
|
||||
|
||||
Tip — pipe to a paste buffer:
|
||||
hung-dump 1336765 |t /tmp/hung.log
|
||||
"""
|
||||
|
||||
def _hung_dump(args):
|
||||
import subprocess as sp
|
||||
from pathlib import Path
|
||||
|
||||
if not args:
|
||||
print('usage: hung-dump <pid> [<pid> ...]')
|
||||
return 1
|
||||
|
||||
def kids(pid):
|
||||
try:
|
||||
out = sp.check_output(
|
||||
['pgrep', '-P', str(pid)],
|
||||
text=True,
|
||||
)
|
||||
except sp.CalledProcessError:
|
||||
return []
|
||||
return [int(p) for p in out.split() if p]
|
||||
|
||||
def tree(pid):
|
||||
out = [pid]
|
||||
for k in kids(pid):
|
||||
out.extend(tree(k))
|
||||
return out
|
||||
|
||||
pids = [
|
||||
p
|
||||
for r in (int(a) for a in args)
|
||||
for p in tree(r)
|
||||
]
|
||||
|
||||
print(f'# tree: {pids}')
|
||||
print('\n## ps forest')
|
||||
$[ps -o pid,ppid,pgid,stat,cmd -p @(','.join(map(str, pids)))]
|
||||
|
||||
for p in pids:
|
||||
print(f'\n## pid {p}')
|
||||
for f in ('wchan', 'stack'):
|
||||
path = Path(f'/proc/{p}/{f}')
|
||||
try:
|
||||
txt = path.read_text().rstrip()
|
||||
print(f'-- /proc/{p}/{f} --\n{txt}')
|
||||
except PermissionError:
|
||||
# `stack` requires CAP_SYS_PTRACE; retry
|
||||
# via sudo -n so creds-cached users don't
|
||||
# block on prompt.
|
||||
try:
|
||||
txt = sp.check_output(
|
||||
['sudo', '-n', 'cat', str(path)],
|
||||
text=True,
|
||||
stderr=sp.DEVNULL,
|
||||
).rstrip()
|
||||
print(f'-- /proc/{p}/{f} (via sudo) --\n{txt}')
|
||||
except sp.CalledProcessError:
|
||||
print(
|
||||
f'-- /proc/{p}/{f}: '
|
||||
'PermissionError (try `sudo true` first) --'
|
||||
)
|
||||
except FileNotFoundError:
|
||||
print(f'-- /proc/{p}/{f}: proc gone --')
|
||||
|
||||
print(f'-- py-spy {p} --')
|
||||
try:
|
||||
$[sudo -n py-spy dump --pid @(p)]
|
||||
except Exception as e:
|
||||
print(f' (py-spy failed: {e})')
|
||||
|
||||
|
||||
aliases['hung-dump'] = _hung_dump
|
||||
Loading…
Reference in New Issue