diff --git a/scripts/hung-dump.xsh b/scripts/hung-dump.xsh new file mode 100644 index 00000000..564f8467 --- /dev/null +++ b/scripts/hung-dump.xsh @@ -0,0 +1,89 @@ +""" +`hung-dump`: snapshot diagnostic state for a hung +`pytest`/`tractor` process tree. + +For each pid (and all `pgrep -P` descendants), prints +- `ps` forest header, +- `/proc//wchan` + `/proc//stack` (kernel-side + blocked-on syscall), +- `sudo py-spy dump` (python-side stack). + +Usage (xonsh): + source-foreign xonsh ./scripts/hung-dump.xsh + hung-dump [ ...] + +Or just: + source ./scripts/hung-dump.xsh + +Tip — pipe to a paste buffer: + hung-dump 1336765 |t /tmp/hung.log +""" + +def _hung_dump(args): + import subprocess as sp + from pathlib import Path + + if not args: + print('usage: hung-dump [ ...]') + return 1 + + def kids(pid): + try: + out = sp.check_output( + ['pgrep', '-P', str(pid)], + text=True, + ) + except sp.CalledProcessError: + return [] + return [int(p) for p in out.split() if p] + + def tree(pid): + out = [pid] + for k in kids(pid): + out.extend(tree(k)) + return out + + pids = [ + p + for r in (int(a) for a in args) + for p in tree(r) + ] + + print(f'# tree: {pids}') + print('\n## ps forest') + $[ps -o pid,ppid,pgid,stat,cmd -p @(','.join(map(str, pids)))] + + for p in pids: + print(f'\n## pid {p}') + for f in ('wchan', 'stack'): + path = Path(f'/proc/{p}/{f}') + try: + txt = path.read_text().rstrip() + print(f'-- /proc/{p}/{f} --\n{txt}') + except PermissionError: + # `stack` requires CAP_SYS_PTRACE; retry + # via sudo -n so creds-cached users don't + # block on prompt. + try: + txt = sp.check_output( + ['sudo', '-n', 'cat', str(path)], + text=True, + stderr=sp.DEVNULL, + ).rstrip() + print(f'-- /proc/{p}/{f} (via sudo) --\n{txt}') + except sp.CalledProcessError: + print( + f'-- /proc/{p}/{f}: ' + 'PermissionError (try `sudo true` first) --' + ) + except FileNotFoundError: + print(f'-- /proc/{p}/{f}: proc gone --') + + print(f'-- py-spy {p} --') + try: + $[sudo -n py-spy dump --pid @(p)] + except Exception as e: + print(f' (py-spy failed: {e})') + + +aliases['hung-dump'] = _hung_dump