Just import `._runtime` ns in `._root`; be a bit more explicit

Seems to add one more cancellation suite failure as well as now cause
the discovery test to error instead of fail?

.github/workflows/ci.yml

@@ -8,70 +8,46 @@ on:
   workflow_dispatch:
 
 jobs:
-  # ------ sdist ------
+
+  mypy:
+    name: 'MyPy'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: pip install -U . --upgrade-strategy eager -r requirements-test.txt
+
+      - name: Run MyPy check
+        run: mypy tractor/ --ignore-missing-imports --show-traceback
+
   # test that we can generate a software distribution and install it
   # thus avoid missing file issues after packaging.
-  #
-  # -[x] produce sdist with uv
-  # ------ - ------
   sdist-linux:
     name: 'sdist'
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
-      - name: Install latest uv
-        uses: astral-sh/setup-uv@v6
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
 
-      - name: Build sdist as tar.gz
-        run: uv build --sdist --python=3.13
+      - name: Build sdist
+        run: python setup.py sdist --formats=zip
 
-      - name: Install sdist from .tar.gz
-        run: python -m pip install dist/*.tar.gz
-
-  # ------ type-check ------
-  # mypy:
-  #   name: 'MyPy'
-  #   runs-on: ubuntu-latest
-
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v4
-
-  #     - name: Install latest uv
-  #       uses: astral-sh/setup-uv@v6
-
-  #     # faster due to server caching?
-  #     # https://docs.astral.sh/uv/guides/integration/github/#setting-up-python
-  #     - name: "Set up Python"
-  #       uses: actions/setup-python@v6
-  #       with:
-  #         python-version-file: "pyproject.toml"
-
-  #     # w uv
-  #     # - name: Set up Python
-  #     #   run: uv python install
-
-  #     - name: Setup uv venv
-  #       run: uv venv .venv --python=3.13
-
-  #     - name: Install
-  #       run: uv sync --dev
-
-  #     # TODO, ty cmd over repo
-  #     # - name: type check with ty
-  #     #   run: ty ./tractor/
-
-  #     # - uses: actions/cache@v3
-  #     #     name: Cache uv virtenv as default .venv
-  #     #     with:
-  #     #       path: ./.venv
-  #     #       key: venv-${{ hashFiles('uv.lock') }}
-
-  #     - name: Run MyPy check
-  #       run: mypy tractor/ --ignore-missing-imports --show-traceback
+      - name: Install sdist from .zips
+        run: python -m pip install dist/*.zip
 
 
   testing-linux:
@@ -83,45 +59,32 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.13']
+        python: ['3.11']
         spawn_backend: [
           'trio',
-          # 'mp_spawn',
-          # 'mp_forkserver',
+          'mp_spawn',
+          'mp_forkserver',
         ]
 
     steps:
 
-      - uses: actions/checkout@v4
+      - name: Checkout
+        uses: actions/checkout@v2
 
-      - name: 'Install uv + py-${{ matrix.python-version }}'
-        uses: astral-sh/setup-uv@v6
+      - name: Setup python
+        uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: '${{ matrix.python }}'
 
-      # GH way.. faster?
-      # - name: setup-python@v6
-      #   uses: actions/setup-python@v6
-      #   with:
-      #     python-version: '${{ matrix.python-version }}'
+      - name: Install dependencies
+        run: pip install -U . -r requirements-test.txt -r requirements-docs.txt --upgrade-strategy eager
 
-      # consider caching for speedups?
-      # https://docs.astral.sh/uv/guides/integration/github/#caching
-
-      - name: Install the project w uv
-        run: uv sync --all-extras --dev
-
-      # - name: Install dependencies
-      #   run: pip install -U . -r requirements-test.txt -r requirements-docs.txt --upgrade-strategy eager
-
-      - name: List deps tree
-        run: uv tree
+      - name: List dependencies
+        run: pip list
 
       - name: Run tests
-        run: uv run pytest tests/ --spawn-backend=${{ matrix.spawn_backend }} -rsx
+        run: pytest tests/ --spawn-backend=${{ matrix.spawn_backend }} -rsx
 
-  # XXX legacy NOTE XXX
-  #
   # We skip 3.10 on windows for now due to not having any collabs to
   # debug the CI failures. Anyone wanting to hack and solve them is very
   # welcome, but our primary user base is not using that OS.

docs/README.rst

@@ -689,11 +689,9 @@ channel`_!
 .. _msgspec: https://jcristharif.com/msgspec/
 .. _guest: https://trio.readthedocs.io/en/stable/reference-lowlevel.html?highlight=guest%20mode#using-guest-mode-to-run-trio-on-top-of-other-event-loops
 
-..
-   NOTE, on generating badge links from the UI
-   https://docs.github.com/en/actions/how-tos/monitoring-and-troubleshooting-workflows/monitoring-workflows/adding-a-workflow-status-badge?ref=gitguardian-blog-automated-secrets-detection#using-the-ui
-.. |gh_actions| image:: https://github.com/goodboy/tractor/actions/workflows/ci.yml/badge.svg?branch=main
-    :target: https://github.com/goodboy/tractor/actions/workflows/ci.yml
+
+.. |gh_actions| image:: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fgoodboy%2Ftractor%2Fbadge&style=popout-square
+    :target: https://actions-badge.atrox.dev/goodboy/tractor/goto
 
 .. |docs| image:: https://readthedocs.org/projects/tractor/badge/?version=latest
     :target: https://tractor.readthedocs.io/en/latest/?badge=latest

examples/advanced_faults/ipc_failure_during_stream.py

@@ -120,6 +120,7 @@ async def main(
     break_parent_ipc_after: int|bool = False,
     break_child_ipc_after: int|bool = False,
     pre_close: bool = False,
+    tpt_proto: str = 'tcp',
 
 ) -> None:
 
@@ -131,6 +132,7 @@ async def main(
             # a hang since it never engages due to broken IPC
             debug_mode=debug_mode,
             loglevel=loglevel,
+            enable_transports=[tpt_proto],
 
         ) as an,
     ):
@@ -145,7 +147,8 @@ async def main(
             _testing.expect_ctxc(
                 yay=(
                     break_parent_ipc_after
-                    or break_child_ipc_after
+                    or
+                    break_child_ipc_after
                 ),
                 # TODO: we CAN'T remove this right?
                 # since we need the ctxc to bubble up from either

examples/debugging/asyncio_bp.py

@@ -29,7 +29,7 @@ async def bp_then_error(
     to_trio.send_nowait('start')
 
     # NOTE: what happens here inside the hook needs some refinement..
-    # => seems like it's still `._debug._set_trace()` but
+    # => seems like it's still `.debug._set_trace()` but
     #    we set `Lock.local_task_in_debug = 'sync'`, we probably want
     #    some further, at least, meta-data about the task/actor in debug
     #    in terms of making it clear it's `asyncio` mucking about.

examples/debugging/restore_builtin_breakpoint.py

@@ -5,7 +5,7 @@ import trio
 import tractor
 
 # ensure mod-path is correct!
-from tractor.devx._debug import (
+from tractor.devx.debug import (
     _sync_pause_from_builtin as _sync_pause_from_builtin,
 )
 
@@ -19,19 +19,22 @@ async def main() -> None:
     async with tractor.open_nursery(
         debug_mode=True,
         loglevel='devx',
-    ) as an:
-        assert an
+        maybe_enable_greenback=True,
+        # ^XXX REQUIRED to enable `breakpoint()` support (from sync
+        # fns) and thus required here to avoid an assertion err
+        # on the next line
+    ):
         assert (
             (pybp_var := os.environ['PYTHONBREAKPOINT'])
             ==
-            'tractor.devx._debug._sync_pause_from_builtin'
+            'tractor.devx.debug._sync_pause_from_builtin'
         )
 
         # TODO: an assert that verifies the hook has indeed been, hooked
         # XD
         assert (
             (pybp_hook := sys.breakpointhook)
-            is not tractor.devx._debug._set_trace
+            is not tractor.devx.debug._set_trace
         )
 
         print(

examples/debugging/sync_bp.py

@@ -6,7 +6,7 @@ import tractor
 
 # TODO: only import these when not running from test harness?
 # can we detect `pexpect` usage maybe?
-# from tractor.devx._debug import (
+# from tractor.devx.debug import (
 #     get_lock,
 #     get_debug_req,
 # )

pyproject.toml

@@ -61,8 +61,7 @@ dev = [
   # `tractor.devx` tooling
   "greenback>=1.2.1,<2",
   "stackscope>=0.2.2,<0.3",
-  # ^ requires this?
-  "typing-extensions>=4.14.1",
+  "typing-extensions>=4.13.2",  # needed for stackscope
   "pyperclip>=1.9.0",
   "prompt-toolkit>=3.0.50",
   "xonsh>=0.19.2",

tests/conftest.py

@@ -1,24 +1,27 @@
 """
-``tractor`` testing!!
+Top level of the testing suites!
+
 """
+from __future__ import annotations
 import sys
 import subprocess
 import os
-import random
 import signal
 import platform
 import time
 
 import pytest
-import tractor
 from tractor._testing import (
     examples_dir as examples_dir,
     tractor_test as tractor_test,
     expect_ctxc as expect_ctxc,
 )
 
-# TODO: include wtv plugin(s) we build in `._testing.pytest`?
-pytest_plugins = ['pytester']
+pytest_plugins: list[str] = [
+    'pytester',
+    'tractor._testing.pytest',
+]
+
 
 # Sending signal.SIGINT on subprocess fails on windows. Use CTRL_* alternatives
 if platform.system() == 'Windows':
@@ -30,7 +33,11 @@ else:
     _KILL_SIGNAL = signal.SIGKILL
     _INT_SIGNAL = signal.SIGINT
     _INT_RETURN_CODE = 1 if sys.version_info < (3, 8) else -signal.SIGINT.value
-    _PROC_SPAWN_WAIT = 0.6 if sys.version_info < (3, 7) else 0.4
+    _PROC_SPAWN_WAIT = (
+        0.6
+        if sys.version_info < (3, 7)
+        else 0.4
+    )
 
 
 no_windows = pytest.mark.skipif(
@@ -39,7 +46,12 @@ no_windows = pytest.mark.skipif(
 )
 
 
-def pytest_addoption(parser):
+def pytest_addoption(
+    parser: pytest.Parser,
+):
+    # ?TODO? should this be exposed from our `._testing.pytest`
+    # plugin or should we make it more explicit with `--tl` for
+    # tractor logging like we do in other client projects?
     parser.addoption(
         "--ll",
         action="store",
@@ -47,42 +59,10 @@ def pytest_addoption(parser):
         default='ERROR', help="logging level to set when testing"
     )
 
-    parser.addoption(
-        "--spawn-backend",
-        action="store",
-        dest='spawn_backend',
-        default='trio',
-        help="Processing spawning backend to use for test run",
-    )
-
-    parser.addoption(
-        "--tpdb", "--debug-mode",
-        action="store_true",
-        dest='tractor_debug_mode',
-        # default=False,
-        help=(
-            'Enable a flag that can be used by tests to to set the '
-            '`debug_mode: bool` for engaging the internal '
-            'multi-proc debugger sys.'
-        ),
-    )
-
-
-def pytest_configure(config):
-    backend = config.option.spawn_backend
-    tractor._spawn.try_set_start_method(backend)
-
-
-@pytest.fixture(scope='session')
-def debug_mode(request):
-    debug_mode: bool = request.config.option.tractor_debug_mode
-    # if debug_mode:
-    #     breakpoint()
-    return debug_mode
-
 
 @pytest.fixture(scope='session', autouse=True)
 def loglevel(request):
+    import tractor
     orig = tractor.log._default_loglevel
     level = tractor.log._default_loglevel = request.config.option.loglevel
     tractor.log.get_console_log(level)
@@ -90,106 +70,44 @@ def loglevel(request):
     tractor.log._default_loglevel = orig
 
 
-@pytest.fixture(scope='session')
-def spawn_backend(request) -> str:
-    return request.config.option.spawn_backend
-
-
-# @pytest.fixture(scope='function', autouse=True)
-# def debug_enabled(request) -> str:
-#     from tractor import _state
-#     if _state._runtime_vars['_debug_mode']:
-#         breakpoint()
-
 _ci_env: bool = os.environ.get('CI', False)
 
 
 @pytest.fixture(scope='session')
 def ci_env() -> bool:
     '''
-    Detect CI envoirment.
+    Detect CI environment.
 
     '''
     return _ci_env
 
 
-# TODO: also move this to `._testing` for now?
-# -[ ] possibly generalize and re-use for multi-tree spawning
-#    along with the new stuff for multi-addrs in distribute_dis
-#    branch?
-#
-# choose randomly at import time
-_reg_addr: tuple[str, int] = (
-    '127.0.0.1',
-    random.randint(1000, 9999),
-)
-
-
-@pytest.fixture(scope='session')
-def reg_addr() -> tuple[str, int]:
-
-    # globally override the runtime to the per-test-session-dynamic
-    # addr so that all tests never conflict with any other actor
-    # tree using the default.
-    from tractor import _root
-    _root._default_lo_addrs = [_reg_addr]
-
-    return _reg_addr
-
-
-def pytest_generate_tests(metafunc):
-    spawn_backend = metafunc.config.option.spawn_backend
-
-    if not spawn_backend:
-        # XXX some weird windows bug with `pytest`?
-        spawn_backend = 'trio'
-
-    # TODO: maybe just use the literal `._spawn.SpawnMethodKey`?
-    assert spawn_backend in (
-        'mp_spawn',
-        'mp_forkserver',
-        'trio',
-    )
-
-    # NOTE: used to be used to dyanmically parametrize tests for when
-    # you just passed --spawn-backend=`mp` on the cli, but now we expect
-    # that cli input to be manually specified, BUT, maybe we'll do
-    # something like this again in the future?
-    if 'start_method' in metafunc.fixturenames:
-        metafunc.parametrize("start_method", [spawn_backend], scope='module')
-
-
-# TODO: a way to let test scripts (like from `examples/`)
-# guarantee they won't registry addr collide!
-# @pytest.fixture
-# def open_test_runtime(
-#     reg_addr: tuple,
-# ) -> AsyncContextManager:
-#     return partial(
-#         tractor.open_nursery,
-#         registry_addrs=[reg_addr],
-#     )
-
-
-def sig_prog(proc, sig):
+def sig_prog(
+    proc: subprocess.Popen,
+    sig: int,
+    canc_timeout: float = 0.1,
+) -> int:
     "Kill the actor-process with ``sig``."
     proc.send_signal(sig)
-    time.sleep(0.1)
+    time.sleep(canc_timeout)
     if not proc.poll():
         # TODO: why sometimes does SIGINT not work on teardown?
         # seems to happen only when trace logging enabled?
         proc.send_signal(_KILL_SIGNAL)
-    ret = proc.wait()
+    ret: int = proc.wait()
     assert ret
 
 
 # TODO: factor into @cm and move to `._testing`?
 @pytest.fixture
 def daemon(
+    debug_mode: bool,
     loglevel: str,
-    testdir,
+    testdir: pytest.Pytester,
     reg_addr: tuple[str, int],
-):
+    tpt_proto: str,
+
+) -> subprocess.Popen:
     '''
     Run a daemon root actor as a separate actor-process tree and
     "remote registrar" for discovery-protocol related tests.
@@ -200,28 +118,100 @@ def daemon(
         loglevel: str = 'info'
 
     code: str = (
-            "import tractor; "
-            "tractor.run_daemon([], registry_addrs={reg_addrs}, loglevel={ll})"
+        "import tractor; "
+        "tractor.run_daemon([], "
+        "registry_addrs={reg_addrs}, "
+        "debug_mode={debug_mode}, "
+        "loglevel={ll})"
     ).format(
         reg_addrs=str([reg_addr]),
         ll="'{}'".format(loglevel) if loglevel else None,
+        debug_mode=debug_mode,
     )
     cmd: list[str] = [
         sys.executable,
         '-c', code,
     ]
+    # breakpoint()
     kwargs = {}
     if platform.system() == 'Windows':
         # without this, tests hang on windows forever
         kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
 
-    proc = testdir.popen(
+    proc: subprocess.Popen = testdir.popen(
         cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
         **kwargs,
     )
-    assert not proc.returncode
+
+    # UDS sockets are **really** fast to bind()/listen()/connect()
+    # so it's often required that we delay a bit more starting
+    # the first actor-tree..
+    if tpt_proto == 'uds':
+        global _PROC_SPAWN_WAIT
+        _PROC_SPAWN_WAIT = 0.6
+
     time.sleep(_PROC_SPAWN_WAIT)
+
+    assert not proc.returncode
     yield proc
     sig_prog(proc, _INT_SIGNAL)
+
+    # XXX! yeah.. just be reaaal careful with this bc sometimes it
+    # can lock up on the `_io.BufferedReader` and hang..
+    stderr: str = proc.stderr.read().decode()
+    if stderr:
+        print(
+            f'Daemon actor tree produced STDERR:\n'
+            f'{proc.args}\n'
+            f'\n'
+            f'{stderr}\n'
+        )
+    if proc.returncode != -2:
+        raise RuntimeError(
+            'Daemon actor tree failed !?\n'
+            f'{proc.args}\n'
+        )
+
+
+# @pytest.fixture(autouse=True)
+# def shared_last_failed(pytestconfig):
+#     val = pytestconfig.cache.get("example/value", None)
+#     breakpoint()
+#     if val is None:
+#         pytestconfig.cache.set("example/value", val)
+#     return val
+
+
+# TODO: a way to let test scripts (like from `examples/`)
+# guarantee they won't `registry_addrs` collide!
+# -[ ] maybe use some kinda standard `def main()` arg-spec that
+#     we can introspect from a fixture that is called from the test
+#     body?
+# -[ ] test and figure out typing for below prototype! Bp
+#
+# @pytest.fixture
+# def set_script_runtime_args(
+#     reg_addr: tuple,
+# ) -> Callable[[...], None]:
+
+#     def import_n_partial_in_args_n_triorun(
+#         script: Path,  # under examples?
+#         **runtime_args,
+#     ) -> Callable[[], Any]:  # a `partial`-ed equiv of `trio.run()`
+
+#         # NOTE, below is taken from
+#         # `.test_advanced_faults.test_ipc_channel_break_during_stream`
+#         mod: ModuleType = import_path(
+#             examples_dir() / 'advanced_faults'
+#             / 'ipc_failure_during_stream.py',
+#             root=examples_dir(),
+#             consider_namespace_packages=False,
+#         )
+#         return partial(
+#             trio.run,
+#             partial(
+#                 mod.main,
+#                 **runtime_args,
+#             )
+#         )
+#     return import_n_partial_in_args_n_triorun

tests/devx/conftest.py

@@ -2,9 +2,11 @@
 `tractor.devx.*` tooling sub-pkg test space.
 
 '''
+from __future__ import annotations
 import time
 from typing import (
     Callable,
+    TYPE_CHECKING,
 )
 
 import pytest
@@ -16,7 +18,7 @@ from pexpect.spawnbase import SpawnBase
 from tractor._testing import (
     mk_cmd,
 )
-from tractor.devx._debug import (
+from tractor.devx.debug import (
     _pause_msg as _pause_msg,
     _crash_msg as _crash_msg,
     _repl_fail_msg as _repl_fail_msg,
@@ -26,14 +28,22 @@ from ..conftest import (
     _ci_env,
 )
 
+if TYPE_CHECKING:
+    from pexpect import pty_spawn
+
+
+# a fn that sub-instantiates a `pexpect.spawn()`
+# and returns it.
+type PexpectSpawner = Callable[[str], pty_spawn.spawn]
+
 
 @pytest.fixture
 def spawn(
-    start_method,
+    start_method: str,
     testdir: pytest.Pytester,
     reg_addr: tuple[str, int],
 
-) -> Callable[[str], None]:
+) -> PexpectSpawner:
     '''
     Use the `pexpect` module shipped via `testdir.spawn()` to
     run an `./examples/..` script by name.
@@ -59,7 +69,7 @@ def spawn(
     def _spawn(
         cmd: str,
         **mkcmd_kwargs,
-    ):
+    ) -> pty_spawn.spawn:
         unset_colors()
         return testdir.spawn(
             cmd=mk_cmd(
@@ -73,7 +83,7 @@ def spawn(
         )
 
     # such that test-dep can pass input script name.
-    return _spawn
+    return _spawn  # the `PexpectSpawner`, type alias.
 
 
 @pytest.fixture(
@@ -111,7 +121,7 @@ def ctlc(
         # XXX: disable pygments highlighting for auto-tests
         # since some envs (like actions CI) will struggle
         # the the added color-char encoding..
-        from tractor.devx._debug import TractorConfig
+        from tractor.devx.debug import TractorConfig
         TractorConfig.use_pygements = False
 
     yield use_ctlc

tests/devx/test_debugger.py

@@ -310,7 +310,6 @@ def test_subactor_breakpoint(
 
     assert in_prompt_msg(
         child, [
-        'MessagingError:',
         'RemoteActorError:',
          "('breakpoint_forever'",
          'bdb.BdbQuit',
@@ -528,7 +527,7 @@ def test_multi_daemon_subactors(
     # now the root actor won't clobber the bp_forever child
     # during it's first access to the debug lock, but will instead
     # wait for the lock to release, by the edge triggered
-    # ``devx._debug.Lock.no_remote_has_tty`` event before sending cancel messages
+    # ``devx.debug.Lock.no_remote_has_tty`` event before sending cancel messages
     # (via portals) to its underlings B)
 
     # at some point here there should have been some warning msg from

tests/devx/test_tooling.py

@@ -13,9 +13,13 @@ TODO:
   when debugging a problem inside the stack vs. in their app.
 
 '''
+from __future__ import annotations
 import os
 import signal
 import time
+from typing import (
+    TYPE_CHECKING,
+)
 
 from .conftest import (
     expect,
@@ -29,9 +33,12 @@ from pexpect.exceptions import (
     EOF,
 )
 
+if TYPE_CHECKING:
+    from ..conftest import PexpectSpawner
+
 
 def test_shield_pause(
-    spawn,
+    spawn: PexpectSpawner,
 ):
     '''
     Verify the `tractor.pause()/.post_mortem()` API works inside an
@@ -126,7 +133,7 @@ def test_shield_pause(
 
 
 def test_breakpoint_hook_restored(
-    spawn,
+    spawn: PexpectSpawner,
 ):
     '''
     Ensures our actor runtime sets a custom `breakpoint()` hook
@@ -140,16 +147,22 @@ def test_breakpoint_hook_restored(
     child = spawn('restore_builtin_breakpoint')
 
     child.expect(PROMPT)
-    assert_before(
-        child,
-        [
-            _pause_msg,
-            "<Task '__main__.main'",
-            "('root'",
-            "first bp, tractor hook set",
-        ]
-    )
-    child.sendline('c')
+    try:
+        assert_before(
+            child,
+            [
+                _pause_msg,
+                "<Task '__main__.main'",
+                "('root'",
+                "first bp, tractor hook set",
+            ]
+        )
+    # XXX if the above raises `AssertionError`, without sending
+    # the final 'continue' cmd to the REPL-active sub-process,
+    # we'll hang waiting for that pexpect instance to terminate..
+    finally:
+        child.sendline('c')
+
     child.expect(PROMPT)
     assert_before(
         child,

tests/ipc/__init__.py

@@ -0,0 +1,4 @@
+'''
+`tractor.ipc` subsystem(s)/unit testing suites.
+
+'''

tests/ipc/test_server.py

@@ -0,0 +1,72 @@
+'''
+High-level `.ipc._server` unit tests.
+
+'''
+from __future__ import annotations
+
+import pytest
+import trio
+from tractor import (
+    devx,
+    ipc,
+    log,
+)
+from tractor._testing.addr import (
+    get_rando_addr,
+)
+# TODO, use/check-roundtripping with some of these wrapper types?
+#
+# from .._addr import Address
+# from ._chan import Channel
+# from ._transport import MsgTransport
+# from ._uds import UDSAddress
+# from ._tcp import TCPAddress
+
+
+@pytest.mark.parametrize(
+    '_tpt_proto',
+    ['uds', 'tcp']
+)
+def test_basic_ipc_server(
+    _tpt_proto: str,
+    debug_mode: bool,
+    loglevel: str,
+):
+
+    # so we see the socket-listener reporting on console
+    log.get_console_log("INFO")
+
+    rando_addr: tuple = get_rando_addr(
+        tpt_proto=_tpt_proto,
+    )
+    async def main():
+        async with ipc._server.open_ipc_server() as server:
+
+            assert (
+                server._parent_tn
+                and
+                server._parent_tn is server._stream_handler_tn
+            )
+            assert server._no_more_peers.is_set()
+
+            eps: list[ipc.IPCEndpoint] = await server.listen_on(
+                accept_addrs=[rando_addr],
+                stream_handler_nursery=None,
+            )
+            assert (
+                len(eps) == 1
+                and
+                (ep := eps[0])._listener
+                and
+                not ep.peer_tpts
+            )
+
+            server._parent_tn.cancel_scope.cancel()
+
+        # !TODO! actually make a bg-task connection from a client
+        # using `ipc._chan._connect_chan()`
+
+    with devx.maybe_open_crash_handler(
+        pdb=debug_mode,
+    ):
+        trio.run(main)

tests/test_advanced_faults.py

@@ -10,6 +10,9 @@ import pytest
 from _pytest.pathlib import import_path
 import trio
 import tractor
+from tractor import (
+    TransportClosed,
+)
 from tractor._testing import (
     examples_dir,
     break_ipc,
@@ -74,6 +77,7 @@ def test_ipc_channel_break_during_stream(
     spawn_backend: str,
     ipc_break: dict|None,
     pre_aclose_msgstream: bool,
+    tpt_proto: str,
 ):
     '''
     Ensure we can have an IPC channel break its connection during
@@ -91,7 +95,7 @@ def test_ipc_channel_break_during_stream(
         # non-`trio` spawners should never hit the hang condition that
         # requires the user to do ctl-c to cancel the actor tree.
         # expect_final_exc = trio.ClosedResourceError
-        expect_final_exc = tractor.TransportClosed
+        expect_final_exc = TransportClosed
 
     mod: ModuleType = import_path(
         examples_dir() / 'advanced_faults'
@@ -104,6 +108,8 @@ def test_ipc_channel_break_during_stream(
     # period" wherein the user eventually hits ctl-c to kill the
     # root-actor tree.
     expect_final_exc: BaseException = KeyboardInterrupt
+    expect_final_cause: BaseException|None = None
+
     if (
         # only expect EoC if trans is broken on the child side,
         ipc_break['break_child_ipc_after'] is not False
@@ -138,6 +144,9 @@ def test_ipc_channel_break_during_stream(
         # a user sending ctl-c by raising a KBI.
         if pre_aclose_msgstream:
             expect_final_exc = KeyboardInterrupt
+            if tpt_proto == 'uds':
+                expect_final_exc = TransportClosed
+                expect_final_cause = trio.BrokenResourceError
 
             # XXX OLD XXX
             # if child calls `MsgStream.aclose()` then expect EoC.
@@ -157,6 +166,10 @@ def test_ipc_channel_break_during_stream(
         if pre_aclose_msgstream:
             expect_final_exc = KeyboardInterrupt
 
+            if tpt_proto == 'uds':
+                expect_final_exc = TransportClosed
+                expect_final_cause = trio.BrokenResourceError
+
     # NOTE when the parent IPC side dies (even if the child does as well
     # but the child fails BEFORE the parent) we always expect the
     # IPC layer to raise a closed-resource, NEVER do we expect
@@ -169,8 +182,8 @@ def test_ipc_channel_break_during_stream(
         and
         ipc_break['break_child_ipc_after'] is False
     ):
-        # expect_final_exc = trio.ClosedResourceError
         expect_final_exc = tractor.TransportClosed
+        expect_final_cause = trio.ClosedResourceError
 
     # BOTH but, PARENT breaks FIRST
     elif (
@@ -181,8 +194,8 @@ def test_ipc_channel_break_during_stream(
             ipc_break['break_parent_ipc_after']
         )
     ):
-        # expect_final_exc = trio.ClosedResourceError
         expect_final_exc = tractor.TransportClosed
+        expect_final_cause = trio.ClosedResourceError
 
     with pytest.raises(
         expected_exception=(
@@ -198,6 +211,7 @@ def test_ipc_channel_break_during_stream(
                     start_method=spawn_backend,
                     loglevel=loglevel,
                     pre_close=pre_aclose_msgstream,
+                    tpt_proto=tpt_proto,
                     **ipc_break,
                 )
             )
@@ -220,10 +234,15 @@ def test_ipc_channel_break_during_stream(
                 )
             cause: Exception = tc.__cause__
             assert (
-                type(cause) is trio.ClosedResourceError
-                and
-                cause.args[0] == 'another task closed this fd'
+                # type(cause) is trio.ClosedResourceError
+                type(cause) is expect_final_cause
+
+                # TODO, should we expect a certain exc-message (per
+                # tpt) as well??
+                # and
+                # cause.args[0] == 'another task closed this fd'
             )
+
             raise
 
     # get raw instance from pytest wrapper

tests/test_context_stream_semantics.py

@@ -252,7 +252,7 @@ def test_simple_context(
             pass
         except BaseExceptionGroup as beg:
             # XXX: on windows it seems we may have to expect the group error
-            from tractor._exceptions import is_multi_cancelled
+            from tractor.trionics import is_multi_cancelled
             assert is_multi_cancelled(beg)
     else:
         trio.run(main)

tests/test_discovery.py

@@ -7,7 +7,9 @@ import platform
 from functools import partial
 import itertools
 
+import psutil
 import pytest
+import subprocess
 import tractor
 from tractor._testing import tractor_test
 import trio
@@ -152,13 +154,23 @@ async def unpack_reg(actor_or_portal):
 async def spawn_and_check_registry(
     reg_addr: tuple,
     use_signal: bool,
+    debug_mode: bool = False,
     remote_arbiter: bool = False,
     with_streaming: bool = False,
+    maybe_daemon: tuple[
+        subprocess.Popen,
+        psutil.Process,
+    ]|None = None,
 
 ) -> None:
 
+    if maybe_daemon:
+        popen, proc = maybe_daemon
+        # breakpoint()
+
     async with tractor.open_root_actor(
         registry_addrs=[reg_addr],
+        debug_mode=debug_mode,
     ):
         async with tractor.get_registry(reg_addr) as portal:
             # runtime needs to be up to call this
@@ -176,11 +188,11 @@ async def spawn_and_check_registry(
                 extra = 2  # local root actor + remote arbiter
 
             # ensure current actor is registered
-            registry = await get_reg()
+            registry: dict = await get_reg()
             assert actor.uid in registry
 
             try:
-                async with tractor.open_nursery() as n:
+                async with tractor.open_nursery() as an:
                     async with trio.open_nursery(
                         strict_exception_groups=False,
                     ) as trion:
@@ -189,17 +201,17 @@ async def spawn_and_check_registry(
                         for i in range(3):
                             name = f'a{i}'
                             if with_streaming:
-                                portals[name] = await n.start_actor(
+                                portals[name] = await an.start_actor(
                                     name=name, enable_modules=[__name__])
 
                             else:  # no streaming
-                                portals[name] = await n.run_in_actor(
+                                portals[name] = await an.run_in_actor(
                                     trio.sleep_forever, name=name)
 
                         # wait on last actor to come up
                         async with tractor.wait_for_actor(name):
                             registry = await get_reg()
-                            for uid in n._children:
+                            for uid in an._children:
                                 assert uid in registry
 
                         assert len(portals) + extra == len(registry)
@@ -232,6 +244,7 @@ async def spawn_and_check_registry(
 @pytest.mark.parametrize('use_signal', [False, True])
 @pytest.mark.parametrize('with_streaming', [False, True])
 def test_subactors_unregister_on_cancel(
+    debug_mode: bool,
     start_method,
     use_signal,
     reg_addr,
@@ -248,6 +261,7 @@ def test_subactors_unregister_on_cancel(
                 spawn_and_check_registry,
                 reg_addr,
                 use_signal,
+                debug_mode=debug_mode,
                 remote_arbiter=False,
                 with_streaming=with_streaming,
             ),
@@ -257,7 +271,8 @@ def test_subactors_unregister_on_cancel(
 @pytest.mark.parametrize('use_signal', [False, True])
 @pytest.mark.parametrize('with_streaming', [False, True])
 def test_subactors_unregister_on_cancel_remote_daemon(
-    daemon,
+    daemon: subprocess.Popen,
+    debug_mode: bool,
     start_method,
     use_signal,
     reg_addr,
@@ -273,8 +288,13 @@ def test_subactors_unregister_on_cancel_remote_daemon(
                 spawn_and_check_registry,
                 reg_addr,
                 use_signal,
+                debug_mode=debug_mode,
                 remote_arbiter=True,
                 with_streaming=with_streaming,
+                maybe_daemon=(
+                    daemon,
+                    psutil.Process(daemon.pid)
+                ),
             ),
         )
 
@@ -373,7 +393,7 @@ def test_close_channel_explicit(
 
 @pytest.mark.parametrize('use_signal', [False, True])
 def test_close_channel_explicit_remote_arbiter(
-    daemon,
+    daemon: subprocess.Popen,
     start_method,
     use_signal,
     reg_addr,

tests/test_infected_asyncio.py

@@ -889,7 +889,7 @@ async def manage_file(
 
         # NOTE: turns out you don't even need to sched an aio task
         # since the original issue, even though seemingly was due to
-        # the guest-run being abandoned + a `._debug.pause()` inside
+        # the guest-run being abandoned + a `.debug.pause()` inside
         # `._runtime._async_main()` (which was originally trying to
         # debug the `.lifetime_stack` not closing), IS NOT actually
         # the core issue?
@@ -1101,7 +1101,7 @@ def test_sigint_closes_lifetime_stack(
 #    => completed using `.bestow_portal(task)` inside
 #     `.to_asyncio._run_asyncio_task()` right?
 #   -[ ] translation func to get from `asyncio` task calling to 
-#     `._debug.wait_for_parent_stdin_hijack()` which does root
+#     `.debug.wait_for_parent_stdin_hijack()` which does root
 #     call to do TTY locking.
 #
 def test_sync_breakpoint():

tests/test_inter_peer_cancellation.py

@@ -410,7 +410,6 @@ def test_peer_canceller(
     '''
     async def main():
         async with tractor.open_nursery(
-            # NOTE: to halt the peer tasks on ctxc, uncomment this.
             debug_mode=debug_mode,
         ) as an:
             canceller: Portal = await an.start_actor(

tests/test_remote_exc_relay.py

@@ -0,0 +1,237 @@
+'''
+Special case testing for issues not (dis)covered in the primary
+`Context` related functional/scenario suites.
+
+**NOTE: this mod is a WIP** space for handling
+odd/rare/undiscovered/not-yet-revealed faults which either
+loudly (ideal case) breakl our supervision protocol
+or (worst case) result in distributed sys hangs.
+
+Suites here further try to clarify (if [partially] ill-defined) and
+verify our edge case semantics for inter-actor-relayed-exceptions
+including,
+
+- lowlevel: what remote obj-data is interchanged for IPC and what is
+  native-obj form is expected from unpacking in the the new
+  mem-domain.
+
+- which kinds of `RemoteActorError` (and its derivs) are expected by which
+  (types of) peers (parent, child, sibling, etc) with what
+  particular meta-data set such as,
+
+  - `.src_uid`: the original (maybe) peer who raised.
+  - `.relay_uid`: the next-hop-peer who sent it.
+  - `.relay_path`: the sequence of peer actor hops.
+  - `.is_inception`: a predicate that denotes multi-hop remote errors.
+
+- when should `ExceptionGroup`s be relayed from a particular
+  remote endpoint, they should never be caused by implicit `._rpc`
+  nursery machinery!
+
+- various special `trio` edge cases around its cancellation semantics
+  and how we (currently) leverage `trio.Cancelled` as a signal for
+  whether a `Context` task should raise `ContextCancelled` (ctx).
+
+'''
+import pytest
+import trio
+import tractor
+from tractor import (  # typing
+    ActorNursery,
+    Portal,
+    Context,
+    ContextCancelled,
+)
+
+
+@tractor.context
+async def sleep_n_chkpt_in_finally(
+    ctx: Context,
+    sleep_n_raise: bool,
+
+    chld_raise_delay: float,
+    chld_finally_delay: float,
+
+    rent_cancels: bool,
+    rent_ctxc_delay: float,
+
+    expect_exc: str|None = None,
+
+) -> None:
+    '''
+    Sync, open a tn, then wait for cancel, run a chkpt inside
+    the user's `finally:` teardown.
+
+    This covers a footgun case that `trio` core doesn't seem to care about
+    wherein an exc can be masked by a `trio.Cancelled` raised inside a tn emedded
+    `finally:`.
+
+    Also see `test_trioisms::test_acm_embedded_nursery_propagates_enter_err`
+    for the down and gritty details.
+
+    Since a `@context` endpoint fn can also contain code like this,
+    **and** bc we currently have no easy way other then
+    `trio.Cancelled` to signal cancellation on each side of an IPC `Context`,
+    the footgun issue can compound itself as demonstrated in this suite..
+
+    Here are some edge cases codified with our WIP "sclang" syntax
+    (note the parent(rent)/child(chld) naming here is just
+    pragmatism, generally these most of these cases can occurr
+    regardless of the distributed-task's supervision hiearchy),
+
+    - rent c)=> chld.raises-then-taskc-in-finally
+     |_ chld's body raises an `exc: BaseException`.
+      _ in its `finally:` block it runs a chkpoint
+        which raises a taskc (`trio.Cancelled`) which
+        masks `exc` instead raising taskc up to the first tn.
+      _ the embedded/chld tn captures the masking taskc and then
+        raises it up to the ._rpc-ep-tn instead of `exc`.
+      _ the rent thinks the child ctxc-ed instead of errored..
+
+    '''
+    await ctx.started()
+
+    if expect_exc:
+        expect_exc: BaseException = tractor._exceptions.get_err_type(
+            type_name=expect_exc,
+        )
+
+    berr: BaseException|None = None
+    try:
+        if not sleep_n_raise:
+            await trio.sleep_forever()
+        elif sleep_n_raise:
+
+            # XXX this sleep is less then the sleep the parent
+            # does before calling `ctx.cancel()`
+            await trio.sleep(chld_raise_delay)
+
+            # XXX this will be masked by a taskc raised in
+            # the `finally:` if this fn doesn't terminate
+            # before any ctxc-req arrives AND a checkpoint is hit
+            # in that `finally:`.
+            raise RuntimeError('my app krurshed..')
+
+    except BaseException as _berr:
+        berr = _berr
+
+        # TODO: it'd sure be nice to be able to inject our own
+        # `ContextCancelled` here instead of of `trio.Cancelled`
+        # so that our runtime can expect it and this "user code"
+        # would be able to tell the diff between a generic trio
+        # cancel and a tractor runtime-IPC cancel.
+        if expect_exc:
+            if not isinstance(
+                berr,
+                expect_exc,
+            ):
+                raise ValueError(
+                    f'Unexpected exc type ??\n'
+                    f'{berr!r}\n'
+                    f'\n'
+                    f'Expected a {expect_exc!r}\n'
+                )
+
+        raise berr
+
+    # simulate what user code might try even though
+    # it's a known boo-boo..
+    finally:
+        # maybe wait for rent ctxc to arrive
+        with trio.CancelScope(shield=True):
+            await trio.sleep(chld_finally_delay)
+
+        # !!XXX this will raise `trio.Cancelled` which
+        # will mask the RTE from above!!!
+        #
+        # YES, it's the same case as our extant
+        # `test_trioisms::test_acm_embedded_nursery_propagates_enter_err`
+        try:
+            await trio.lowlevel.checkpoint()
+        except trio.Cancelled as taskc:
+            if (scope_err := taskc.__context__):
+                print(
+                    f'XXX MASKED REMOTE ERROR XXX\n'
+                    f'ENDPOINT exception -> {scope_err!r}\n'
+                    f'will be masked by -> {taskc!r}\n'
+                )
+                # await tractor.pause(shield=True)
+
+            raise taskc
+
+
+@pytest.mark.parametrize(
+    'chld_callspec',
+    [
+        dict(
+            sleep_n_raise=None,
+            chld_raise_delay=0.1,
+            chld_finally_delay=0.1,
+            expect_exc='Cancelled',
+            rent_cancels=True,
+            rent_ctxc_delay=0.1,
+        ),
+        dict(
+            sleep_n_raise='RuntimeError',
+            chld_raise_delay=0.1,
+            chld_finally_delay=1,
+            expect_exc='RuntimeError',
+            rent_cancels=False,
+            rent_ctxc_delay=0.1,
+        ),
+    ],
+    ids=lambda item: f'chld_callspec={item!r}'
+)
+def test_unmasked_remote_exc(
+    debug_mode: bool,
+    chld_callspec: dict,
+    tpt_proto: str,
+):
+    expect_exc_str: str|None = chld_callspec['sleep_n_raise']
+    rent_ctxc_delay: float|None = chld_callspec['rent_ctxc_delay']
+    async def main():
+        an: ActorNursery
+        async with tractor.open_nursery(
+            debug_mode=debug_mode,
+            enable_transports=[tpt_proto],
+        ) as an:
+            ptl: Portal = await an.start_actor(
+                'cancellee',
+                enable_modules=[__name__],
+            )
+            ctx: Context
+            async with (
+                ptl.open_context(
+                    sleep_n_chkpt_in_finally,
+                    **chld_callspec,
+                ) as (ctx, sent),
+            ):
+                assert not sent
+                await trio.sleep(rent_ctxc_delay)
+                await ctx.cancel()
+
+                # recv error or result from chld
+                ctxc: ContextCancelled = await ctx.wait_for_result()
+                assert (
+                    ctxc is ctx.outcome
+                    and
+                    isinstance(ctxc, ContextCancelled)
+                )
+
+            # always graceful terminate the sub in non-error cases
+            await an.cancel()
+
+    if expect_exc_str:
+        expect_exc: BaseException = tractor._exceptions.get_err_type(
+            type_name=expect_exc_str,
+        )
+        with pytest.raises(
+            expected_exception=tractor.RemoteActorError,
+        ) as excinfo:
+            trio.run(main)
+
+        rae = excinfo.value
+        assert expect_exc == rae.boxed_type
+
+    else:
+        trio.run(main)

tests/test_resource_cache.py

@@ -100,16 +100,29 @@ async def streamer(
 @acm
 async def open_stream() -> Awaitable[tractor.MsgStream]:
 
-    async with tractor.open_nursery() as tn:
-        portal = await tn.start_actor('streamer', enable_modules=[__name__])
-        async with (
-            portal.open_context(streamer) as (ctx, first),
-            ctx.open_stream() as stream,
-        ):
-            yield stream
+    try:
+        async with tractor.open_nursery() as an:
+            portal = await an.start_actor(
+                'streamer',
+                enable_modules=[__name__],
+            )
+            async with (
+                portal.open_context(streamer) as (ctx, first),
+                ctx.open_stream() as stream,
+            ):
+                yield stream
 
-        await portal.cancel_actor()
-    print('CANCELLED STREAMER')
+            print('Cancelling streamer')
+            await portal.cancel_actor()
+            print('Cancelled streamer')
+
+    except Exception as err:
+        print(
+            f'`open_stream()` errored?\n'
+            f'{err!r}\n'
+        )
+        await tractor.pause(shield=True)
+        raise err
 
 
 @acm
@@ -132,19 +145,28 @@ async def maybe_open_stream(taskname: str):
             yield stream
 
 
-def test_open_local_sub_to_stream():
+def test_open_local_sub_to_stream(
+    debug_mode: bool,
+):
     '''
     Verify a single inter-actor stream can can be fanned-out shared to
-    N local tasks using ``trionics.maybe_open_context():``.
+    N local tasks using `trionics.maybe_open_context()`.
 
     '''
-    timeout: float = 3.6 if platform.system() != "Windows" else 10
+    timeout: float = 3.6
+    if platform.system() == "Windows":
+        timeout: float = 10
+
+    if debug_mode:
+        timeout = 999
 
     async def main():
 
         full = list(range(1000))
 
         async def get_sub_and_pull(taskname: str):
+
+            stream: tractor.MsgStream
             async with (
                 maybe_open_stream(taskname) as stream,
             ):
@@ -165,17 +187,27 @@ def test_open_local_sub_to_stream():
                 assert set(seq).issubset(set(full))
             print(f'{taskname} finished')
 
-        with trio.fail_after(timeout):
+        with trio.fail_after(timeout) as cs:
             # TODO: turns out this isn't multi-task entrant XD
             # We probably need an indepotent entry semantic?
-            async with tractor.open_root_actor():
+            async with tractor.open_root_actor(
+                debug_mode=debug_mode,
+            ):
                 async with (
-                    trio.open_nursery() as nurse,
+                    trio.open_nursery() as tn,
                 ):
                     for i in range(10):
-                        nurse.start_soon(get_sub_and_pull, f'task_{i}')
+                        tn.start_soon(
+                            get_sub_and_pull,
+                            f'task_{i}',
+                        )
                         await trio.sleep(0.001)
 
                 print('all consumer tasks finished')
 
+        if cs.cancelled_caught:
+            pytest.fail(
+                'Should NOT time out in `open_root_actor()` ?'
+            )
+
     trio.run(main)

tests/test_root_runtime.py

@@ -83,3 +83,26 @@ def test_implicit_root_via_first_nursery(
             assert tractor.current_actor().aid.name == 'root'
 
     trio.run(main)
+
+
+def test_runtime_vars_unset(
+    reg_addr: tuple,
+    debug_mode: bool
+):
+    '''
+    Ensure any `._state._runtime_vars` are restored to default values
+    after the root actor-runtime exits!
+
+    '''
+    assert not tractor._state._runtime_vars['_debug_mode']
+    async def main():
+        assert not tractor._state._runtime_vars['_debug_mode']
+        async with tractor.open_nursery(
+            debug_mode=True,
+        ):
+            assert tractor._state._runtime_vars['_debug_mode']
+
+        # after runtime closure, should be reverted!
+        assert not tractor._state._runtime_vars['_debug_mode']
+
+    trio.run(main)

tests/test_trioisms.py

@@ -112,55 +112,11 @@ def test_acm_embedded_nursery_propagates_enter_err(
     '''
     import tractor
 
-    @acm
-    async def maybe_raise_from_masking_exc(
-        tn: trio.Nursery,
-        unmask_from: BaseException|None = trio.Cancelled
-
-        # TODO, maybe offer a collection?
-        # unmask_from: set[BaseException] = {
-        #     trio.Cancelled,
-        # },
-    ):
-        if not unmask_from:
-            yield
-            return
-
-        try:
-            yield
-        except* unmask_from as be_eg:
-
-            # TODO, if we offer `unmask_from: set`
-            # for masker_exc_type in unmask_from:
-
-            matches, rest = be_eg.split(unmask_from)
-            if not matches:
-                raise
-
-            for exc_match in be_eg.exceptions:
-                if (
-                    (exc_ctx := exc_match.__context__)
-                    and
-                    type(exc_ctx) not in {
-                        # trio.Cancelled,  # always by default?
-                        unmask_from,
-                    }
-                ):
-                    exc_ctx.add_note(
-                        f'\n'
-                        f'WARNING: the above error was masked by a {unmask_from!r} !?!\n'
-                        f'Are you always cancelling? Say from a `finally:` ?\n\n'
-
-                        f'{tn!r}'
-                    )
-                    raise exc_ctx from exc_match
-
-
     @acm
     async def wraps_tn_that_always_cancels():
         async with (
             trio.open_nursery() as tn,
-            maybe_raise_from_masking_exc(
+            tractor.trionics.maybe_raise_from_masking_exc(
                 tn=tn,
                 unmask_from=(
                     trio.Cancelled
@@ -180,7 +136,8 @@ def test_acm_embedded_nursery_propagates_enter_err(
         with tractor.devx.maybe_open_crash_handler(
             pdb=debug_mode,
         ) as bxerr:
-            assert not bxerr.value
+            if bxerr:
+                assert not bxerr.value
 
             async with (
                 wraps_tn_that_always_cancels() as tn,
@@ -201,3 +158,60 @@ def test_acm_embedded_nursery_propagates_enter_err(
     assert_eg, rest_eg = eg.split(AssertionError)
 
     assert len(assert_eg.exceptions) == 1
+
+
+
+def test_gatherctxs_with_memchan_breaks_multicancelled(
+    debug_mode: bool,
+):
+    '''
+    Demo how a using an `async with sndchan` inside a `.trionics.gather_contexts()` task
+    will break a strict-eg-tn's multi-cancelled absorption..
+
+    '''
+    from tractor import (
+        trionics,
+    )
+
+    @acm
+    async def open_memchan() -> trio.abc.ReceiveChannel:
+
+        task: trio.Task = trio.lowlevel.current_task()
+        print(
+            f'Opening {task!r}\n'
+        )
+
+        # 1 to force eager sending
+        send, recv = trio.open_memory_channel(16)
+
+        try:
+            async with send:
+                yield recv
+        finally:
+            print(
+                f'Closed {task!r}\n'
+            )
+
+
+    async def main():
+        async with (
+            # XXX should ensure ONLY the KBI
+            # is relayed upward
+            trionics.collapse_eg(),
+            trio.open_nursery(
+                # strict_exception_groups=False,
+            ), # as tn,
+
+            trionics.gather_contexts([
+                open_memchan(),
+                open_memchan(),
+            ]) as recv_chans,
+        ):
+            assert len(recv_chans) == 2
+
+            await trio.sleep(1)
+            raise KeyboardInterrupt
+            # tn.cancel_scope.cancel()
+
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(main)

tractor/_context.py

@@ -101,6 +101,9 @@ from ._state import (
     debug_mode,
     _ctxvar_Context,
 )
+from .trionics import (
+    collapse_eg,
+)
 # ------ - ------
 if TYPE_CHECKING:
     from ._portal import Portal
@@ -292,7 +295,7 @@ class Context:
     # - `._runtime._invoke()` will check this flag before engaging
     #   the crash handler REPL in such cases where the "callee"
     #   raises the cancellation,
-    # - `.devx._debug.lock_stdio_for_peer()` will set it to `False` if
+    # - `.devx.debug.lock_stdio_for_peer()` will set it to `False` if
     #   the global tty-lock has been configured to filter out some
     #   actors from being able to acquire the debugger lock.
     _enter_debugger_on_cancel: bool = True
@@ -940,7 +943,7 @@ class Context:
         self.cancel_called = True
 
         header: str = (
-            f'Cancelling ctx from {side.upper()}-side\n'
+            f'Cancelling ctx from {side!r}-side\n'
         )
         reminfo: str = (
             # ' =>\n'
@@ -948,7 +951,7 @@ class Context:
             f'\n'
             f'c)=> {self.chan.uid}\n'
             f'   |_[{self.dst_maddr}\n'
-            f'     >>{self.repr_rpc}\n'
+            f'     >> {self.repr_rpc}\n'
             # f'    >> {self._nsf}() -> {codec}[dict]:\n\n'
             # TODO: pull msg-type from spec re #320
         )
@@ -1251,8 +1254,8 @@ class Context:
 
             # ?XXX, should already be set in `._deliver_msg()` right?
             if self._outcome_msg is not Unresolved:
-                # from .devx import _debug
-                # await _debug.pause()
+                # from .devx import debug
+                # await debug.pause()
                 assert self._outcome_msg is outcome_msg
             else:
                 self._outcome_msg = outcome_msg
@@ -2023,10 +2026,8 @@ async def open_context_from_portal(
     ctxc_from_callee: ContextCancelled|None = None
     try:
         async with (
-            trio.open_nursery(
-                strict_exception_groups=False,
-            ) as tn,
-
+            collapse_eg(),
+            trio.open_nursery() as tn,
             msgops.maybe_limit_plds(
                 ctx=ctx,
                 spec=ctx_meta.get('pld_spec'),
@@ -2187,7 +2188,7 @@ async def open_context_from_portal(
         #   debugging the tractor-runtime itself using it's
         #   own `.devx.` tooling!
         # 
-        # await _debug.pause()
+        # await debug.pause()
 
         # CASE 2: context was cancelled by local task calling
         # `.cancel()`, we don't raise and the exit block should
@@ -2254,7 +2255,7 @@ async def open_context_from_portal(
         # NOTE: `Context.cancel()` is conversely NEVER CALLED in
         # the `ContextCancelled` "self cancellation absorbed" case
         # handled in the block above ^^^ !!
-        # await _debug.pause()
+        # await debug.pause()
         # log.cancel(
         match scope_err:
             case trio.Cancelled:
@@ -2269,11 +2270,11 @@ async def open_context_from_portal(
         )
 
         if debug_mode():
-            # async with _debug.acquire_debug_lock(portal.actor.uid):
+            # async with debug.acquire_debug_lock(portal.actor.uid):
             #     pass
             # TODO: factor ^ into below for non-root cases?
             #
-            from .devx._debug import maybe_wait_for_debugger
+            from .devx.debug import maybe_wait_for_debugger
             was_acquired: bool = await maybe_wait_for_debugger(
                 # header_msg=(
                 #     'Delaying `ctx.cancel()` until debug lock '
@@ -2336,8 +2337,8 @@ async def open_context_from_portal(
                 raise
 
             # yes this worx!
-            # from .devx import _debug
-            # await _debug.pause()
+            # from .devx import debug
+            # await debug.pause()
 
             # an exception type boxed in a `RemoteActorError`
             # is returned (meaning it was obvi not raised)
@@ -2372,7 +2373,7 @@ async def open_context_from_portal(
         # where the root is waiting on the lock to clear but the
         # child has already cleared it and clobbered IPC.
         if debug_mode():
-            from .devx._debug import maybe_wait_for_debugger
+            from .devx.debug import maybe_wait_for_debugger
             await maybe_wait_for_debugger()
 
         # though it should be impossible for any tasks

tractor/_discovery.py

@@ -28,7 +28,10 @@ from typing import (
 from contextlib import asynccontextmanager as acm
 
 from tractor.log import get_logger
-from .trionics import gather_contexts
+from .trionics import (
+    gather_contexts,
+    collapse_eg,
+)
 from .ipc import _connect_chan, Channel
 from ._addr import (
     UnwrappedAddress,
@@ -48,6 +51,7 @@ from ._state import (
 
 if TYPE_CHECKING:
     from ._runtime import Actor
+    from .ipc._server import IPCServer
 
 
 log = get_logger(__name__)
@@ -79,7 +83,7 @@ async def get_registry(
         )
     else:
         # TODO: try to look pre-existing connection from
-        # `Server._peers` and use it instead?
+        # `IPCServer._peers` and use it instead?
         async with (
             _connect_chan(addr) as chan,
             open_portal(chan) as regstr_ptl,
@@ -87,7 +91,6 @@ async def get_registry(
             yield regstr_ptl
 
 
-
 @acm
 async def get_root(
     **kwargs,
@@ -111,23 +114,18 @@ def get_peer_by_name(
 ) -> list[Channel]|None:  # at least 1
     '''
     Scan for an existing connection (set) to a named actor
-    and return any channels from `Server._peers: dict`.
+    and return any channels from `IPCServer._peers: dict`.
 
     This is an optimization method over querying the registrar for
     the same info.
 
     '''
     actor: Actor = current_actor()
-    to_scan: dict[tuple, list[Channel]] = actor.ipc_server._peers.copy()
-
-    # TODO: is this ever needed? creates a duplicate channel on actor._peers
-    # when multiple find_actor calls are made to same actor from a single ctx
-    # which causes actor exit to hang waiting forever on
-    # `actor._no_more_peers.wait()` in `_runtime.async_main`
-
-    # pchan: Channel|None = actor._parent_chan
-    # if pchan and pchan.uid not in to_scan:
-    #     to_scan[pchan.uid].append(pchan)
+    server: IPCServer = actor.ipc_server
+    to_scan: dict[tuple, list[Channel]] = server._peers.copy()
+    pchan: Channel|None = actor._parent_chan
+    if pchan:
+        to_scan[pchan.uid].append(pchan)
 
     for aid, chans in to_scan.items():
         _, peer_name = aid
@@ -253,9 +251,12 @@ async def find_actor(
         for addr in registry_addrs
     )
     portals: list[Portal]
-    async with gather_contexts(
-        mngrs=maybe_portals,
-    ) as portals:
+    async with (
+        collapse_eg(),
+        gather_contexts(
+            mngrs=maybe_portals,
+        ) as portals,
+    ):
         # log.runtime(
         #     'Gathered portals:\n'
         #     f'{portals}'

tractor/_entry.py

@@ -22,7 +22,6 @@ from __future__ import annotations
 from functools import partial
 import multiprocessing as mp
 import os
-import textwrap
 from typing import (
     Any,
     TYPE_CHECKING,
@@ -35,7 +34,10 @@ from .log import (
     get_logger,
 )
 from . import _state
-from .devx import _debug
+from .devx import (
+    _frame_stack,
+    pformat,
+)
 from .to_asyncio import run_as_asyncio_guest
 from ._addr import UnwrappedAddress
 from ._runtime import (
@@ -103,107 +105,6 @@ def _mp_main(
         )
 
 
-# TODO: move this func to some kinda `.devx._conc_lang.py` eventually
-# as we work out our multi-domain state-flow-syntax!
-def nest_from_op(
-    input_op: str,
-    #
-    # ?TODO? an idea for a syntax to the state of concurrent systems
-    # as a "3-domain" (execution, scope, storage) model and using
-    # a minimal ascii/utf-8 operator-set.
-    #
-    # try not to take any of this seriously yet XD
-    #
-    # > is a "play operator" indicating (CPU bound)
-    #   exec/work/ops required at the "lowest level computing"
-    #
-    # execution primititves (tasks, threads, actors..) denote their
-    # lifetime with '(' and ')' since parentheses normally are used
-    # in many langs to denote function calls.
-    #
-    # starting = (
-    # >(  opening/starting; beginning of the thread-of-exec (toe?)
-    # (>  opened/started,  (finished spawning toe)
-    # |_<Task: blah blah..>  repr of toe, in py these look like <objs>
-    #
-    # >) closing/exiting/stopping,
-    # )> closed/exited/stopped,
-    # |_<Task: blah blah..>
-    #   [OR <), )< ?? ]
-    #
-    # ending = )
-    # >c) cancelling to close/exit
-    # c)> cancelled (caused close), OR?
-    #  |_<Actor: ..>
-    #   OR maybe "<c)" which better indicates the cancel being
-    #   "delivered/returned" / returned" to LHS?
-    #
-    # >x)  erroring to eventuall exit
-    # x)>  errored and terminated
-    #  |_<Actor: ...>
-    #
-    # scopes: supers/nurseries, IPC-ctxs, sessions, perms, etc.
-    # >{  opening
-    # {>  opened
-    # }>  closed
-    # >}  closing
-    #
-    # storage: like queues, shm-buffers, files, etc..
-    # >[  opening
-    # [>  opened
-    #  |_<FileObj: ..>
-    #
-    # >]  closing
-    # ]>  closed
-
-    # IPC ops: channels, transports, msging
-    # =>  req msg
-    # <=  resp msg
-    # <=> 2-way streaming (of msgs)
-    # <-  recv 1 msg
-    # ->  send 1 msg
-    #
-    # TODO: still not sure on R/L-HS approach..?
-    # =>(  send-req to exec start (task, actor, thread..)
-    # (<=  recv-req to ^
-    #
-    # (<=  recv-req ^
-    # <=(  recv-resp opened remote exec primitive
-    # <=)  recv-resp closed
-    #
-    # )<=c req to stop due to cancel
-    # c=>) req to stop due to cancel
-    #
-    # =>{  recv-req to open
-    # <={  send-status that it closed
-
-    tree_str: str,
-
-    # NOTE: so move back-from-the-left of the `input_op` by
-    # this amount.
-    back_from_op: int = 0,
-) -> str:
-    '''
-    Depth-increment the input (presumably hierarchy/supervision)
-    input "tree string" below the provided `input_op` execution
-    operator, so injecting a `"\n|_{input_op}\n"`and indenting the
-    `tree_str` to nest content aligned with the ops last char.
-
-    '''
-    return (
-        f'{input_op}\n'
-        +
-        textwrap.indent(
-            tree_str,
-            prefix=(
-                len(input_op)
-                -
-                (back_from_op + 1)
-            ) * ' ',
-        )
-    )
-
-
 def _trio_main(
     actor: Actor,
     *,
@@ -215,7 +116,7 @@ def _trio_main(
     Entry point for a `trio_run_in_process` subactor.
 
     '''
-    _debug.hide_runtime_frames()
+    _frame_stack.hide_runtime_frames()
 
     _state._current_actor = actor
     trio_main = partial(
@@ -236,7 +137,7 @@ def _trio_main(
         log.info(
             'Starting new `trio` subactor:\n'
             +
-            nest_from_op(
+            pformat.nest_from_op(
                 input_op='>(',  # see syntax ideas above
                 tree_str=actor_info,
                 back_from_op=2,  # since "complete"
@@ -246,7 +147,7 @@ def _trio_main(
     exit_status: str = (
         'Subactor exited\n'
         +
-        nest_from_op(
+        pformat.nest_from_op(
             input_op=')>',  # like a "closed-to-play"-icon from super perspective
             tree_str=actor_info,
             back_from_op=1,
@@ -264,7 +165,7 @@ def _trio_main(
         exit_status: str = (
             'Actor received KBI (aka an OS-cancel)\n'
             +
-            nest_from_op(
+            pformat.nest_from_op(
                 input_op='c)>',  # closed due to cancel (see above)
                 tree_str=actor_info,
             )
@@ -274,7 +175,7 @@ def _trio_main(
         exit_status: str = (
             'Main actor task exited due to crash?\n'
             +
-            nest_from_op(
+            pformat.nest_from_op(
                 input_op='x)>',  # closed by error
                 tree_str=actor_info,
             )

tractor/_exceptions.py

@@ -1246,55 +1246,6 @@ def unpack_error(
     return exc
 
 
-def is_multi_cancelled(
-    exc: BaseException|BaseExceptionGroup,
-
-    ignore_nested: set[BaseException] = set(),
-
-) -> bool|BaseExceptionGroup:
-    '''
-    Predicate to determine if an `BaseExceptionGroup` only contains
-    some (maybe nested) set of sub-grouped exceptions (like only
-    `trio.Cancelled`s which get swallowed silently by default) and is
-    thus the result of "gracefully cancelling" a collection of
-    sub-tasks (or other conc primitives) and receiving a "cancelled
-    ACK" from each after termination.
-
-    Docs:
-    ----
-    - https://docs.python.org/3/library/exceptions.html#exception-groups
-    - https://docs.python.org/3/library/exceptions.html#BaseExceptionGroup.subgroup
-
-    '''
-
-    if (
-        not ignore_nested
-        or
-        trio.Cancelled in ignore_nested
-        # XXX always count-in `trio`'s native signal
-    ):
-        ignore_nested.update({trio.Cancelled})
-
-    if isinstance(exc, BaseExceptionGroup):
-        matched_exc: BaseExceptionGroup|None = exc.subgroup(
-            tuple(ignore_nested),
-
-            # TODO, complain about why not allowed XD
-            # condition=tuple(ignore_nested),
-        )
-        if matched_exc is not None:
-            return matched_exc
-
-    # NOTE, IFF no excs types match (throughout the error-tree)
-    # -> return `False`, OW return the matched sub-eg.
-    #
-    # IOW, for the inverse of ^ for the purpose of
-    # maybe-enter-REPL--logic: "only debug when the err-tree contains
-    # at least one exc-type NOT in `ignore_nested`" ; i.e. the case where
-    # we fallthrough and return `False` here.
-    return False
-
-
 def _raise_from_unexpected_msg(
     ctx: Context,
     msg: MsgType,

tractor/_portal.py

@@ -39,7 +39,10 @@ import warnings
 
 import trio
 
-from .trionics import maybe_open_nursery
+from .trionics import (
+    maybe_open_nursery,
+    collapse_eg,
+)
 from ._state import (
     current_actor,
 )
@@ -558,14 +561,13 @@ async def open_portal(
     assert actor
     was_connected: bool = False
 
-    async with maybe_open_nursery(
-        tn,
-        shield=shield,
-        strict_exception_groups=False,
-        # ^XXX^ TODO? soo roll our own then ??
-        # -> since we kinda want the "if only one `.exception` then
-        # just raise that" interface?
-    ) as tn:
+    async with (
+        collapse_eg(),
+        maybe_open_nursery(
+            tn,
+            shield=shield,
+        ) as tn,
+    ):
 
         if not channel.connected():
             await channel.connect()

tractor/_root.py

@@ -37,14 +37,12 @@ import warnings
 
 import trio
 
-from ._runtime import (
-    Actor,
-    Arbiter,
-    # TODO: rename and make a non-actor subtype?
-    # Arbiter as Registry,
-    async_main,
+from . import _runtime
+from .devx import (
+    debug,
+    _frame_stack,
+    pformat as _pformat,
 )
-from .devx import _debug
 from . import _spawn
 from . import _state
 from . import log
@@ -58,16 +56,19 @@ from ._addr import (
     mk_uuid,
     wrap_address,
 )
+from .trionics import (
+    is_multi_cancelled,
+    collapse_eg,
+)
 from ._exceptions import (
     RuntimeFailure,
-    is_multi_cancelled,
 )
 
 
 logger = log.get_logger('tractor')
 
 
-# TODO: stick this in a `@acm` defined in `devx._debug`?
+# TODO: stick this in a `@acm` defined in `devx.debug`?
 # -[ ] also maybe consider making this a `wrapt`-deco to
 #     save an indent level?
 #
@@ -89,7 +90,7 @@ async def maybe_block_bp(
         debug_mode
         and maybe_enable_greenback
         and (
-            maybe_mod := await _debug.maybe_init_greenback(
+            maybe_mod := await debug.maybe_init_greenback(
                 raise_not_found=False,
             )
         )
@@ -99,7 +100,7 @@ async def maybe_block_bp(
             'Enabling `tractor.pause_from_sync()` support!\n'
         )
         os.environ['PYTHONBREAKPOINT'] = (
-            'tractor.devx._debug._sync_pause_from_builtin'
+            'tractor.devx.debug._sync_pause_from_builtin'
         )
         _state._runtime_vars['use_greenback'] = True
         bp_blocked = False
@@ -163,7 +164,9 @@ async def open_root_actor(
 
     # enables the multi-process debugger support
     debug_mode: bool = False,
-    maybe_enable_greenback: bool = True,  # `.pause_from_sync()/breakpoint()` support
+    maybe_enable_greenback: bool = False,  # `.pause_from_sync()/breakpoint()` support
+    # ^XXX NOTE^ the perf implications of use,
+    # https://greenback.readthedocs.io/en/latest/principle.html#performance
     enable_stack_on_sig: bool = False,
 
     # internal logging
@@ -178,7 +181,7 @@ async def open_root_actor(
 
     hide_tb: bool = True,
 
-    # XXX, proxied directly to `.devx._debug._maybe_enter_pm()`
+    # XXX, proxied directly to `.devx.debug._maybe_enter_pm()`
     # for REPL-entry logic.
     debug_filter: Callable[
         [BaseException|BaseExceptionGroup],
@@ -189,9 +192,13 @@ async def open_root_actor(
     # read-only state to sublayers?
     # extra_rt_vars: dict|None = None,
 
-) -> Actor:
+) -> _runtime.Actor:
     '''
-    Runtime init entry point for ``tractor``.
+    Initialize the `tractor` runtime by starting a "root actor" in
+    a parent-most Python process.
+
+    All (disjoint) actor-process-trees-as-programs are created via
+    this entrypoint.
 
     '''
     # XXX NEVER allow nested actor-trees!
@@ -217,23 +224,18 @@ async def open_root_actor(
     ):
         if enable_transports is None:
             enable_transports: list[str] = _state.current_ipc_protos()
-        else:
-            _state._runtime_vars['_enable_tpts'] = enable_transports
 
-        # TODO! support multi-tpts per actor!
-        # Bo
-        if not len(enable_transports) == 1:
-            raise RuntimeError(
-                f'No multi-tpt support yet!\n'
-                f'enable_transports={enable_transports!r}\n'
-            )
+            # TODO! support multi-tpts per actor! Bo
+            assert (
+                len(enable_transports) == 1
+            ), 'No multi-tpt support yet!'
 
-        _debug.hide_runtime_frames()
+        _frame_stack.hide_runtime_frames()
         __tracebackhide__: bool = hide_tb
 
         # attempt to retreive ``trio``'s sigint handler and stash it
         # on our debugger lock state.
-        _debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
+        debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
 
         # mark top most level process as root actor
         _state._runtime_vars['_is_root'] = True
@@ -281,13 +283,14 @@ async def open_root_actor(
 
         if (
             debug_mode
-            and _spawn._spawn_method == 'trio'
+            and
+            _spawn._spawn_method == 'trio'
         ):
             _state._runtime_vars['_debug_mode'] = True
 
             # expose internal debug module to every actor allowing for
             # use of ``await tractor.pause()``
-            enable_modules.append('tractor.devx._debug')
+            enable_modules.append('tractor.devx.debug._tty_lock')
 
             # if debug mode get's enabled *at least* use that level of
             # logging for some informative console prompts.
@@ -375,7 +378,7 @@ async def open_root_actor(
                 f'Registry(s) seem(s) to exist @ {ponged_addrs}'
             )
 
-            actor = Actor(
+            actor = _runtime.Actor(
                 name=name or 'anonymous',
                 uuid=mk_uuid(),
                 registry_addrs=ponged_addrs,
@@ -410,7 +413,8 @@ async def open_root_actor(
             # https://github.com/goodboy/tractor/pull/348
             # https://github.com/goodboy/tractor/issues/296
 
-            actor = Arbiter(
+            # TODO: rename as `RootActor` or is that even necessary?
+            actor = _runtime.Arbiter(
                 name=name or 'registrar',
                 uuid=mk_uuid(),
                 registry_addrs=registry_addrs,
@@ -437,13 +441,13 @@ async def open_root_actor(
                 f'{ml_addrs_str}'
             )
 
-            # start the actor runtime in a new task
-            async with trio.open_nursery(
-                strict_exception_groups=False,
-                # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
-            ) as nursery:
+            # start runtime in a bg sub-task, yield to caller.
+            async with (
+                collapse_eg(),
+                trio.open_nursery() as root_tn,
+            ):
 
-                # ``_runtime.async_main()`` creates an internal nursery
+                # `_runtime.async_main()` creates an internal nursery
                 # and blocks here until any underlying actor(-process)
                 # tree has terminated thereby conducting so called
                 # "end-to-end" structured concurrency throughout an
@@ -451,9 +455,9 @@ async def open_root_actor(
                 # "actor runtime" primitives are SC-compat and thus all
                 # transitively spawned actors/processes must be as
                 # well.
-                await nursery.start(
+                await root_tn.start(
                     partial(
-                        async_main,
+                        _runtime.async_main,
                         actor,
                         accept_addrs=trans_bind_addrs,
                         parent_addr=None
@@ -469,7 +473,7 @@ async def open_root_actor(
                     # TODO, in beginning to handle the subsubactor with
                     # crashed grandparent cases..
                     #
-                    # was_locked: bool = await _debug.maybe_wait_for_debugger(
+                    # was_locked: bool = await debug.maybe_wait_for_debugger(
                     #     child_in_debug=True,
                     # )
                     # XXX NOTE XXX see equiv note inside
@@ -477,7 +481,7 @@ async def open_root_actor(
                     # non-root or root-that-opened-this-mahually case we
                     # wait for the local actor-nursery to exit before
                     # exiting the transport channel handler.
-                    entered: bool = await _debug._maybe_enter_pm(
+                    entered: bool = await debug._maybe_enter_pm(
                         err,
                         api_frame=inspect.currentframe(),
                         debug_filter=debug_filter,
@@ -510,15 +514,28 @@ async def open_root_actor(
                     #     for an in nurseries:
                     #         tempn.start_soon(an.exited.wait)
 
+                    op_nested_actor_repr: str = _pformat.nest_from_op(
+                        input_op='>) ',
+                        tree_str=actor.pformat(),
+                        nest_prefix='|_',
+                    )
                     logger.info(
                         f'Closing down root actor\n'
-                        f'>)\n'
-                        f'|_{actor}\n'
+                        f'{op_nested_actor_repr}\n'
                     )
                     await actor.cancel(None)  # self cancel
         finally:
+            # revert all process-global runtime state
+            if (
+                debug_mode
+                and
+                _spawn._spawn_method == 'trio'
+            ):
+                _state._runtime_vars['_debug_mode'] = False
+
             _state._current_actor = None
             _state._last_actor_terminated = actor
+
             logger.runtime(
                 f'Root actor terminated\n'
                 f')>\n'

tractor/_rpc.py

@@ -37,6 +37,7 @@ import warnings
 
 import trio
 from trio import (
+    Cancelled,
     CancelScope,
     Nursery,
     TaskStatus,
@@ -52,12 +53,16 @@ from ._exceptions import (
     ModuleNotExposed,
     MsgTypeError,
     TransportClosed,
-    is_multi_cancelled,
     pack_error,
     unpack_error,
 )
+from .trionics import (
+    collapse_eg,
+    is_multi_cancelled,
+    maybe_raise_from_masking_exc,
+)
 from .devx import (
-    _debug,
+    debug,
     add_div,
 )
 from . import _state
@@ -266,7 +271,7 @@ async def _errors_relayed_via_ipc(
 
     # TODO: a debug nursery when in debug mode!
     # async with maybe_open_debugger_nursery() as debug_tn:
-    # => see matching comment in side `._debug._pause()`
+    # => see matching comment in side `.debug._pause()`
     rpc_err: BaseException|None = None
     try:
         yield  # run RPC invoke body
@@ -318,7 +323,7 @@ async def _errors_relayed_via_ipc(
                     'RPC task crashed, attempting to enter debugger\n'
                     f'|_{ctx}'
                 )
-                entered_debug = await _debug._maybe_enter_pm(
+                entered_debug = await debug._maybe_enter_pm(
                     err,
                     api_frame=inspect.currentframe(),
                 )
@@ -462,7 +467,7 @@ async def _invoke(
     ):
         # XXX for .pause_from_sync()` usage we need to make sure
         # `greenback` is boostrapped in the subactor!
-        await _debug.maybe_init_greenback()
+        await debug.maybe_init_greenback()
 
     # TODO: possibly a specially formatted traceback
     # (not sure what typing is for this..)?
@@ -616,32 +621,40 @@ async def _invoke(
         #  -> the below scope is never exposed to the
         #     `@context` marked RPC function.
         # - `._portal` is never set.
+        scope_err: BaseException|None = None
         try:
-            tn: trio.Nursery
+            # TODO: better `trionics` primitive/tooling usage here!
+            # -[ ] should would be nice to have our `TaskMngr`
+            #   nursery here!
+            # -[ ] payload value checking like we do with
+            #   `.started()` such that the debbuger can engage
+            #   here in the child task instead of waiting for the
+            #   parent to crash with it's own MTE..
+            #
+            tn: Nursery
             rpc_ctx_cs: CancelScope
             async with (
-                trio.open_nursery(
-                    strict_exception_groups=False,
-                    # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
-
-                ) as tn,
+                collapse_eg(),
+                trio.open_nursery() as tn,
                 msgops.maybe_limit_plds(
                     ctx=ctx,
                     spec=ctx_meta.get('pld_spec'),
                     dec_hook=ctx_meta.get('dec_hook'),
                 ),
+
+                # XXX NOTE, this being the "most embedded"
+                # scope ensures unasking of the `await coro` below
+                # *should* never be interfered with!!
+                maybe_raise_from_masking_exc(
+                    tn=tn,
+                    unmask_from=Cancelled,
+                ) as _mbme,  # maybe boxed masked exc
             ):
                 ctx._scope_nursery = tn
                 rpc_ctx_cs = ctx._scope = tn.cancel_scope
                 task_status.started(ctx)
 
-                # TODO: better `trionics` tooling:
-                # -[ ] should would be nice to have our `TaskMngr`
-                #   nursery here!
-                # -[ ] payload value checking like we do with
-                #   `.started()` such that the debbuger can engage
-                #   here in the child task instead of waiting for the
-                #   parent to crash with it's own MTE..
+                # invoke user endpoint fn.
                 res: Any|PayloadT = await coro
                 return_msg: Return|CancelAck = return_msg_type(
                     cid=cid,
@@ -743,43 +756,52 @@ async def _invoke(
             BaseExceptionGroup,
             BaseException,
             trio.Cancelled,
-
-        ) as scope_error:
+        ) as _scope_err:
+            scope_err = _scope_err
             if (
-                isinstance(scope_error, RuntimeError)
-                and scope_error.args
-                and 'Cancel scope stack corrupted' in scope_error.args[0]
+                isinstance(scope_err, RuntimeError)
+                and
+                scope_err.args
+                and
+                'Cancel scope stack corrupted' in scope_err.args[0]
             ):
                 log.exception('Cancel scope stack corrupted!?\n')
-                # _debug.mk_pdb().set_trace()
+                # debug.mk_pdb().set_trace()
 
             # always set this (child) side's exception as the
             # local error on the context
-            ctx._local_error: BaseException = scope_error
+            ctx._local_error: BaseException = scope_err
             # ^-TODO-^ question,
             # does this matter other then for
             # consistentcy/testing?
             # |_ no user code should be in this scope at this point
             #    AND we already set this in the block below?
 
-            # if a remote error was set then likely the
-            # exception group was raised due to that, so
+            # XXX if a remote error was set then likely the
+            # exc group was raised due to that, so
             # and we instead raise that error immediately!
-            ctx.maybe_raise()
+            maybe_re: (
+                ContextCancelled|RemoteActorError
+            ) = ctx.maybe_raise()
+            if maybe_re:
+                log.cancel(
+                    f'Suppressing remote-exc from peer,\n'
+                    f'{maybe_re!r}\n'
+                )
 
             # maybe TODO: pack in come kinda
             # `trio.Cancelled.__traceback__` here so they can be
             # unwrapped and displayed on the caller side? no se..
-            raise
+            raise scope_err
 
         # `@context` entrypoint task bookeeping.
         # i.e. only pop the context tracking if used ;)
         finally:
-            assert chan.uid
+            assert chan.aid
 
             # don't pop the local context until we know the
             # associated child isn't in debug any more
-            await _debug.maybe_wait_for_debugger()
+            await debug.maybe_wait_for_debugger()
             ctx: Context = actor._contexts.pop((
                 chan.uid,
                 cid,
@@ -802,6 +824,9 @@ async def _invoke(
                     descr_str += (
                         f'\n{merr!r}\n'  # needed?
                         f'{tb_str}\n'
+                        f'\n'
+                        f'scope_error:\n'
+                        f'{scope_err!r}\n'
                     )
                 else:
                     descr_str += f'\n{merr!r}\n'
@@ -983,7 +1008,7 @@ async def process_messages(
                         # XXX NOTE XXX don't start entire actor
                         # runtime cancellation if this actor is
                         # currently in debug mode!
-                        pdb_complete: trio.Event|None = _debug.DebugStatus.repl_release
+                        pdb_complete: trio.Event|None = debug.DebugStatus.repl_release
                         if pdb_complete:
                             await pdb_complete.wait()
 

tractor/_runtime.py

@@ -55,6 +55,7 @@ from typing import (
     TYPE_CHECKING,
 )
 import uuid
+import textwrap
 from types import ModuleType
 import warnings
 
@@ -73,6 +74,9 @@ from tractor.msg import (
     pretty_struct,
     types as msgtypes,
 )
+from .trionics import (
+    collapse_eg,
+)
 from .ipc import (
     Channel,
     # IPCServer,  # causes cycles atm..
@@ -97,7 +101,10 @@ from ._exceptions import (
     MsgTypeError,
     unpack_error,
 )
-from .devx import _debug
+from .devx import (
+    debug,
+    pformat as _pformat
+)
 from ._discovery import get_registry
 from ._portal import Portal
 from . import _state
@@ -237,7 +244,7 @@ class Actor:
         # TODO? only add this when `is_debug_mode() == True` no?
         # always include debugging tools module
         if _state.is_root_process():
-            enable_modules.append('tractor.devx._debug')
+            enable_modules.append('tractor.devx.debug._tty_lock')
 
         self.enable_modules: dict[str, str] = get_mod_nsps2fps(
             mod_ns_paths=enable_modules,
@@ -339,46 +346,76 @@ class Actor:
     def pid(self) -> int:
         return self._aid.pid
 
-    def pformat(self) -> str:
-        ds: str = '='
+    def pformat(
+        self,
+        ds: str = ': ',
+        indent: int = 0,
+    ) -> str:
+        fields_sect_prefix: str = ' |_'
         parent_uid: tuple|None = None
         if rent_chan := self._parent_chan:
             parent_uid = rent_chan.uid
 
         peers: list = []
         server: _server.IPCServer = self.ipc_server
+        ipc_server_sect: str = ''
         if server:
             peers: list[tuple] = list(server._peer_connected)
 
+            # create field ln as a key-header indented under
+            # and up to the section's key prefix.
+            # field_ln_header: str = textwrap.indent(
+            #     text=f"ipc_server{ds}",
+            #     prefix=' '*len(fields_sect_prefix),
+            # )
+            # ^XXX if we were to indent `repr(Server)` to
+            # '<key>: '
+            #  _here_^
+            server_repr: str = textwrap.indent(
+                text=self._ipc_server.pformat(),
+                # prefix=' '*len(field_ln_header),
+                prefix=' '*len(fields_sect_prefix),
+            )
+            ipc_server_sect: str = (
+                # f'{field_ln_header}\n'
+                f'{server_repr}'
+            )
+
         fmtstr: str = (
             f' |_id: {self.aid!r}\n'
             # f"   aid{ds}{self.aid!r}\n"
             f"   parent{ds}{parent_uid}\n"
-            f'\n'
+            # f'\n'
             f' |_ipc: {len(peers)!r} connected peers\n'
             f"   peers{ds}{peers!r}\n"
-            f"   ipc_server{ds}{self._ipc_server}\n"
-            f'\n'
+            f"{ipc_server_sect}"
+            # f'\n'
             f' |_rpc: {len(self._rpc_tasks)} tasks\n'
             f"   ctxs{ds}{len(self._contexts)}\n"
-            f'\n'
+            # f'\n'
             f' |_runtime: ._task{ds}{self._task!r}\n'
             f'   _spawn_method{ds}{self._spawn_method}\n'
             f'   _actoruid2nursery{ds}{self._actoruid2nursery}\n'
             f'   _forkserver_info{ds}{self._forkserver_info}\n'
-            f'\n'
+            # f'\n'
             f' |_state: "TODO: .repr_state()"\n'
             f'   _cancel_complete{ds}{self._cancel_complete}\n'
             f'   _cancel_called_by_remote{ds}{self._cancel_called_by_remote}\n'
             f'   _cancel_called{ds}{self._cancel_called}\n'
         )
-        return (
+        _repr: str = (
             '<Actor(\n'
             +
             fmtstr
             +
             ')>\n'
         )
+        if indent:
+            _repr: str = textwrap.indent(
+                text=_repr,
+                prefix=' '*indent,
+            )
+        return _repr
 
     __repr__ = pformat
 
@@ -877,7 +914,9 @@ class Actor:
             return (
                 chan,
                 accept_addrs,
-                _state._runtime_vars['_enable_tpts']
+                None,
+                # ^TODO, preferred tpts list from rent!
+                # -[ ] need to extend the `SpawnSpec` tho!
             )
 
         # failed to connect back?
@@ -968,7 +1007,7 @@ class Actor:
 
             # kill any debugger request task to avoid deadlock
             # with the root actor in this tree
-            debug_req = _debug.DebugStatus
+            debug_req = debug.DebugStatus
             lock_req_ctx: Context = debug_req.req_ctx
             if (
                 lock_req_ctx
@@ -978,7 +1017,7 @@ class Actor:
                 msg += (
                     f'\n'
                     f'-> Cancelling active debugger request..\n'
-                    f'|_{_debug.Lock.repr()}\n\n'
+                    f'|_{debug.Lock.repr()}\n\n'
                     f'|_{lock_req_ctx}\n\n'
                 )
                 # lock_req_ctx._scope.cancel()
@@ -1018,6 +1057,7 @@ class Actor:
         cid: str,
         parent_chan: Channel,
         requesting_uid: tuple[str, str]|None,
+        # ^^TODO! use the `Aid` directly here!
 
         ipc_msg: dict|None|bool = False,
 
@@ -1063,9 +1103,12 @@ class Actor:
 
         log.cancel(
             'Rxed cancel request for RPC task\n'
-            f'<=c) {requesting_uid}\n'
-            f' |_{ctx._task}\n'
-            f'    >> {ctx.repr_rpc}\n'
+            f'{ctx._task!r} <=c) {requesting_uid}\n'
+            f'|_>> {ctx.repr_rpc}\n'
+
+            # f'|_{ctx._task}\n'
+            # f'   >> {ctx.repr_rpc}\n'
+
             # f'=> {ctx._task}\n'
             # f'  >> Actor._cancel_task() => {ctx._task}\n'
             # f'  |_ {ctx._task}\n\n'
@@ -1308,7 +1351,7 @@ async def async_main(
 
     # attempt to retreive ``trio``'s sigint handler and stash it
     # on our debugger state.
-    _debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
+    debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
 
     is_registered: bool = False
     try:
@@ -1350,10 +1393,12 @@ async def async_main(
         # parent is kept alive as a resilient service until
         # cancellation steps have (mostly) occurred in
         # a deterministic way.
-        async with trio.open_nursery(
-            strict_exception_groups=False,
-        ) as root_nursery:
-            actor._root_n = root_nursery
+        root_tn: trio.Nursery
+        async with (
+            collapse_eg(),
+            trio.open_nursery() as root_tn,
+        ):
+            actor._root_n = root_tn
             assert actor._root_n
 
             ipc_server: _server.IPCServer
@@ -1401,7 +1446,7 @@ async def async_main(
                 # try:
                 #     actor.load_modules()
                 # except ModuleNotFoundError as err:
-                #     _debug.pause_from_sync()
+                #     debug.pause_from_sync()
                 #     import pdbp; pdbp.set_trace()
                 #     raise
 
@@ -1433,7 +1478,7 @@ async def async_main(
                     # tranport address bind errors - normally it's
                     # something silly like the wrong socket-address
                     # passed via a config or CLI Bo
-                    entered_debug: bool = await _debug._maybe_enter_pm(
+                    entered_debug: bool = await debug._maybe_enter_pm(
                         oserr,
                     )
                     if not entered_debug:
@@ -1451,12 +1496,10 @@ async def async_main(
                 # all sub-actors should be able to speak to
                 # their root actor over that channel.
                 if _state._runtime_vars['_is_root']:
-                    raddrs: list[Address] = _state._runtime_vars['_root_addrs']
                     for addr in accept_addrs:
                         waddr: Address = wrap_address(addr)
-                        raddrs.append(addr)
-                    else:
-                        _state._runtime_vars['_root_mailbox'] = raddrs[0]
+                        if waddr == waddr.get_root():
+                            _state._runtime_vars['_root_mailbox'] = addr
 
                 # Register with the arbiter if we're told its addr
                 log.runtime(
@@ -1473,7 +1516,7 @@ async def async_main(
                         waddr = wrap_address(addr)
                         assert waddr.is_valid
                     except AssertionError:
-                        await _debug.pause()
+                        await debug.pause()
 
                     async with get_registry(addr) as reg_portal:
                         for accept_addr in accept_addrs:
@@ -1499,7 +1542,7 @@ async def async_main(
                 # start processing parent requests until our channel
                 # server is 100% up and running.
                 if actor._parent_chan:
-                    await root_nursery.start(
+                    await root_tn.start(
                         partial(
                             _rpc.process_messages,
                             chan=actor._parent_chan,
@@ -1590,7 +1633,7 @@ async def async_main(
             # prevents any `infected_aio` actor from continuing
             # and any callbacks in the `ls` here WILL NOT be
             # called!!
-            # await _debug.pause(shield=True)
+            # await debug.pause(shield=True)
 
         ls.close()
 
@@ -1603,7 +1646,7 @@ async def async_main(
         #
         # if actor.name == 'brokerd.ib':
         #     with CancelScope(shield=True):
-        #         await _debug.breakpoint()
+        #         await debug.breakpoint()
 
         # Unregister actor from the registry-sys / registrar.
         if (
@@ -1654,10 +1697,15 @@ async def async_main(
             '-> All peer channels are complete\n'
         )
 
+    op_nested_actor_repr: str = _pformat.nest_from_op(
+        input_op=')> ',
+        tree_str=actor.pformat(),
+        nest_prefix='|_',
+        back_from_op=2,
+    )
     teardown_report += (
-        'Actor runtime exiting\n'
-        f'>)\n'
-        f'|_{actor}\n'
+        'Actor runtime exited\n'
+        f'{op_nested_actor_repr}\n'
     )
     log.info(teardown_report)
 
@@ -1792,7 +1840,7 @@ class Arbiter(Actor):
         waddr: Address = wrap_address(addr)
         if not waddr.is_valid:
             # should never be 0-dynamic-os-alloc
-            await _debug.pause()
+            await debug.pause()
 
         self._registry[uid] = addr
 

tractor/_spawn.py

@@ -34,7 +34,7 @@ from typing import (
 import trio
 from trio import TaskStatus
 
-from .devx._debug import (
+from .devx.debug import (
     maybe_wait_for_debugger,
     acquire_debug_lock,
 )

tractor/_state.py

@@ -37,34 +37,28 @@ if TYPE_CHECKING:
     from ._context import Context
 
 
-# default IPC transport protocol settings
-TransportProtocolKey = Literal[
-    'tcp',
-    'uds',
-]
-_def_tpt_proto: TransportProtocolKey = 'tcp'
-
 _current_actor: Actor|None = None  # type: ignore # noqa
 _last_actor_terminated: Actor|None = None
 
 # TODO: mk this a `msgspec.Struct`!
+# -[ ] type out all fields obvi!
+# -[ ] (eventually) mk wire-ready for monitoring?
 _runtime_vars: dict[str, Any] = {
-    '_debug_mode': False,
     # root of actor-process tree info
     '_is_root': False,  # bool
     '_root_mailbox': (None, None),  # tuple[str|None, str|None]
-    '_root_addrs': [],  # tuple[str|None, str|None]
-
-    # parent->chld ipc protocol caps
-    '_enable_tpts': [_def_tpt_proto],
 
     # registrar info
     '_registry_addrs': [],
 
-    '_is_infected_aio': False,
-
+    # `debug_mode: bool` settings
+    '_debug_mode': False,  # bool
+    'repl_fixture': False,  # |AbstractContextManager[bool]
     # for `tractor.pause_from_sync()` & `breakpoint()` support
     'use_greenback': False,
+
+    # infected-`asyncio`-mode: `trio` running as guest.
+    '_is_infected_aio': False,
 }
 
 
@@ -186,6 +180,14 @@ def get_rt_dir(
     return rtdir
 
 
+# default IPC transport protocol settings
+TransportProtocolKey = Literal[
+    'tcp',
+    'uds',
+]
+_def_tpt_proto: TransportProtocolKey = 'tcp'
+
+
 def current_ipc_protos() -> list[str]:
     '''
     Return the list of IPC transport protocol keys currently
@@ -195,4 +197,4 @@ def current_ipc_protos() -> list[str]:
     concrete-backend sub-types defined throughout `tractor.ipc`.
 
     '''
-    return _runtime_vars['_enable_tpts']
+    return [_def_tpt_proto]

tractor/_streaming.py

@@ -426,8 +426,8 @@ class MsgStream(trio.abc.Channel):
             self._closed = re
 
         # if caught_eoc:
-        #     # from .devx import _debug
-        #     # await _debug.pause()
+        #     # from .devx import debug
+        #     # await debug.pause()
         #     with trio.CancelScope(shield=True):
         #         await rx_chan.aclose()
 

tractor/_supervise.py

@@ -31,7 +31,7 @@ import warnings
 import trio
 
 
-from .devx._debug import maybe_wait_for_debugger
+from .devx.debug import maybe_wait_for_debugger
 from ._addr import (
     UnwrappedAddress,
     mk_uuid,
@@ -40,8 +40,11 @@ from ._state import current_actor, is_main_process
 from .log import get_logger, get_loglevel
 from ._runtime import Actor
 from ._portal import Portal
-from ._exceptions import (
+from .trionics import (
     is_multi_cancelled,
+    collapse_eg,
+)
+from ._exceptions import (
     ContextCancelled,
 )
 from ._root import (
@@ -322,9 +325,10 @@ class ActorNursery:
         server: IPCServer = self._actor.ipc_server
 
         with trio.move_on_after(3) as cs:
-            async with trio.open_nursery(
-                strict_exception_groups=False,
-            ) as tn:
+            async with (
+                collapse_eg(),
+                trio.open_nursery() as tn,
+            ):
 
                 subactor: Actor
                 proc: trio.Process
@@ -417,10 +421,10 @@ async def _open_and_supervise_one_cancels_all_nursery(
     # `ActorNursery.start_actor()`).
 
     # errors from this daemon actor nursery bubble up to caller
-    async with trio.open_nursery(
-        strict_exception_groups=False,
-        # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
-    ) as da_nursery:
+    async with (
+        collapse_eg(),
+        trio.open_nursery() as da_nursery,
+    ):
         try:
             # This is the inner level "run in actor" nursery. It is
             # awaited first since actors spawned in this way (using
@@ -430,11 +434,10 @@ async def _open_and_supervise_one_cancels_all_nursery(
             # immediately raised for handling by a supervisor strategy.
             # As such if the strategy propagates any error(s) upwards
             # the above "daemon actor" nursery will be notified.
-            async with trio.open_nursery(
-                strict_exception_groups=False,
-                # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
-            ) as ria_nursery:
-
+            async with (
+                collapse_eg(),
+                trio.open_nursery() as ria_nursery,
+            ):
                 an = ActorNursery(
                     actor,
                     ria_nursery,

tractor/_testing/__init__.py

@@ -26,7 +26,7 @@ import os
 import pathlib
 
 import tractor
-from tractor.devx._debug import (
+from tractor.devx.debug import (
     BoxedMaybeException,
 )
 from .pytest import (
@@ -37,6 +37,9 @@ from .fault_simulation import (
 )
 
 
+# TODO, use dulwhich for this instead?
+# -> we're going to likely need it (or something similar)
+#   for supporting hot-coad reload feats eventually anyway!
 def repodir() -> pathlib.Path:
     '''
     Return the abspath to the repo directory.

tractor/_testing/addr.py

@@ -0,0 +1,70 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Random IPC addr generation for isolating
+the discovery space between test sessions.
+
+Might be eventually useful to expose as a util set from
+our `tractor.discovery` subsys?
+
+'''
+import random
+from typing import (
+    Type,
+)
+from tractor import (
+    _addr,
+)
+
+
+def get_rando_addr(
+    tpt_proto: str,
+    *,
+
+    # choose random port at import time
+    _rando_port: str = random.randint(1000, 9999)
+
+) -> tuple[str, str|int]:
+    '''
+    Used to globally override the runtime to the
+    per-test-session-dynamic addr so that all tests never conflict
+    with any other actor tree using the default.
+
+    '''
+    addr_type: Type[_addr.Addres] = _addr._address_types[tpt_proto]
+    def_reg_addr: tuple[str, int] = _addr._default_lo_addrs[tpt_proto]
+
+    # this is the "unwrapped" form expected to be passed to
+    # `.open_root_actor()` by test body.
+    testrun_reg_addr: tuple[str, int|str]
+    match tpt_proto:
+        case 'tcp':
+            testrun_reg_addr = (
+                addr_type.def_bindspace,
+                _rando_port,
+            )
+
+        # NOTE, file-name uniqueness (no-collisions) will be based on
+        # the runtime-directory and root (pytest-proc's) pid.
+        case 'uds':
+            testrun_reg_addr = addr_type.get_random().unwrap()
+
+    # XXX, as sanity it should never the same as the default for the
+    # host-singleton registry actor.
+    assert def_reg_addr != testrun_reg_addr
+
+    return testrun_reg_addr

tractor/_testing/pytest.py

@@ -26,29 +26,46 @@ from functools import (
 import inspect
 import platform
 
+import pytest
 import tractor
 import trio
 
 
 def tractor_test(fn):
     '''
-    Decorator for async test funcs to present them as "native"
-    looking sync funcs runnable by `pytest` using `trio.run()`.
+    Decorator for async test fns to decorator-wrap them as "native"
+    looking sync funcs runnable by `pytest` and auto invoked with
+    `trio.run()` (much like the `pytest-trio` plugin's approach).
 
-    Use:
+    Further the test fn body will be invoked AFTER booting the actor
+    runtime, i.e. from inside a `tractor.open_root_actor()` block AND
+    with various runtime and tooling parameters implicitly passed as
+    requested by by the test session's config; see immediately below.
 
-    @tractor_test
-    async def test_whatever():
-        await ...
+    Basic deco use:
+    ---------------
 
-    If fixtures:
+      @tractor_test
+      async def test_whatever():
+          await ...
 
-        - ``reg_addr`` (a socket addr tuple where arbiter is listening)
-        - ``loglevel`` (logging level passed to tractor internals)
-        - ``start_method`` (subprocess spawning backend)
 
-    are defined in the `pytest` fixture space they will be automatically
-    injected to tests declaring these funcargs.
+    Runtime config via special fixtures:
+    ------------------------------------
+    If any of the following fixture are requested by the wrapped test
+    fn (via normal func-args declaration),
+
+    - `reg_addr` (a socket addr tuple where arbiter is listening)
+    - `loglevel` (logging level passed to tractor internals)
+    - `start_method` (subprocess spawning backend)
+
+    (TODO support)
+    - `tpt_proto` (IPC transport protocol key)
+
+    they will be automatically injected to each test as normally
+    expected as well as passed to the initial
+    `tractor.open_root_actor()` funcargs.
+
     '''
     @wraps(fn)
     def wrapper(
@@ -111,3 +128,164 @@ def tractor_test(fn):
         return trio.run(main)
 
     return wrapper
+
+
+def pytest_addoption(
+    parser: pytest.Parser,
+):
+    # parser.addoption(
+    #     "--ll",
+    #     action="store",
+    #     dest='loglevel',
+    #     default='ERROR', help="logging level to set when testing"
+    # )
+
+    parser.addoption(
+        "--spawn-backend",
+        action="store",
+        dest='spawn_backend',
+        default='trio',
+        help="Processing spawning backend to use for test run",
+    )
+
+    parser.addoption(
+        "--tpdb",
+        "--debug-mode",
+        action="store_true",
+        dest='tractor_debug_mode',
+        # default=False,
+        help=(
+            'Enable a flag that can be used by tests to to set the '
+            '`debug_mode: bool` for engaging the internal '
+            'multi-proc debugger sys.'
+        ),
+    )
+
+    # provide which IPC transport protocols opting-in test suites
+    # should accumulatively run against.
+    parser.addoption(
+        "--tpt-proto",
+        nargs='+',  # accumulate-multiple-args
+        action="store",
+        dest='tpt_protos',
+        default=['tcp'],
+        help="Transport protocol to use under the `tractor.ipc.Channel`",
+    )
+
+
+def pytest_configure(config):
+    backend = config.option.spawn_backend
+    tractor._spawn.try_set_start_method(backend)
+
+
+@pytest.fixture(scope='session')
+def debug_mode(request) -> bool:
+    '''
+    Flag state for whether `--tpdb` (for `tractor`-py-debugger)
+    was passed to the test run.
+
+    Normally tests should pass this directly to `.open_root_actor()`
+    to allow the user to opt into suite-wide crash handling.
+
+    '''
+    debug_mode: bool = request.config.option.tractor_debug_mode
+    return debug_mode
+
+
+@pytest.fixture(scope='session')
+def spawn_backend(request) -> str:
+    return request.config.option.spawn_backend
+
+
+@pytest.fixture(scope='session')
+def tpt_protos(request) -> list[str]:
+
+    # allow quoting on CLI
+    proto_keys: list[str] = [
+        proto_key.replace('"', '').replace("'", "")
+        for proto_key in request.config.option.tpt_protos
+    ]
+
+    # ?TODO, eventually support multiple protos per test-sesh?
+    if len(proto_keys) > 1:
+        pytest.fail(
+            'We only support one `--tpt-proto <key>` atm!\n'
+        )
+
+    # XXX ensure we support the protocol by name via lookup!
+    for proto_key in proto_keys:
+        addr_type = tractor._addr._address_types[proto_key]
+        assert addr_type.proto_key == proto_key
+
+    yield proto_keys
+
+
+@pytest.fixture(
+    scope='session',
+    autouse=True,
+)
+def tpt_proto(
+    tpt_protos: list[str],
+) -> str:
+    proto_key: str = tpt_protos[0]
+
+    from tractor import _state
+    if _state._def_tpt_proto != proto_key:
+        _state._def_tpt_proto = proto_key
+
+    yield proto_key
+
+
+@pytest.fixture(scope='session')
+def reg_addr(
+    tpt_proto: str,
+) -> tuple[str, int|str]:
+    '''
+    Deliver a test-sesh unique registry address such
+    that each run's (tests which use this fixture) will
+    have no conflicts/cross-talk when running simultaneously
+    nor will interfere with other live `tractor` apps active
+    on the same network-host (namespace).
+
+    '''
+    from tractor._testing.addr import get_rando_addr
+    return get_rando_addr(
+        tpt_proto=tpt_proto,
+    )
+
+
+def pytest_generate_tests(
+    metafunc: pytest.Metafunc,
+):
+    spawn_backend: str = metafunc.config.option.spawn_backend
+
+    if not spawn_backend:
+        # XXX some weird windows bug with `pytest`?
+        spawn_backend = 'trio'
+
+    # TODO: maybe just use the literal `._spawn.SpawnMethodKey`?
+    assert spawn_backend in (
+        'mp_spawn',
+        'mp_forkserver',
+        'trio',
+    )
+
+    # NOTE: used-to-be-used-to dyanmically parametrize tests for when
+    # you just passed --spawn-backend=`mp` on the cli, but now we expect
+    # that cli input to be manually specified, BUT, maybe we'll do
+    # something like this again in the future?
+    if 'start_method' in metafunc.fixturenames:
+        metafunc.parametrize(
+            "start_method",
+            [spawn_backend],
+            scope='module',
+        )
+
+    # TODO, parametrize any `tpt_proto: str` declaring tests!
+    # proto_tpts: list[str] = metafunc.config.option.proto_tpts
+    # if 'tpt_proto' in metafunc.fixturenames:
+    #     metafunc.parametrize(
+    #         'tpt_proto',
+    #         proto_tpts,  # TODO, double check this list usage!
+    #         scope='module',
+    #     )

tractor/devx/__init__.py

@@ -20,7 +20,7 @@ Runtime "developer experience" utils and addons to aid our
 and working with/on the actor runtime.
 
 """
-from ._debug import (
+from .debug import (
     maybe_wait_for_debugger as maybe_wait_for_debugger,
     acquire_debug_lock as acquire_debug_lock,
     breakpoint as breakpoint,

tractor/devx/_frame_stack.py

@@ -20,13 +20,18 @@ as it pertains to improving the grok-ability of our runtime!
 
 '''
 from __future__ import annotations
+from contextlib import (
+    _GeneratorContextManager,
+    _AsyncGeneratorContextManager,
+)
 from functools import partial
 import inspect
+import textwrap
 from types import (
     FrameType,
     FunctionType,
     MethodType,
-    # CodeType,
+    CodeType,
 )
 from typing import (
     Any,
@@ -34,6 +39,9 @@ from typing import (
     Type,
 )
 
+import pdbp
+from tractor.log import get_logger
+import trio
 from tractor.msg import (
     pretty_struct,
     NamespacePath,
@@ -41,6 +49,8 @@ from tractor.msg import (
 import wrapt
 
 
+log = get_logger(__name__)
+
 # TODO: yeah, i don't love this and we should prolly just
 # write a decorator that actually keeps a stupid ref to the func
 # obj..
@@ -301,3 +311,70 @@ def api_frame(
 #     error_set: set[BaseException],
 # ) -> TracebackType:
 #     ...
+
+
+def hide_runtime_frames() -> dict[FunctionType, CodeType]:
+    '''
+    Hide call-stack frames for various std-lib and `trio`-API primitives
+    such that the tracebacks presented from our runtime are as minimized
+    as possible, particularly from inside a `PdbREPL`.
+
+    '''
+    # XXX HACKZONE XXX
+    #  hide exit stack frames on nurseries and cancel-scopes!
+    # |_ so avoid seeing it when the `pdbp` REPL is first engaged from
+    #    inside a `trio.open_nursery()` scope (with no line after it
+    #    in before the block end??).
+    #
+    # TODO: FINALLY got this workin originally with
+    #  `@pdbp.hideframe` around the `wrapper()` def embedded inside
+    #  `_ki_protection_decoratior()`.. which is in the module:
+    #  /home/goodboy/.virtualenvs/tractor311/lib/python3.11/site-packages/trio/_core/_ki.py
+    #
+    # -[ ] make an issue and patch for `trio` core? maybe linked
+    #    to the long outstanding `pdb` one below?
+    #   |_ it's funny that there's frame hiding throughout `._run.py`
+    #      but not where it matters on the below exit funcs..
+    #
+    # -[ ] provide a patchset for the lonstanding
+    #   |_ https://github.com/python-trio/trio/issues/1155
+    #
+    # -[ ] make a linked issue to ^ and propose allowing all the
+    #     `._core._run` code to have their `__tracebackhide__` value
+    #     configurable by a `RunVar` to allow getting scheduler frames
+    #     if desired through configuration?
+    #
+    # -[ ] maybe dig into the core `pdb` issue why the extra frame is shown
+    #      at all?
+    #
+    funcs: list[FunctionType] = [
+        trio._core._run.NurseryManager.__aexit__,
+        trio._core._run.CancelScope.__exit__,
+         _GeneratorContextManager.__exit__,
+         _AsyncGeneratorContextManager.__aexit__,
+         _AsyncGeneratorContextManager.__aenter__,
+         trio.Event.wait,
+    ]
+    func_list_str: str = textwrap.indent(
+        "\n".join(f.__qualname__ for f in funcs),
+        prefix=' |_ ',
+    )
+    log.devx(
+        'Hiding the following runtime frames by default:\n'
+        f'{func_list_str}\n'
+    )
+
+    codes: dict[FunctionType, CodeType] = {}
+    for ref in funcs:
+        # stash a pre-modified version of each ref's code-obj
+        # so it can be reverted later if needed.
+        codes[ref] = ref.__code__
+        pdbp.hideframe(ref)
+    #
+    # pdbp.hideframe(trio._core._run.NurseryManager.__aexit__)
+    # pdbp.hideframe(trio._core._run.CancelScope.__exit__)
+    # pdbp.hideframe(_GeneratorContextManager.__exit__)
+    # pdbp.hideframe(_AsyncGeneratorContextManager.__aexit__)
+    # pdbp.hideframe(_AsyncGeneratorContextManager.__aenter__)
+    # pdbp.hideframe(trio.Event.wait)
+    return codes

tractor/devx/_stackscope.py

@@ -49,7 +49,7 @@ from tractor import (
     _state,
     log as logmod,
 )
-from tractor.devx import _debug
+from tractor.devx import debug
 
 log = logmod.get_logger(__name__)
 
@@ -82,7 +82,7 @@ def dump_task_tree() -> None:
     if (
         current_sigint_handler
         is not
-        _debug.DebugStatus._trio_handler
+        debug.DebugStatus._trio_handler
     ):
         sigint_handler_report: str = (
             'The default `trio` SIGINT handler was replaced?!'
@@ -238,7 +238,8 @@ def enable_stack_on_sig(
         import stackscope
     except ImportError:
         log.error(
-            '`stackscope` not installed for use in debug mode!'
+            '`stackscope` not installed for use in debug mode!\n'
+            '`Ignoring {enable_stack_on_sig!r} call!\n'
         )
         return None
 

tractor/devx/debug/__init__.py

@@ -0,0 +1,100 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+Multi-actor debugging for da peeps!
+
+'''
+from __future__ import annotations
+from tractor.log import get_logger
+from ._repl import (
+    PdbREPL as PdbREPL,
+    mk_pdb as mk_pdb,
+    TractorConfig as TractorConfig,
+)
+from ._tty_lock import (
+    DebugStatus as DebugStatus,
+    DebugStateError as DebugStateError,
+)
+from ._trace import (
+    Lock as Lock,
+    _pause_msg as _pause_msg,
+    _repl_fail_msg as _repl_fail_msg,
+    _set_trace as _set_trace,
+    _sync_pause_from_builtin as _sync_pause_from_builtin,
+    breakpoint as breakpoint,
+    maybe_init_greenback as maybe_init_greenback,
+    maybe_import_greenback as maybe_import_greenback,
+    pause as pause,
+    pause_from_sync as pause_from_sync,
+)
+from ._post_mortem import (
+    BoxedMaybeException as BoxedMaybeException,
+    maybe_open_crash_handler as maybe_open_crash_handler,
+    open_crash_handler as open_crash_handler,
+    post_mortem as post_mortem,
+    _crash_msg as _crash_msg,
+    _maybe_enter_pm as _maybe_enter_pm,
+)
+from ._sync import (
+    maybe_wait_for_debugger as maybe_wait_for_debugger,
+    acquire_debug_lock as acquire_debug_lock,
+)
+from ._sigint import (
+    sigint_shield as sigint_shield,
+    _ctlc_ignore_header as _ctlc_ignore_header
+)
+
+log = get_logger(__name__)
+
+# ----------------
+# XXX PKG TODO XXX
+# ----------------
+# refine the internal impl and APIs!
+#
+# -[ ] rework `._pause()` and it's branch-cases for root vs.
+#     subactor:
+#  -[ ] `._pause_from_root()` + `_pause_from_subactor()`?
+#  -[ ]  do the de-factor based on bg-thread usage in
+#    `.pause_from_sync()` & `_pause_from_bg_root_thread()`.
+#  -[ ] drop `debug_func == None` case which is confusing af..
+#  -[ ]  factor out `_enter_repl_sync()` into a util func for calling
+#    the `_set_trace()` / `_post_mortem()` APIs?
+#
+# -[ ] figure out if we need `acquire_debug_lock()` and/or re-implement
+#    it as part of the `.pause_from_sync()` rework per above?
+#
+# -[ ] pair the `._pause_from_subactor()` impl with a "debug nursery"
+#   that's dynamically allocated inside the `._rpc` task thus
+#   avoiding the `._service_n.start()` usage for the IPC request?
+#  -[ ] see the TODO inside `._rpc._errors_relayed_via_ipc()`
+#
+# -[ ] impl a `open_debug_request()` which encaps all
+#   `request_root_stdio_lock()` task scheduling deats
+#   + `DebugStatus` state mgmt; which should prolly be re-branded as
+#   a `DebugRequest` type anyway AND with suppoort for bg-thread
+#   (from root actor) usage?
+#
+# -[ ] handle the `xonsh` case for bg-root-threads in the SIGINT
+#     handler!
+#   -[ ] do we need to do the same for subactors?
+#   -[ ] make the failing tests finally pass XD
+#
+# -[ ] simplify `maybe_wait_for_debugger()` to be a root-task only
+#     API?
+#   -[ ] currently it's implemented as that so might as well make it
+#     formal?

tractor/devx/debug/_post_mortem.py

@@ -0,0 +1,410 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+Post-mortem debugging APIs and surrounding machinery for both
+sync and async contexts.
+
+Generally we maintain the same semantics a `pdb.post.mortem()` but
+with actor-tree-wide sync/cooperation around any (sub)actor's use of
+the root's TTY.
+
+'''
+from __future__ import annotations
+import bdb
+from contextlib import (
+    AbstractContextManager,
+    contextmanager as cm,
+    nullcontext,
+)
+from functools import (
+    partial,
+)
+import inspect
+import sys
+import traceback
+from typing import (
+    Callable,
+    Sequence,
+    Type,
+    TYPE_CHECKING,
+)
+from types import (
+    TracebackType,
+    FrameType,
+)
+
+from msgspec import Struct
+import trio
+from tractor._exceptions import (
+    NoRuntime,
+)
+from tractor import _state
+from tractor._state import (
+    current_actor,
+    debug_mode,
+)
+from tractor.log import get_logger
+from tractor.trionics import (
+    is_multi_cancelled,
+)
+from ._trace import (
+    _pause,
+)
+from ._tty_lock import (
+    DebugStatus,
+)
+from ._repl import (
+    PdbREPL,
+    mk_pdb,
+    TractorConfig as TractorConfig,
+)
+
+if TYPE_CHECKING:
+    from trio.lowlevel import Task
+    from tractor._runtime import (
+        Actor,
+    )
+
+_crash_msg: str = (
+    'Opening a pdb REPL in crashed actor'
+)
+
+log = get_logger(__package__)
+
+
+class BoxedMaybeException(Struct):
+    '''
+    Box a maybe-exception for post-crash introspection usage
+    from the body of a `open_crash_handler()` scope.
+
+    '''
+    value: BaseException|None = None
+
+    # handler can suppress crashes dynamically
+    raise_on_exit: bool|Sequence[Type[BaseException]] = True
+
+    def pformat(self) -> str:
+        '''
+        Repr the boxed `.value` error in more-than-string
+        repr form.
+
+        '''
+        if not self.value:
+            return f'<{type(self).__name__}( .value=None )>'
+
+        return (
+            f'<{type(self.value).__name__}(\n'
+            f' |_.value = {self.value}\n'
+            f')>\n'
+        )
+
+    __repr__ = pformat
+
+
+def _post_mortem(
+    repl: PdbREPL,  # normally passed by `_pause()`
+
+    # XXX all `partial`-ed in by `post_mortem()` below!
+    tb: TracebackType,
+    api_frame: FrameType,
+
+    shield: bool = False,
+    hide_tb: bool = True,
+
+    # maybe pre/post REPL entry
+    repl_fixture: (
+        AbstractContextManager[bool]
+        |None
+    ) = None,
+
+    boxed_maybe_exc: BoxedMaybeException|None = None,
+
+) -> None:
+    '''
+    Enter the ``pdbpp`` port mortem entrypoint using our custom
+    debugger instance.
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    # maybe enter any user fixture
+    enter_repl: bool = DebugStatus.maybe_enter_repl_fixture(
+        repl=repl,
+        repl_fixture=repl_fixture,
+        boxed_maybe_exc=boxed_maybe_exc,
+    )
+    if not enter_repl:
+        return
+    try:
+        actor: Actor = current_actor()
+        actor_repr: str = str(actor.uid)
+        # ^TODO, instead a nice runtime-info + maddr + uid?
+        # -[ ] impl a `Actor.__repr()__`??
+        #  |_ <task>:<thread> @ <actor>
+
+    except NoRuntime:
+        actor_repr: str = '<no-actor-runtime?>'
+
+    try:
+        task_repr: Task = trio.lowlevel.current_task()
+    except RuntimeError:
+        task_repr: str = '<unknown-Task>'
+
+    # TODO: print the actor supervion tree up to the root
+    # here! Bo
+    log.pdb(
+        f'{_crash_msg}\n'
+        f'x>(\n'
+        f' |_ {task_repr} @ {actor_repr}\n'
+
+    )
+
+    # XXX NOTE(s) on `pdbp.xpm()` version..
+    #
+    # - seems to lose the up-stack tb-info?
+    # - currently we're (only) replacing this from `pdbp.xpm()`
+    #   to add the `end=''` to the print XD
+    #
+    print(traceback.format_exc(), end='')
+    caller_frame: FrameType = api_frame.f_back
+
+    # NOTE, see the impl details of these in the lib to
+    # understand usage:
+    # - `pdbp.post_mortem()`
+    # - `pdbp.xps()`
+    # - `bdb.interaction()`
+    repl.reset()
+    repl.interaction(
+        frame=caller_frame,
+        # frame=None,
+        traceback=tb,
+    )
+
+    # XXX NOTE XXX: this is abs required to avoid hangs!
+    #
+    # Since we presume the post-mortem was enaged to
+    # a task-ending error, we MUST release the local REPL request
+    # so that not other local task nor the root remains blocked!
+    DebugStatus.release()
+
+
+async def post_mortem(
+    *,
+    tb: TracebackType|None = None,
+    api_frame: FrameType|None = None,
+    hide_tb: bool = False,
+
+    # TODO: support shield here just like in `pause()`?
+    # shield: bool = False,
+
+    **_pause_kwargs,
+
+) -> None:
+    '''
+    Our builtin async equivalient of `pdb.post_mortem()` which can be
+    used inside exception handlers.
+
+    It's also used for the crash handler when `debug_mode == True` ;)
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    tb: TracebackType = tb or sys.exc_info()[2]
+
+    # TODO: do upward stack scan for highest @api_frame and
+    # use its parent frame as the expected user-app code
+    # interact point.
+    api_frame: FrameType = api_frame or inspect.currentframe()
+
+    # TODO, move to submod `._pausing` or ._api? _trace
+    await _pause(
+        debug_func=partial(
+            _post_mortem,
+            api_frame=api_frame,
+            tb=tb,
+        ),
+        hide_tb=hide_tb,
+        **_pause_kwargs
+    )
+
+
+async def _maybe_enter_pm(
+    err: BaseException,
+    *,
+    tb: TracebackType|None = None,
+    api_frame: FrameType|None = None,
+    hide_tb: bool = False,
+
+    # only enter debugger REPL when returns `True`
+    debug_filter: Callable[
+        [BaseException|BaseExceptionGroup],
+        bool,
+    ] = lambda err: not is_multi_cancelled(err),
+    **_pause_kws,
+):
+    if (
+        debug_mode()
+
+        # NOTE: don't enter debug mode recursively after quitting pdb
+        # Iow, don't re-enter the repl if the `quit` command was issued
+        # by the user.
+        and not isinstance(err, bdb.BdbQuit)
+
+        # XXX: if the error is the likely result of runtime-wide
+        # cancellation, we don't want to enter the debugger since
+        # there's races between when the parent actor has killed all
+        # comms and when the child tries to contact said parent to
+        # acquire the tty lock.
+
+        # Really we just want to mostly avoid catching KBIs here so there
+        # might be a simpler check we can do?
+        and
+        debug_filter(err)
+    ):
+        api_frame: FrameType = api_frame or inspect.currentframe()
+        tb: TracebackType = tb or sys.exc_info()[2]
+        await post_mortem(
+            api_frame=api_frame,
+            tb=tb,
+            **_pause_kws,
+        )
+        return True
+
+    else:
+        return False
+
+
+# TODO: better naming and what additionals?
+# - [ ] optional runtime plugging?
+# - [ ] detection for sync vs. async code?
+# - [ ] specialized REPL entry when in distributed mode?
+# -[x] hide tb by def
+# - [x] allow ignoring kbi Bo
+@cm
+def open_crash_handler(
+    catch: set[BaseException] = {
+        BaseException,
+    },
+    ignore: set[BaseException] = {
+        KeyboardInterrupt,
+        trio.Cancelled,
+    },
+    hide_tb: bool = True,
+
+    repl_fixture: (
+        AbstractContextManager[bool]  # pre/post REPL entry
+        |None
+    ) = None,
+    raise_on_exit: bool|Sequence[Type[BaseException]] = True,
+):
+    '''
+    Generic "post mortem" crash handler using `pdbp` REPL debugger.
+
+    We expose this as a CLI framework addon to both `click` and
+    `typer` users so they can quickly wrap cmd endpoints which get
+    automatically wrapped to use the runtime's `debug_mode: bool`
+    AND `pdbp.pm()` around any code that is PRE-runtime entry
+    - any sync code which runs BEFORE the main call to
+      `trio.run()`.
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    # TODO, yield a `outcome.Error`-like boxed type?
+    # -[~] use `outcome.Value/Error` X-> frozen!
+    # -[x] write our own..?
+    # -[ ] consider just wtv is used by `pytest.raises()`?
+    #
+    boxed_maybe_exc = BoxedMaybeException(
+        raise_on_exit=raise_on_exit,
+    )
+    err: BaseException
+    try:
+        yield boxed_maybe_exc
+    except tuple(catch) as err:
+        boxed_maybe_exc.value = err
+        if (
+            type(err) not in ignore
+            and
+            not is_multi_cancelled(
+                err,
+                ignore_nested=ignore
+            )
+        ):
+            try:
+                # use our re-impl-ed version of `pdbp.xpm()`
+                _post_mortem(
+                    repl=mk_pdb(),
+                    tb=sys.exc_info()[2],
+                    api_frame=inspect.currentframe().f_back,
+                    hide_tb=hide_tb,
+
+                    repl_fixture=repl_fixture,
+                    boxed_maybe_exc=boxed_maybe_exc,
+                )
+            except bdb.BdbQuit:
+                __tracebackhide__: bool = False
+                raise err
+
+        if (
+            raise_on_exit is True
+            or (
+                raise_on_exit is not False
+                and (
+                    set(raise_on_exit)
+                    and
+                    type(err) in raise_on_exit
+                )
+            )
+            and
+            boxed_maybe_exc.raise_on_exit == raise_on_exit
+        ):
+            raise err
+
+
+@cm
+def maybe_open_crash_handler(
+    pdb: bool|None = None,
+    hide_tb: bool = True,
+
+    **kwargs,
+):
+    '''
+    Same as `open_crash_handler()` but with bool input flag
+    to allow conditional handling.
+
+    Normally this is used with CLI endpoints such that if the --pdb
+    flag is passed the pdb REPL is engaed on any crashes B)
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    if pdb is None:
+        pdb: bool = _state.is_debug_mode()
+
+    rtctx = nullcontext(
+        enter_result=BoxedMaybeException()
+    )
+    if pdb:
+        rtctx = open_crash_handler(
+            hide_tb=hide_tb,
+            **kwargs,
+        )
+
+    with rtctx as boxed_maybe_exc:
+        yield boxed_maybe_exc

tractor/devx/debug/_repl.py

@@ -0,0 +1,207 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+`pdpp.Pdb` extentions/customization and other delegate usage.
+
+'''
+from functools import (
+    cached_property,
+)
+import os
+
+import pdbp
+from tractor._state import (
+    is_root_process,
+)
+
+from ._tty_lock import (
+    Lock,
+    DebugStatus,
+)
+
+
+class TractorConfig(pdbp.DefaultConfig):
+    '''
+    Custom `pdbp` config which tries to use the best tradeoff
+    between pretty and minimal.
+
+    '''
+    use_pygments: bool = True
+    sticky_by_default: bool = False
+    enable_hidden_frames: bool = True
+
+    # much thanks @mdmintz for the hot tip!
+    # fixes line spacing issue when resizing terminal B)
+    truncate_long_lines: bool = False
+
+    # ------ - ------
+    # our own custom config vars mostly
+    # for syncing with the actor tree's singleton
+    # TTY `Lock`.
+
+
+class PdbREPL(pdbp.Pdb):
+    '''
+    Add teardown hooks and local state describing any
+    ongoing TTY `Lock` request dialog.
+
+    '''
+    # override the pdbp config with our coolio one
+    # NOTE: this is only loaded when no `~/.pdbrc` exists
+    # so we should prolly pass it into the .__init__() instead?
+    # i dunno, see the `DefaultFactory` and `pdb.Pdb` impls.
+    DefaultConfig = TractorConfig
+
+    status = DebugStatus
+
+    # NOTE: see details in stdlib's `bdb.py`
+    # def user_exception(self, frame, exc_info):
+    #     '''
+    #     Called when we stop on an exception.
+    #     '''
+    #     log.warning(
+    #         'Exception during REPL sesh\n\n'
+    #         f'{frame}\n\n'
+    #         f'{exc_info}\n\n'
+    #     )
+
+    # NOTE: this actually hooks but i don't see anyway to detect
+    # if an error was caught.. this is why currently we just always
+    # call `DebugStatus.release` inside `_post_mortem()`.
+    # def preloop(self):
+    #     print('IN PRELOOP')
+    #     super().preloop()
+
+    # TODO: cleaner re-wrapping of all this?
+    # -[ ] figure out how to disallow recursive .set_trace() entry
+    #     since that'll cause deadlock for us.
+    # -[ ] maybe a `@cm` to call `super().<same_meth_name>()`?
+    # -[ ] look at hooking into the `pp` hook specially with our
+    #     own set of pretty-printers?
+    #    * `.pretty_struct.Struct.pformat()`
+    #    * `.pformat(MsgType.pld)`
+    #    * `.pformat(Error.tb_str)`?
+    #    * .. maybe more?
+    #
+    def set_continue(self):
+        try:
+            super().set_continue()
+        finally:
+            # NOTE: for subactors the stdio lock is released via the
+            # allocated RPC locker task, so for root we have to do it
+            # manually.
+            if (
+                is_root_process()
+                and
+                Lock._debug_lock.locked()
+                and
+                DebugStatus.is_main_trio_thread()
+            ):
+                # Lock.release(raise_on_thread=False)
+                Lock.release()
+
+            # XXX AFTER `Lock.release()` for root local repl usage
+            DebugStatus.release()
+
+    def set_quit(self):
+        try:
+            super().set_quit()
+        finally:
+            if (
+                is_root_process()
+                and
+                Lock._debug_lock.locked()
+                and
+                DebugStatus.is_main_trio_thread()
+            ):
+                # Lock.release(raise_on_thread=False)
+                Lock.release()
+
+            # XXX after `Lock.release()` for root local repl usage
+            DebugStatus.release()
+
+    # XXX NOTE: we only override this because apparently the stdlib pdb
+    # bois likes to touch the SIGINT handler as much as i like to touch
+    # my d$%&.
+    def _cmdloop(self):
+        self.cmdloop()
+
+    @cached_property
+    def shname(self) -> str | None:
+        '''
+        Attempt to return the login shell name with a special check for
+        the infamous `xonsh` since it seems to have some issues much
+        different from std shells when it comes to flushing the prompt?
+
+        '''
+        # SUPER HACKY and only really works if `xonsh` is not used
+        # before spawning further sub-shells..
+        shpath = os.getenv('SHELL', None)
+
+        if shpath:
+            if (
+                os.getenv('XONSH_LOGIN', default=False)
+                or 'xonsh' in shpath
+            ):
+                return 'xonsh'
+
+            return os.path.basename(shpath)
+
+        return None
+
+
+def mk_pdb() -> PdbREPL:
+    '''
+    Deliver a new `PdbREPL`: a multi-process safe `pdbp.Pdb`-variant
+    using the magic of `tractor`'s SC-safe IPC.
+
+    B)
+
+    Our `pdb.Pdb` subtype accomplishes multi-process safe debugging
+    by:
+
+    - mutexing access to the root process' std-streams (& thus parent
+      process TTY) via an IPC managed `Lock` singleton per
+      actor-process tree.
+
+    - temporarily overriding any subactor's SIGINT handler to shield
+      during live REPL sessions in sub-actors such that cancellation
+      is never (mistakenly) triggered by a ctrl-c and instead only by
+      explicit runtime API requests or after the
+      `pdb.Pdb.interaction()` call has returned.
+
+    FURTHER, the `pdbp.Pdb` instance is configured to be `trio`
+    "compatible" from a SIGINT handling perspective; we mask out
+    the default `pdb` handler and instead apply `trio`s default
+    which mostly addresses all issues described in:
+
+     - https://github.com/python-trio/trio/issues/1155
+
+    The instance returned from this factory should always be
+    preferred over the default `pdb[p].set_trace()` whenever using
+    a `pdb` REPL inside a `trio` based runtime.
+
+    '''
+    pdb = PdbREPL()
+
+    # XXX: These are the important flags mentioned in
+    # https://github.com/python-trio/trio/issues/1155
+    # which resolve the traceback spews to console.
+    pdb.allow_kbdint = True
+    pdb.nosigint = True
+    return pdb

tractor/devx/debug/_sigint.py

@@ -0,0 +1,333 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+A custom SIGINT handler which mainly shields actor (task)
+cancellation during REPL interaction.
+
+'''
+from __future__ import annotations
+from typing import (
+    TYPE_CHECKING,
+)
+import trio
+from tractor.log import get_logger
+from tractor._state import (
+    current_actor,
+    is_root_process,
+)
+from ._repl import (
+    PdbREPL,
+)
+from ._tty_lock import (
+    any_connected_locker_child,
+    DebugStatus,
+    Lock,
+)
+
+if TYPE_CHECKING:
+    from tractor.ipc import (
+        Channel,
+    )
+    from tractor._runtime import (
+        Actor,
+    )
+
+log = get_logger(__name__)
+
+_ctlc_ignore_header: str = (
+    'Ignoring SIGINT while debug REPL in use'
+)
+
+
+def sigint_shield(
+    signum: int,
+    frame: 'frame',  # type: ignore # noqa
+    *args,
+
+) -> None:
+    '''
+    Specialized, debugger-aware SIGINT handler.
+
+    In childred we always ignore/shield for SIGINT to avoid
+    deadlocks since cancellation should always be managed by the
+    supervising parent actor. The root actor-proces is always
+    cancelled on ctrl-c.
+
+    '''
+    __tracebackhide__: bool = True
+    actor: Actor = current_actor()
+
+    def do_cancel():
+        # If we haven't tried to cancel the runtime then do that instead
+        # of raising a KBI (which may non-gracefully destroy
+        # a ``trio.run()``).
+        if not actor._cancel_called:
+            actor.cancel_soon()
+
+        # If the runtime is already cancelled it likely means the user
+        # hit ctrl-c again because teardown didn't fully take place in
+        # which case we do the "hard" raising of a local KBI.
+        else:
+            raise KeyboardInterrupt
+
+    # only set in the actor actually running the REPL
+    repl: PdbREPL|None = DebugStatus.repl
+
+    # TODO: maybe we should flatten out all these cases using
+    # a match/case?
+    #
+    # root actor branch that reports whether or not a child
+    # has locked debugger.
+    if is_root_process():
+        # log.warning(
+        log.devx(
+            'Handling SIGINT in root actor\n'
+            f'{Lock.repr()}'
+            f'{DebugStatus.repr()}\n'
+        )
+        # try to see if the supposed (sub)actor in debug still
+        # has an active connection to *this* actor, and if not
+        # it's likely they aren't using the TTY lock / debugger
+        # and we should propagate SIGINT normally.
+        any_connected: bool = any_connected_locker_child()
+
+        problem = (
+            f'root {actor.uid} handling SIGINT\n'
+            f'any_connected: {any_connected}\n\n'
+
+            f'{Lock.repr()}\n'
+        )
+
+        if (
+            (ctx := Lock.ctx_in_debug)
+            and
+            (uid_in_debug := ctx.chan.uid) # "someone" is (ostensibly) using debug `Lock`
+        ):
+            name_in_debug: str = uid_in_debug[0]
+            assert not repl
+            # if not repl:  # but it's NOT us, the root actor.
+            # sanity: since no repl ref is set, we def shouldn't
+            # be the lock owner!
+            assert name_in_debug != 'root'
+
+            # IDEAL CASE: child has REPL as expected
+            if any_connected:  # there are subactors we can contact
+                # XXX: only if there is an existing connection to the
+                # (sub-)actor in debug do we ignore SIGINT in this
+                # parent! Otherwise we may hang waiting for an actor
+                # which has already terminated to unlock.
+                #
+                # NOTE: don't emit this with `.pdb()` level in
+                # root without a higher level.
+                log.runtime(
+                    _ctlc_ignore_header
+                    +
+                    f' by child '
+                    f'{uid_in_debug}\n'
+                )
+                problem = None
+
+            else:
+                problem += (
+                    '\n'
+                    f'A `pdb` REPL is SUPPOSEDLY in use by child {uid_in_debug}\n'
+                    f'BUT, no child actors are IPC contactable!?!?\n'
+                )
+
+        # IDEAL CASE: root has REPL as expected
+        else:
+            # root actor still has this SIGINT handler active without
+            # an actor using the `Lock` (a bug state) ??
+            # => so immediately cancel any stale lock cs and revert
+            # the handler!
+            if not DebugStatus.repl:
+                # TODO: WHEN should we revert back to ``trio``
+                # handler if this one is stale?
+                # -[ ] maybe after a counts work of ctl-c mashes?
+                # -[ ] use a state var like `stale_handler: bool`?
+                problem += (
+                    'No subactor is using a `pdb` REPL according `Lock.ctx_in_debug`?\n'
+                    'BUT, the root should be using it, WHY this handler ??\n\n'
+                    'So either..\n'
+                    '- some root-thread is using it but has no `.repl` set?, OR\n'
+                    '- something else weird is going on outside the runtime!?\n'
+                )
+            else:
+                # NOTE: since we emit this msg on ctl-c, we should
+                # also always re-print the prompt the tail block!
+                log.pdb(
+                    _ctlc_ignore_header
+                    +
+                    f' by root actor..\n'
+                    f'{DebugStatus.repl_task}\n'
+                    f' |_{repl}\n'
+                )
+                problem = None
+
+        # XXX if one is set it means we ARE NOT operating an ideal
+        # case where a child subactor or us (the root) has the
+        # lock without any other detected problems.
+        if problem:
+
+            # detect, report and maybe clear a stale lock request
+            # cancel scope.
+            lock_cs: trio.CancelScope = Lock.get_locking_task_cs()
+            maybe_stale_lock_cs: bool = (
+                lock_cs is not None
+                and not lock_cs.cancel_called
+            )
+            if maybe_stale_lock_cs:
+                problem += (
+                    '\n'
+                    'Stale `Lock.ctx_in_debug._scope: CancelScope` detected?\n'
+                    f'{Lock.ctx_in_debug}\n\n'
+
+                    '-> Calling ctx._scope.cancel()!\n'
+                )
+                lock_cs.cancel()
+
+            # TODO: wen do we actually want/need this, see above.
+            # DebugStatus.unshield_sigint()
+            log.warning(problem)
+
+    # child actor that has locked the debugger
+    elif not is_root_process():
+        log.debug(
+            f'Subactor {actor.uid} handling SIGINT\n\n'
+            f'{Lock.repr()}\n'
+        )
+
+        rent_chan: Channel = actor._parent_chan
+        if (
+            rent_chan is None
+            or
+            not rent_chan.connected()
+        ):
+            log.warning(
+                'This sub-actor thinks it is debugging '
+                'but it has no connection to its parent ??\n'
+                f'{actor.uid}\n'
+                'Allowing SIGINT propagation..'
+            )
+            DebugStatus.unshield_sigint()
+
+        repl_task: str|None = DebugStatus.repl_task
+        req_task: str|None = DebugStatus.req_task
+        if (
+            repl_task
+            and
+            repl
+        ):
+            log.pdb(
+                _ctlc_ignore_header
+                +
+                f' by local task\n\n'
+                f'{repl_task}\n'
+                f' |_{repl}\n'
+            )
+        elif req_task:
+            log.debug(
+                _ctlc_ignore_header
+                +
+                f' by local request-task and either,\n'
+                f'- someone else is already REPL-in and has the `Lock`, or\n'
+                f'- some other local task already is replin?\n\n'
+                f'{req_task}\n'
+            )
+
+        # TODO can we remove this now?
+        # -[ ] does this path ever get hit any more?
+        else:
+            msg: str = (
+                'SIGINT shield handler still active BUT, \n\n'
+            )
+            if repl_task is None:
+                msg += (
+                    '- No local task claims to be in debug?\n'
+                )
+
+            if repl is None:
+                msg += (
+                    '- No local REPL is currently active?\n'
+                )
+
+            if req_task is None:
+                msg += (
+                    '- No debug request task is active?\n'
+                )
+
+            log.warning(
+                msg
+                +
+                'Reverting handler to `trio` default!\n'
+            )
+            DebugStatus.unshield_sigint()
+
+            # XXX ensure that the reverted-to-handler actually is
+            # able to rx what should have been **this** KBI ;)
+            do_cancel()
+
+        # TODO: how to handle the case of an intermediary-child actor
+        # that **is not** marked in debug mode? See oustanding issue:
+        # https://github.com/goodboy/tractor/issues/320
+        # elif debug_mode():
+
+    # maybe redraw/print last REPL output to console since
+    # we want to alert the user that more input is expect since
+    # nothing has been done dur to ignoring sigint.
+    if (
+        DebugStatus.repl  # only when current actor has a REPL engaged
+    ):
+        flush_status: str = (
+            'Flushing stdout to ensure new prompt line!\n'
+        )
+
+        # XXX: yah, mega hack, but how else do we catch this madness XD
+        if (
+            repl.shname == 'xonsh'
+        ):
+            flush_status += (
+                '-> ALSO re-flushing due to `xonsh`..\n'
+            )
+            repl.stdout.write(repl.prompt)
+
+        # log.warning(
+        log.devx(
+            flush_status
+        )
+        repl.stdout.flush()
+
+        # TODO: better console UX to match the current "mode":
+        # -[ ] for example if in sticky mode where if there is output
+        #   detected as written to the tty we redraw this part underneath
+        #   and erase the past draw of this same bit above?
+        # repl.sticky = True
+        # repl._print_if_sticky()
+
+        # also see these links for an approach from `ptk`:
+        # https://github.com/goodboy/tractor/issues/130#issuecomment-663752040
+        # https://github.com/prompt-toolkit/python-prompt-toolkit/blob/c2c6af8a0308f9e5d7c0e28cb8a02963fe0ce07a/prompt_toolkit/patch_stdout.py
+    else:
+        log.devx(
+        # log.warning(
+            'Not flushing stdout since not needed?\n'
+            f'|_{repl}\n'
+        )
+
+    # XXX only for tracing this handler
+    log.devx('exiting SIGINT')

tractor/devx/debug/_sync.py

@@ -0,0 +1,220 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+Debugger synchronization APIs to ensure orderly access and
+non-TTY-clobbering graceful teardown.
+
+
+'''
+from __future__ import annotations
+from contextlib import (
+    asynccontextmanager as acm,
+)
+from functools import (
+    partial,
+)
+from typing import (
+    AsyncGenerator,
+    Callable,
+)
+
+from tractor.log import get_logger
+import trio
+from trio.lowlevel import (
+    current_task,
+    Task,
+)
+from tractor._context import Context
+from tractor._state import (
+    current_actor,
+    debug_mode,
+    is_root_process,
+)
+from ._repl import (
+    TractorConfig as TractorConfig,
+)
+from ._tty_lock import (
+    Lock,
+    request_root_stdio_lock,
+    any_connected_locker_child,
+)
+from ._sigint import (
+    sigint_shield as sigint_shield,
+    _ctlc_ignore_header as _ctlc_ignore_header
+)
+
+log = get_logger(__package__)
+
+
+async def maybe_wait_for_debugger(
+    poll_steps: int = 2,
+    poll_delay: float = 0.1,
+    child_in_debug: bool = False,
+
+    header_msg: str = '',
+    _ll: str = 'devx',
+
+) -> bool:  # was locked and we polled?
+
+    if (
+        not debug_mode()
+        and
+        not child_in_debug
+    ):
+        return False
+
+    logmeth: Callable = getattr(log, _ll)
+
+    msg: str = header_msg
+    if (
+        is_root_process()
+    ):
+        # If we error in the root but the debugger is
+        # engaged we don't want to prematurely kill (and
+        # thus clobber access to) the local tty since it
+        # will make the pdb repl unusable.
+        # Instead try to wait for pdb to be released before
+        # tearing down.
+        ctx_in_debug: Context|None = Lock.ctx_in_debug
+        in_debug: tuple[str, str]|None = (
+            ctx_in_debug.chan.uid
+            if ctx_in_debug
+            else None
+        )
+        if in_debug == current_actor().uid:
+            log.debug(
+                msg
+                +
+                'Root already owns the TTY LOCK'
+            )
+            return True
+
+        elif in_debug:
+            msg += (
+                f'Debug `Lock` in use by subactor\n|\n|_{in_debug}\n'
+            )
+            # TODO: could this make things more deterministic?
+            # wait to see if a sub-actor task will be
+            # scheduled and grab the tty lock on the next
+            # tick?
+            # XXX => but it doesn't seem to work..
+            # await trio.testing.wait_all_tasks_blocked(cushion=0)
+        else:
+            logmeth(
+                msg
+                +
+                'Root immediately acquired debug TTY LOCK'
+            )
+            return False
+
+        for istep in range(poll_steps):
+            if (
+                Lock.req_handler_finished is not None
+                and not Lock.req_handler_finished.is_set()
+                and in_debug is not None
+            ):
+                # caller_frame_info: str = pformat_caller_frame()
+                logmeth(
+                    msg
+                    +
+                    '\n^^ Root is waiting on tty lock release.. ^^\n'
+                    # f'{caller_frame_info}\n'
+                )
+
+                if not any_connected_locker_child():
+                    Lock.get_locking_task_cs().cancel()
+
+                with trio.CancelScope(shield=True):
+                    await Lock.req_handler_finished.wait()
+
+                log.devx(
+                    f'Subactor released debug lock\n'
+                    f'|_{in_debug}\n'
+                )
+                break
+
+            # is no subactor locking debugger currently?
+            if (
+                in_debug is None
+                and (
+                    Lock.req_handler_finished is None
+                    or Lock.req_handler_finished.is_set()
+                )
+            ):
+                logmeth(
+                    msg
+                    +
+                    'Root acquired tty lock!'
+                )
+                break
+
+            else:
+                logmeth(
+                    'Root polling for debug:\n'
+                    f'poll step: {istep}\n'
+                    f'poll delya: {poll_delay}\n\n'
+                    f'{Lock.repr()}\n'
+                )
+                with trio.CancelScope(shield=True):
+                    await trio.sleep(poll_delay)
+                    continue
+
+        return True
+
+    # else:
+    #     # TODO: non-root call for #320?
+    #     this_uid: tuple[str, str] = current_actor().uid
+    #     async with acquire_debug_lock(
+    #         subactor_uid=this_uid,
+    #     ):
+    #         pass
+    return False
+
+
+@acm
+async def acquire_debug_lock(
+    subactor_uid: tuple[str, str],
+) -> AsyncGenerator[
+    trio.CancelScope|None,
+    tuple,
+]:
+    '''
+    Request to acquire the TTY `Lock` in the root actor, release on
+    exit.
+
+    This helper is for actor's who don't actually need to acquired
+    the debugger but want to wait until the lock is free in the
+    process-tree root such that they don't clobber an ongoing pdb
+    REPL session in some peer or child!
+
+    '''
+    if not debug_mode():
+        yield None
+        return
+
+    task: Task = current_task()
+    async with trio.open_nursery() as n:
+        ctx: Context = await n.start(
+            partial(
+                request_root_stdio_lock,
+                actor_uid=subactor_uid,
+                task_uid=(task.name, id(task)),
+            )
+        )
+        yield ctx
+        ctx.cancel()

tractor/devx/pformat.py

@@ -247,3 +247,113 @@ def pformat_cs(
         +
         fields
     )
+
+
+# TODO: move this func to some kinda `.devx.pformat.py` eventually
+# as we work out our multi-domain state-flow-syntax!
+def nest_from_op(
+    input_op: str,
+    #
+    # ?TODO? an idea for a syntax to the state of concurrent systems
+    # as a "3-domain" (execution, scope, storage) model and using
+    # a minimal ascii/utf-8 operator-set.
+    #
+    # try not to take any of this seriously yet XD
+    #
+    # > is a "play operator" indicating (CPU bound)
+    #   exec/work/ops required at the "lowest level computing"
+    #
+    # execution primititves (tasks, threads, actors..) denote their
+    # lifetime with '(' and ')' since parentheses normally are used
+    # in many langs to denote function calls.
+    #
+    # starting = (
+    # >(  opening/starting; beginning of the thread-of-exec (toe?)
+    # (>  opened/started,  (finished spawning toe)
+    # |_<Task: blah blah..>  repr of toe, in py these look like <objs>
+    #
+    # >) closing/exiting/stopping,
+    # )> closed/exited/stopped,
+    # |_<Task: blah blah..>
+    #   [OR <), )< ?? ]
+    #
+    # ending = )
+    # >c) cancelling to close/exit
+    # c)> cancelled (caused close), OR?
+    #  |_<Actor: ..>
+    #   OR maybe "<c)" which better indicates the cancel being
+    #   "delivered/returned" / returned" to LHS?
+    #
+    # >x)  erroring to eventuall exit
+    # x)>  errored and terminated
+    #  |_<Actor: ...>
+    #
+    # scopes: supers/nurseries, IPC-ctxs, sessions, perms, etc.
+    # >{  opening
+    # {>  opened
+    # }>  closed
+    # >}  closing
+    #
+    # storage: like queues, shm-buffers, files, etc..
+    # >[  opening
+    # [>  opened
+    #  |_<FileObj: ..>
+    #
+    # >]  closing
+    # ]>  closed
+
+    # IPC ops: channels, transports, msging
+    # =>  req msg
+    # <=  resp msg
+    # <=> 2-way streaming (of msgs)
+    # <-  recv 1 msg
+    # ->  send 1 msg
+    #
+    # TODO: still not sure on R/L-HS approach..?
+    # =>(  send-req to exec start (task, actor, thread..)
+    # (<=  recv-req to ^
+    #
+    # (<=  recv-req ^
+    # <=(  recv-resp opened remote exec primitive
+    # <=)  recv-resp closed
+    #
+    # )<=c req to stop due to cancel
+    # c=>) req to stop due to cancel
+    #
+    # =>{  recv-req to open
+    # <={  send-status that it closed
+
+    tree_str: str,
+
+    # NOTE: so move back-from-the-left of the `input_op` by
+    # this amount.
+    back_from_op: int = 0,
+    nest_prefix: str = ''
+
+) -> str:
+    '''
+    Depth-increment the input (presumably hierarchy/supervision)
+    input "tree string" below the provided `input_op` execution
+    operator, so injecting a `"\n|_{input_op}\n"`and indenting the
+    `tree_str` to nest content aligned with the ops last char.
+
+    '''
+    indented_tree_str: str = textwrap.indent(
+        tree_str,
+        prefix=' ' *(
+            len(input_op)
+            -
+            (back_from_op + 1)
+        ),
+    )
+    # inject any provided nesting-prefix chars
+    # into the head of the first line.
+    if nest_prefix:
+        indented_tree_str: str = (
+            f'{nest_prefix}'
+            f'{indented_tree_str[len(nest_prefix):]}'
+        )
+    return (
+        f'{input_op}\n'
+        f'{indented_tree_str}'
+    )

tractor/ipc/_server.py

@@ -43,7 +43,7 @@ from trio import (
     SocketListener,
 )
 
-# from ..devx import _debug
+# from ..devx import debug
 from .._exceptions import (
     TransportClosed,
 )
@@ -318,7 +318,7 @@ async def handle_stream_from_peer(
     server._no_more_peers = trio.Event()  # unset by making new
 
     # TODO, debug_mode tooling for when hackin this lower layer?
-    # with _debug.maybe_open_crash_handler(
+    # with debug.maybe_open_crash_handler(
     #     pdb=True,
     # ) as boxerr:
 
@@ -504,8 +504,8 @@ async def handle_stream_from_peer(
                 and
                 _state.is_debug_mode()
             ):
-                from ..devx import _debug
-                pdb_lock = _debug.Lock
+                from ..devx import debug
+                pdb_lock = debug.Lock
                 pdb_lock._blocked.add(uid)
 
                 # TODO: NEEEDS TO BE TESTED!
@@ -540,7 +540,7 @@ async def handle_stream_from_peer(
                                 f'last disconnected child uid: {uid}\n'
                                 f'locking child uid: {pdb_user_uid}\n'
                             )
-                            await _debug.maybe_wait_for_debugger(
+                            await debug.maybe_wait_for_debugger(
                                 child_in_debug=True
                             )
 

tractor/ipc/_tcp.py

@@ -18,7 +18,6 @@ TCP implementation of tractor.ipc._transport.MsgTransport protocol
 
 '''
 from __future__ import annotations
-import ipaddress
 from typing import (
     ClassVar,
 )
@@ -51,45 +50,13 @@ class TCPAddress(
     _host: str
     _port: int
 
-    def __post_init__(self):
-        try:
-            ipaddress.ip_address(self._host)
-        except ValueError as valerr:
-            raise ValueError(
-                'Invalid {type(self).__name__}._host = {self._host!r}\n'
-            ) from valerr
-
     proto_key: ClassVar[str] = 'tcp'
     unwrapped_type: ClassVar[type] = tuple[str, int]
     def_bindspace: ClassVar[str] = '127.0.0.1'
 
-    # ?TODO, actually validate ipv4/6 with stdlib's `ipaddress`
     @property
     def is_valid(self) -> bool:
-        '''
-        Predicate to ensure a valid socket-address pair.
-
-        '''
-        return (
-            self._port != 0
-            and
-            (ipaddr := ipaddress.ip_address(self._host))
-            and not (
-                ipaddr.is_reserved
-                or
-                ipaddr.is_unspecified
-                or
-                ipaddr.is_link_local
-                or
-                ipaddr.is_link_local
-                or
-                ipaddr.is_multicast
-                or
-                ipaddr.is_global
-            )
-        )
-        # ^XXX^ see various properties of invalid addrs here,
-        # https://docs.python.org/3/library/ipaddress.html#ipaddress.IPv4Address
+        return self._port != 0
 
     @property
     def bindspace(self) -> str:

tractor/log.py

@@ -270,7 +270,9 @@ def get_logger(
     subsys_spec: str|None = None,
 
 ) -> StackLevelAdapter:
-    '''Return the package log or a sub-logger for ``name`` if provided.
+    '''
+    Return the `tractor`-library root logger or a sub-logger for
+    `name` if provided.
 
     '''
     log: Logger
@@ -282,7 +284,7 @@ def get_logger(
         name != _proj_name
     ):
 
-        # NOTE: for handling for modules that use ``get_logger(__name__)``
+        # NOTE: for handling for modules that use `get_logger(__name__)`
         # we make the following stylistic choice:
         # - always avoid duplicate project-package token
         #   in msg output: i.e. tractor.tractor.ipc._chan.py in header
@@ -331,7 +333,7 @@ def get_logger(
 
 def get_console_log(
     level: str|None = None,
-    logger: Logger|None = None,
+    logger: Logger|StackLevelAdapter|None = None,
     **kwargs,
 
 ) -> LoggerAdapter:
@@ -344,12 +346,23 @@ def get_console_log(
     Yeah yeah, i know we can use `logging.config.dictConfig()`. You do it.
 
     '''
-    log = get_logger(
-        logger=logger,
-        **kwargs
-    )  # set a root logger
-    logger: Logger = log.logger
+    # get/create a stack-aware-adapter
+    if (
+        logger
+        and
+        isinstance(logger, StackLevelAdapter)
+    ):
+        # XXX, for ex. when passed in by a caller wrapping some
+        # other lib's logger instance with our level-adapter.
+        log = logger
 
+    else:
+        log: StackLevelAdapter = get_logger(
+            logger=logger,
+            **kwargs
+        )
+
+    logger: Logger|StackLevelAdapter = log.logger
     if not level:
         return log
 
@@ -367,10 +380,7 @@ def get_console_log(
             None,
         )
     ):
-        fmt = LOG_FORMAT
-        # if logger:
-        #     fmt = None
-
+        fmt: str = LOG_FORMAT  # always apply our format?
         handler = StreamHandler()
         formatter = colorlog.ColoredFormatter(
             fmt=fmt,

tractor/msg/_ops.py

@@ -608,7 +608,7 @@ async def drain_to_final_msg(
             #
             # -[ ] make sure pause points work here for REPLing
             #   the runtime itself; i.e. ensure there's no hangs!
-            # |_from tractor.devx._debug import pause
+            # |_from tractor.devx.debug import pause
             #   await pause()
 
         # NOTE: we get here if the far end was

tractor/msg/types.py

@@ -170,7 +170,6 @@ class SpawnSpec(
     # a hard `Struct` def for all of these fields!
     _parent_main_data: dict
     _runtime_vars: dict[str, Any]
-    # ^NOTE see `._state._runtime_vars: dict`
 
     # module import capability
     enable_modules: dict[str, str]

tractor/to_asyncio.py

@@ -38,7 +38,6 @@ from typing import (
 import tractor
 from tractor._exceptions import (
     InternalError,
-    is_multi_cancelled,
     TrioTaskExited,
     TrioCancelled,
     AsyncioTaskExited,
@@ -49,7 +48,7 @@ from tractor._state import (
     _runtime_vars,
 )
 from tractor._context import Unresolved
-from tractor.devx import _debug
+from tractor.devx import debug
 from tractor.log import (
     get_logger,
     StackLevelAdapter,
@@ -59,6 +58,9 @@ from tractor.log import (
 # from tractor.msg import (
 #     pretty_struct,
 # )
+from tractor.trionics import (
+    is_multi_cancelled,
+)
 from tractor.trionics._broadcast import (
     broadcast_receiver,
     BroadcastReceiver,
@@ -479,12 +481,12 @@ def _run_asyncio_task(
     if (
         debug_mode()
         and
-        (greenback := _debug.maybe_import_greenback(
+        (greenback := debug.maybe_import_greenback(
             force_reload=True,
             raise_not_found=False,
         ))
     ):
-        log.info(
+        log.devx(
             f'Bestowing `greenback` portal for `asyncio`-task\n'
             f'{task}\n'
         )
@@ -841,7 +843,7 @@ async def translate_aio_errors(
     except BaseException as _trio_err:
         trio_err = chan._trio_err = _trio_err
         # await tractor.pause(shield=True)  # workx!
-        entered: bool = await _debug._maybe_enter_pm(
+        entered: bool = await debug._maybe_enter_pm(
             trio_err,
             api_frame=inspect.currentframe(),
         )
@@ -1406,7 +1408,7 @@ def run_as_asyncio_guest(
             )
             # XXX make it obvi we know this isn't supported yet!
             assert 0
-            # await _debug.maybe_init_greenback(
+            # await debug.maybe_init_greenback(
             #     force_reload=True,
             # )
 

tractor/trionics/__init__.py

@@ -31,4 +31,9 @@ from ._broadcast import (
 )
 from ._beg import (
     collapse_eg as collapse_eg,
+    maybe_collapse_eg as maybe_collapse_eg,
+    is_multi_cancelled as is_multi_cancelled,
+)
+from ._taskc import (
+    maybe_raise_from_masking_exc as maybe_raise_from_masking_exc,
 )

tractor/trionics/_beg.py

@@ -22,6 +22,11 @@ first-class-`trio` from a historical perspective B)
 from contextlib import (
     asynccontextmanager as acm,
 )
+from typing import (
+    Literal,
+)
+
+import trio
 
 
 def maybe_collapse_eg(
@@ -39,7 +44,10 @@ def maybe_collapse_eg(
 
 
 @acm
-async def collapse_eg():
+async def collapse_eg(
+    hide_tb: bool = True,
+    raise_from_src: bool = False,
+):
     '''
     If `BaseExceptionGroup` raised in the body scope is
     "collapse-able" (in the same way that
@@ -47,12 +55,73 @@ async def collapse_eg():
     only raise the lone emedded non-eg in in place.
 
     '''
+    __tracebackhide__: bool = hide_tb
     try:
         yield
     except* BaseException as beg:
         if (
             exc := maybe_collapse_eg(beg)
         ) is not beg:
-            raise exc
+            from_exc = beg if raise_from_src else None
+            raise exc from from_exc
 
         raise beg
+
+
+def is_multi_cancelled(
+    beg: BaseException|BaseExceptionGroup,
+
+    ignore_nested: set[BaseException] = set(),
+
+) -> Literal[False]|BaseExceptionGroup:
+    '''
+    Predicate to determine if an `BaseExceptionGroup` only contains
+    some (maybe nested) set of sub-grouped exceptions (like only
+    `trio.Cancelled`s which get swallowed silently by default) and is
+    thus the result of "gracefully cancelling" a collection of
+    sub-tasks (or other conc primitives) and receiving a "cancelled
+    ACK" from each after termination.
+
+    Docs:
+    ----
+    - https://docs.python.org/3/library/exceptions.html#exception-groups
+    - https://docs.python.org/3/library/exceptions.html#BaseExceptionGroup.subgroup
+
+    '''
+
+    if (
+        not ignore_nested
+        or
+        trio.Cancelled not in ignore_nested
+        # XXX always count-in `trio`'s native signal
+    ):
+        ignore_nested.update({trio.Cancelled})
+
+    if isinstance(beg, BaseExceptionGroup):
+        # https://docs.python.org/3/library/exceptions.html#BaseExceptionGroup.subgroup
+        # |_ "The condition can be an exception type or tuple of
+        #   exception types, in which case each exception is checked
+        #   for a match using the same check that is used in an
+        #   except clause. The condition can also be a callable
+        #   (other than a type object) that accepts an exception as
+        #   its single argument and returns true for the exceptions
+        #   that should be in the subgroup."
+        matched_exc: BaseExceptionGroup|None = beg.subgroup(
+            tuple(ignore_nested),
+
+            # ??TODO, complain about why not allowed to use
+            # named arg style calling???
+            # XD .. wtf?
+            # condition=tuple(ignore_nested),
+        )
+        if matched_exc is not None:
+            return matched_exc
+
+    # NOTE, IFF no excs types match (throughout the error-tree)
+    # -> return `False`, OW return the matched sub-eg.
+    #
+    # IOW, for the inverse of ^ for the purpose of
+    # maybe-enter-REPL--logic: "only debug when the err-tree contains
+    # at least one exc-type NOT in `ignore_nested`" ; i.e. the case where
+    # we fallthrough and return `False` here.
+    return False

tractor/trionics/_mngrs.py

@@ -40,6 +40,8 @@ from typing import (
 import trio
 from tractor._state import current_actor
 from tractor.log import get_logger
+from ._beg import collapse_eg
+
 
 if TYPE_CHECKING:
     from tractor import ActorNursery
@@ -70,8 +72,7 @@ async def maybe_open_nursery(
         yield nursery
     else:
         async with lib.open_nursery(**kwargs) as nursery:
-            if lib == trio:
-                nursery.cancel_scope.shield = shield
+            nursery.cancel_scope.shield = shield
             yield nursery
 
 
@@ -112,17 +113,19 @@ async def gather_contexts(
     None,
 ]:
     '''
-    Concurrently enter a sequence of async context managers (acms),
-    each from a separate `trio` task and deliver the unwrapped
-    `yield`-ed values in the same order once all managers have entered.
+    Concurrently enter a sequence of async context managers (`acm`s),
+    each scheduled in a separate `trio.Task` and deliver their
+    unwrapped `yield`-ed values in the same order once all `@acm`s
+    in every task have entered.
 
-    On exit, all acms are subsequently and concurrently exited.
+    On exit, all `acm`s are subsequently and concurrently exited with
+    **no order guarantees**.
 
     This function is somewhat similar to a batch of non-blocking
     calls to `contextlib.AsyncExitStack.enter_async_context()`
     (inside a loop) *in combo with* a `asyncio.gather()` to get the
     `.__aenter__()`-ed values, except the managers are both
-    concurrently entered and exited and *cancellation just works*(R).
+    concurrently entered and exited and *cancellation-just-works™*.
 
     '''
     seed: int = id(mngrs)
@@ -142,16 +145,15 @@ async def gather_contexts(
     if not mngrs:
         raise ValueError(
             '`.trionics.gather_contexts()` input mngrs is empty?\n'
+            '\n'
             'Did try to use inline generator syntax?\n'
-            'Use a non-lazy iterator or sequence type intead!'
+            'Use a non-lazy iterator or sequence-type intead!\n'
         )
 
-    async with trio.open_nursery(
-        strict_exception_groups=False,
-        # ^XXX^ TODO? soo roll our own then ??
-        # -> since we kinda want the "if only one `.exception` then
-        # just raise that" interface?
-    ) as tn:
+    async with (
+        collapse_eg(),
+        trio.open_nursery() as tn,
+    ):
         for mngr in mngrs:
             tn.start_soon(
                 _enter_and_wait,
@@ -168,7 +170,7 @@ async def gather_contexts(
         try:
             yield tuple(unwrapped.values())
         finally:
-            # NOTE: this is ABSOLUTELY REQUIRED to avoid
+            # XXX NOTE: this is ABSOLUTELY REQUIRED to avoid
             # the following wacky bug:
             # <tractorbugurlhere>
             parent_exit.set()

tractor/trionics/_taskc.py

@@ -0,0 +1,182 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+`trio.Task` cancellation helpers, extensions and "holsters".
+
+'''
+from __future__ import annotations
+from contextlib import (
+    asynccontextmanager as acm,
+)
+from typing import TYPE_CHECKING
+
+import trio
+from tractor.log import get_logger
+
+log = get_logger(__name__)
+
+
+if TYPE_CHECKING:
+    from tractor.devx.debug import BoxedMaybeException
+
+
+def find_masked_excs(
+    maybe_masker: BaseException,
+    unmask_from: set[BaseException],
+) -> BaseException|None:
+    ''''
+    Deliver any `maybe_masker.__context__` provided
+    it a declared masking exc-type entry in `unmask_from`.
+
+    '''
+    if (
+        type(maybe_masker) in unmask_from
+        and
+        (exc_ctx := maybe_masker.__context__)
+
+        # TODO? what about any cases where
+        # they could be the same type but not same instance?
+        # |_i.e. a cancel masking a cancel ??
+        # or (
+        #     exc_ctx is not maybe_masker
+        # )
+    ):
+        return exc_ctx
+
+    return None
+
+
+@acm
+async def maybe_raise_from_masking_exc(
+    tn: trio.Nursery|None = None,
+    unmask_from: (
+        BaseException|
+        tuple[BaseException]
+    ) = (trio.Cancelled,),
+
+    raise_unmasked: bool = True,
+    extra_note: str = (
+        'This can occurr when,\n'
+        ' - a `trio.Nursery` scope embeds a `finally:`-block '
+        'which executes a checkpoint!'
+        #
+        # ^TODO? other cases?
+    ),
+
+    always_warn_on: tuple[BaseException] = (
+        trio.Cancelled,
+    ),
+    # ^XXX, special case(s) where we warn-log bc likely
+    # there will be no operational diff since the exc
+    # is always expected to be consumed.
+) -> BoxedMaybeException:
+    '''
+    Maybe un-mask and re-raise exception(s) suppressed by a known
+    error-used-as-signal type (cough namely `trio.Cancelled`).
+
+    Though this unmasker targets cancelleds, it can be used more
+    generally to capture and unwrap masked excs detected as
+    `.__context__` values which were suppressed by any error type
+    passed in `unmask_from`.
+
+    -------------
+    STILL-TODO ??
+    -------------
+    -[ ] support for egs which have multiple masked entries in
+        `maybe_eg.exceptions`, in which case we should unmask the
+        individual sub-excs but maintain the eg-parent's form right?
+
+    '''
+    from tractor.devx.debug import (
+        BoxedMaybeException,
+        pause,
+    )
+    boxed_maybe_exc = BoxedMaybeException(
+        raise_on_exit=raise_unmasked,
+    )
+    matching: list[BaseException]|None = None
+    maybe_eg: ExceptionGroup|None
+    maybe_eg: ExceptionGroup|None
+
+    if tn:
+        try:  # handle egs
+            yield boxed_maybe_exc
+            return
+        except* unmask_from as _maybe_eg:
+            maybe_eg = _maybe_eg
+            matches: ExceptionGroup
+            matches, _ = maybe_eg.split(
+                unmask_from
+            )
+            if not matches:
+                raise
+
+            matching: list[BaseException] = matches.exceptions
+    else:
+        try:  # handle non-egs
+            yield boxed_maybe_exc
+            return
+        except unmask_from as _maybe_exc:
+            maybe_exc = _maybe_exc
+            matching: list[BaseException] = [
+                maybe_exc
+            ]
+
+        # XXX, only unmask-ed for debuggin!
+        # TODO, remove eventually..
+        except BaseException as _berr:
+            berr = _berr
+            await pause(shield=True)
+            raise berr
+
+    if matching is None:
+        raise
+
+    masked: list[tuple[BaseException, BaseException]] = []
+    for exc_match in matching:
+
+        if exc_ctx := find_masked_excs(
+            maybe_masker=exc_match,
+            unmask_from={unmask_from},
+        ):
+            masked.append((exc_ctx, exc_match))
+            boxed_maybe_exc.value = exc_match
+            note: str = (
+                f'\n'
+                f'^^WARNING^^ the above {exc_ctx!r} was masked by a {unmask_from!r}\n'
+            )
+            if extra_note:
+                note += (
+                    f'\n'
+                    f'{extra_note}\n'
+                )
+            exc_ctx.add_note(note)
+
+            if type(exc_match) in always_warn_on:
+                log.warning(note)
+
+            # await tractor.pause(shield=True)
+            if raise_unmasked:
+
+                if len(masked) < 2:
+                    raise exc_ctx from exc_match
+                else:
+                    # ?TODO, see above but, possibly unmasking sub-exc
+                    # entries if there are > 1
+                    await pause(shield=True)
+    else:
+        raise

uv.lock

@@ -417,7 +417,7 @@ dev = [
     { name = "pyperclip", specifier = ">=1.9.0" },
     { name = "pytest", specifier = ">=8.3.5" },
     { name = "stackscope", specifier = ">=0.2.2,<0.3" },
-    { name = "typing-extensions", specifier = ">=4.14.1" },
+    { name = "typing-extensions", specifier = ">=4.13.2" },
     { name = "xonsh", specifier = ">=0.19.2" },
 ]
 
@@ -452,11 +452,11 @@ wheels = [
 
 [[package]]
 name = "typing-extensions"
-version = "4.14.1"
+version = "4.13.2"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806, upload-time = "2025-04-10T14:19:03.967Z" },
 ]
 
 [[package]]