Merge pull request #358 from goodboy/switch_to_pdbp

Switch to `pdbp` 🏄🏼

.github/workflows/ci.yml

@@ -1,6 +1,11 @@
 name: CI
 
-on: push
+on:
+  # any time someone pushes a new branch to origin
+  push:
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
 
 jobs:
 
@@ -15,13 +20,35 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.10'
 
       - name: Install dependencies
         run: pip install -U . --upgrade-strategy eager -r requirements-test.txt
 
       - name: Run MyPy check
-        run: mypy tractor/ --ignore-missing-imports
+        run: mypy tractor/ --ignore-missing-imports --show-traceback
+
+  # test that we can generate a software distribution and install it
+  # thus avoid missing file issues after packaging.
+  sdist-linux:
+    name: 'sdist'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      - name: Build sdist
+        run: python setup.py sdist --formats=zip
+
+      - name: Install sdist from .zips
+        run: python -m pip install dist/*.zip
+
 
   testing-linux:
     name: '${{ matrix.os }} Python ${{ matrix.python }} - ${{ matrix.spawn_backend }}'
@@ -32,8 +59,12 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python: ['3.9', '3.10']
+        python: ['3.10']
-        spawn_backend: ['trio', 'mp']
+        spawn_backend: [
+          'trio',
+          'mp_spawn',
+          'mp_forkserver',
+        ]
 
     steps:
 
@@ -48,71 +79,53 @@ jobs:
       - name: Install dependencies
         run: pip install -U . -r requirements-test.txt -r requirements-docs.txt --upgrade-strategy eager
 
-      - name: Run tests
+      - name: List dependencies
-        run: pytest tests/ --spawn-backend=${{ matrix.spawn_backend }} -rs
+        run: pip list
-
-  testing-linux-msgspec:
-    # runs jobs on all OS's but with optional `msgspec` dep installed
-    name: '${{ matrix.os }} Python ${{ matrix.python }} - ${{ matrix.spawn_backend }} - msgspec'
-    timeout-minutes: 10
-    runs-on: ${{ matrix.os }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest]
-        python: ['3.9', '3.10']
-        spawn_backend: ['trio', 'mp']
-
-    steps:
-
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Setup python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '${{ matrix.python }}'
-
-      - name: Install dependencies
-        run: pip install -U .[msgspec] -r requirements-test.txt -r requirements-docs.txt --upgrade-strategy eager
 
       - name: Run tests
-        run: pytest tests/ --spawn-backend=${{ matrix.spawn_backend }} -rs
+        run: pytest tests/ --spawn-backend=${{ matrix.spawn_backend }} -rsx
 
-  # We skip 3.10 on windows for now due to
+  # We skip 3.10 on windows for now due to not having any collabs to
-  # https://github.com/pytest-dev/pytest/issues/8733
+  # debug the CI failures. Anyone wanting to hack and solve them is very
-  # some kinda weird `pyreadline` issue..
+  # welcome, but our primary user base is not using that OS.
 
   # TODO: use job filtering to accomplish instead of repeated
   # boilerplate as is above XD:
   # - https://docs.github.com/en/actions/learn-github-actions/managing-complex-workflows
   # - https://docs.github.com/en/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
   # - https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idif
-  testing-windows:
+  # testing-windows:
-    name: '${{ matrix.os }} Python ${{ matrix.python }} - ${{ matrix.spawn_backend }}'
+  #   name: '${{ matrix.os }} Python ${{ matrix.python }} - ${{ matrix.spawn_backend }}'
-    timeout-minutes: 12
+  #   timeout-minutes: 12
-    runs-on: ${{ matrix.os }}
+  #   runs-on: ${{ matrix.os }}
 
-    strategy:
+  #   strategy:
-      fail-fast: false
+  #     fail-fast: false
-      matrix:
+  #     matrix:
-        os: [windows-latest]
+  #       os: [windows-latest]
-        python: ['3.9']
+  #       python: ['3.10']
-        spawn_backend: ['trio', 'mp']
+  #       spawn_backend: ['trio', 'mp']
 
-    steps:
+  #   steps:
 
-      - name: Checkout
+  #     - name: Checkout
-        uses: actions/checkout@v2
+  #       uses: actions/checkout@v2
 
-      - name: Setup python
+  #     - name: Setup python
-        uses: actions/setup-python@v2
+  #       uses: actions/setup-python@v2
-        with:
+  #       with:
-          python-version: '${{ matrix.python }}'
+  #         python-version: '${{ matrix.python }}'
 
-      - name: Install dependencies
+  #     - name: Install dependencies
-        run: pip install -U . -r requirements-test.txt -r requirements-docs.txt --upgrade-strategy eager
+  #       run: pip install -U . -r requirements-test.txt -r requirements-docs.txt --upgrade-strategy eager
 
-      - name: Run tests
+  #     # TODO: pretty sure this solves debugger deps-issues on windows, but it needs to
-        run: pytest tests/ --spawn-backend=${{ matrix.spawn_backend }} -rs
+  #     # be verified by someone with a native setup.
+  #     # - name: Force pyreadline3
+  #     #   run: pip uninstall pyreadline; pip install -U pyreadline3
+
+  #     - name: List dependencies
+  #       run: pip list
+
+  #     - name: Run tests
+  #       run: pytest tests/ --spawn-backend=${{ matrix.spawn_backend }} -rsx

MANIFEST.in

@@ -0,0 +1,2 @@
+# https://packaging.python.org/en/latest/guides/using-manifest-in/#using-manifest-in
+include docs/README.rst

NEWS.rst

@@ -4,6 +4,148 @@ Changelog
 
 .. towncrier release notes start
 
+tractor 0.1.0a5 (2022-08-03)
+============================
+
+This is our final release supporting Python 3.9 since we will be moving
+internals to the new `match:` syntax from 3.10 going forward and
+further, we have officially dropped usage of the `msgpack` library and
+happily adopted `msgspec`.
+
+Features
+--------
+
+- `#165 <https://github.com/goodboy/tractor/issues/165>`_: Add SIGINT
+  protection to our `pdbpp` based debugger subystem such that for
+  (single-depth) actor trees in debug mode we ignore interrupts in any
+  actor currently holding the TTY lock thus avoiding clobbering IPC
+  connections and/or task and process state when working in the REPL.
+
+  As a big note currently so called "nested" actor trees (trees with
+  actors having more then one parent/ancestor) are not fully supported
+  since we don't yet have a mechanism to relay the debug mode knowledge
+  "up" the actor tree (for eg. when handling a crash in a leaf actor).
+  As such currently there is a set of tests and known scenarios which will
+  result in process cloberring by the zombie repaing machinery and these
+  have been documented in https://github.com/goodboy/tractor/issues/320.
+
+  The implementation details include:
+
+  - utilizing a custom SIGINT handler which we apply whenever an actor's
+    runtime enters the debug machinery, which we also make sure the
+    stdlib's `pdb` configuration doesn't override (which it does by
+    default without special instance config).
+  - litter the runtime with `maybe_wait_for_debugger()` mostly in spots
+    where the root actor should block before doing embedded nursery
+    teardown ops which both cancel potential-children-in-deubg as well
+    as eventually trigger zombie reaping machinery.
+  - hardening of the TTY locking semantics/API both in terms of IPC
+    terminations and cancellation and lock release determinism from
+    sync debugger instance methods.
+  - factoring of locking infrastructure into a new `._debug.Lock` global
+    which encapsulates all details of the ``trio`` sync primitives and
+    task/actor uid management and tracking.
+
+  We also add `ctrl-c` cases throughout the test suite though these are
+  disabled for py3.9 (`pdbpp` UX differences that don't seem worth
+  compensating for, especially since this will be our last 3.9 supported
+  release) and there are a slew of marked cases that aren't expected to
+  work in CI more generally (as mentioned in the "nested" tree note
+  above) despite seemingly working  when run manually on linux.
+
+- `#304 <https://github.com/goodboy/tractor/issues/304>`_: Add a new
+  ``to_asyncio.LinkedTaskChannel.subscribe()`` which gives task-oriented
+  broadcast functionality semantically equivalent to
+  ``tractor.MsgStream.subscribe()`` this makes it possible for multiple
+  ``trio``-side tasks to consume ``asyncio``-side task msgs in tandem.
+
+  Further Improvements to the test suite were added in this patch set
+  including a new scenario test for a sub-actor managed "service nursery"
+  (implementing the basics of a "service manager") including use of
+  *infected asyncio* mode. Further we added a lower level
+  ``test_trioisms.py`` to start to track issues we need to work around in
+  ``trio`` itself which in this case included a bug we were trying to
+  solve related to https://github.com/python-trio/trio/issues/2258.
+
+
+Bug Fixes
+---------
+
+- `#318 <https://github.com/goodboy/tractor/issues/318>`_: Fix
+  a previously undetected ``trio``-``asyncio`` task lifetime linking
+  issue with the ``to_asyncio.open_channel_from()`` api where both sides
+  where not properly waiting/signalling termination and it was possible
+  for ``asyncio``-side errors to not propagate due to a race condition.
+
+  The implementation fix summary is:
+  - add state to signal the end of the ``trio`` side task to be
+    read by the ``asyncio`` side and always cancel any ongoing
+    task in such cases.
+  - always wait on the ``asyncio`` task termination from the ``trio``
+    side on error before maybe raising said error.
+  - always close the ``trio`` mem chan on exit to ensure the other
+    side can detect it and follow.
+
+
+Trivial/Internal Changes
+------------------------
+
+- `#248 <https://github.com/goodboy/tractor/issues/248>`_: Adjust the
+  `tractor._spawn.soft_wait()` strategy to avoid sending an actor cancel
+  request (via `Portal.cancel_actor()`) if either the child process is
+  detected as having terminated or the IPC channel is detected to be
+  closed.
+
+  This ensures (even) more deterministic inter-actor cancellation by
+  avoiding the timeout condition where possible when a whild never
+  sucessfully spawned, crashed, or became un-contactable over IPC.
+
+- `#295 <https://github.com/goodboy/tractor/issues/295>`_: Add an
+  experimental ``tractor.msg.NamespacePath`` type for passing Python
+  objects by "reference" through a ``str``-subtype message and using the
+  new ``pkgutil.resolve_name()`` for reference loading.
+
+- `#298 <https://github.com/goodboy/tractor/issues/298>`_: Add a new
+  `tractor.experimental` subpackage for staging new high level APIs and
+  subystems that we might eventually make built-ins.
+
+- `#300 <https://github.com/goodboy/tractor/issues/300>`_: Update to and
+  pin latest ``msgpack`` (1.0.3) and ``msgspec`` (0.4.0) both of which
+  required adjustments for backwards imcompatible API tweaks.
+
+- `#303 <https://github.com/goodboy/tractor/issues/303>`_: Fence off
+  ``multiprocessing`` imports until absolutely necessary in an effort to
+  avoid "resource tracker" spawning side effects that seem to have
+  varying degrees of unreliability per Python release. Port to new
+  ``msgspec.DecodeError``.
+
+- `#305 <https://github.com/goodboy/tractor/issues/305>`_: Add
+  ``tractor.query_actor()`` an addr looker-upper which doesn't deliver
+  a ``Portal`` instance and instead just a socket address ``tuple``.
+
+  Sometimes it's handy to just have a simple way to figure out if
+  a "service" actor is up, so add this discovery helper for that. We'll
+  prolly just leave it undocumented for now until we figure out
+  a longer-term/better discovery system.
+
+- `#316 <https://github.com/goodboy/tractor/issues/316>`_: Run windows
+  CI jobs on python 3.10 after some hacks for ``pdbpp`` dependency
+  issues.
+
+  Issue was to do with the now deprecated `pyreadline` project which
+  should be changed over to `pyreadline3`.
+
+- `#317 <https://github.com/goodboy/tractor/issues/317>`_: Drop use of
+  the ``msgpack`` package and instead move fully to the ``msgspec``
+  codec library.
+
+  We've now used ``msgspec`` extensively in production and there's no
+  reason to not use it as default. Further this change preps us for the up
+  and coming typed messaging semantics (#196), dialog-unprotocol system
+  (#297), and caps-based messaging-protocols (#299) planned before our
+  first beta.
+
+
 tractor 0.1.0a4 (2021-12-18)
 ============================
 

docs/README.rst

@@ -3,13 +3,20 @@
 |gh_actions|
 |docs|
 
-``tractor`` is a `structured concurrent`_, multi-processing_ runtime built on trio_.
+``tractor`` is a `structured concurrent`_, multi-processing_ runtime
+built on trio_.
 
-Fundamentally ``tractor`` gives you parallelism via ``trio``-"*actors*":
+Fundamentally, ``tractor`` gives you parallelism via
-our nurseries_ let you spawn new Python processes which each run a ``trio``
+``trio``-"*actors*": independent Python processes (aka
+non-shared-memory threads) which maintain structured
+concurrency (SC) *end-to-end* inside a *supervision tree*.
+
+Cross-process (and thus cross-host) SC is accomplished through the
+combined use of our "actor nurseries_" and an "SC-transitive IPC
+protocol" constructed on top of multiple Pythons each running a ``trio``
 scheduled runtime - a call to ``trio.run()``.
 
-We believe the system adhere's to the `3 axioms`_ of an "`actor model`_"
+We believe the system adheres to the `3 axioms`_ of an "`actor model`_"
 but likely *does not* look like what *you* probably think an "actor
 model" looks like, and that's *intentional*.
 
@@ -22,9 +29,10 @@ Features
 - **It's just** a ``trio`` API
 - *Infinitely nesteable* process trees
 - Builtin IPC streaming APIs with task fan-out broadcasting
-- A (first ever?) "native" multi-core debugger UX for Python using `pdb++`_
+- A "native" multi-core debugger REPL using `pdbp`_ (a fork & fix of
+  `pdb++`_ thanks to @mdmintz!)
 - Support for a swappable, OS specific, process spawning layer
-- A modular transport stack, allowing for custom serialization (eg.
+- A modular transport stack, allowing for custom serialization (eg. with
   `msgspec`_), communications protocols, and environment specific IPC
   primitives
 - Support for spawning process-level-SC, inter-loop one-to-one-task oriented
@@ -148,7 +156,7 @@ it **is a bug**.
 
 "Native" multi-process debugging
 --------------------------------
-Using the magic of `pdb++`_ and our internal IPC, we've
+Using the magic of `pdbp`_ and our internal IPC, we've
 been able to create a native feeling debugging experience for
 any (sub-)process in your ``tractor`` tree.
 
@@ -489,12 +497,6 @@ From PyPi::
     pip install tractor
 
 
-To try out the (optionally) faster `msgspec`_ codec instead of the
-default ``msgpack`` lib::
-
-    pip install tractor[msgspec]
-
-
 From git::
 
     pip install git+git://github.com/goodboy/tractor.git
@@ -563,11 +565,22 @@ properties of the system.
 
 What's on the TODO:
 -------------------
-Help us push toward the future.
+Help us push toward the future of distributed `Python`.
 
-- Typed messaging protocols (ex. via ``msgspec``, see `#36
+- Erlang-style supervisors via composed context managers (see `#22
+  <https://github.com/goodboy/tractor/issues/22>`_)
+- Typed messaging protocols (ex. via ``msgspec.Struct``, see `#36
   <https://github.com/goodboy/tractor/issues/36>`_)
-- Erlang-style supervisors via composed context managers
+- Typed capability-based (dialog) protocols ( see `#196
+  <https://github.com/goodboy/tractor/issues/196>`_ with draft work
+  started in `#311 <https://github.com/goodboy/tractor/pull/311>`_)
+- We **recently disabled CI-testing on windows** and need help getting
+  it running again! (see `#327
+  <https://github.com/goodboy/tractor/pull/327>`_). **We do have windows
+  support** (and have for quite a while) but since no active hacker
+  exists in the user-base to help test on that OS, for now we're not
+  actively maintaining testing due to the added hassle and general
+  latency..
 
 
 Feel like saying hi?
@@ -579,18 +592,19 @@ say hi, please feel free to reach us in our `matrix channel`_.  If
 matrix seems too hip, we're also mostly all in the the `trio gitter
 channel`_!
 
+.. _structured concurrent: https://trio.discourse.group/t/concise-definition-of-structured-concurrency/228
+.. _multi-processing: https://en.wikipedia.org/wiki/Multiprocessing
+.. _trio: https://github.com/python-trio/trio
 .. _nurseries: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/#nurseries-a-structured-replacement-for-go-statements
 .. _actor model: https://en.wikipedia.org/wiki/Actor_model
-.. _trio: https://github.com/python-trio/trio
-.. _multi-processing: https://en.wikipedia.org/wiki/Multiprocessing
 .. _trionic: https://trio.readthedocs.io/en/latest/design.html#high-level-design-principles
 .. _async sandwich: https://trio.readthedocs.io/en/latest/tutorial.html#async-sandwich
-.. _structured concurrent: https://trio.discourse.group/t/concise-definition-of-structured-concurrency/228
 .. _3 axioms: https://www.youtube.com/watch?v=7erJ1DV_Tlo&t=162s
 .. .. _3 axioms: https://en.wikipedia.org/wiki/Actor_model#Fundamental_concepts
 .. _adherance to: https://www.youtube.com/watch?v=7erJ1DV_Tlo&t=1821s
 .. _trio gitter channel: https://gitter.im/python-trio/general
 .. _matrix channel: https://matrix.to/#/!tractor:matrix.org
+.. _pdbp: https://github.com/mdmintz/pdbp
 .. _pdb++: https://github.com/pdbpp/pdbpp
 .. _guest mode: https://trio.readthedocs.io/en/stable/reference-lowlevel.html?highlight=guest%20mode#using-guest-mode-to-run-trio-on-top-of-other-event-loops
 .. _messages: https://en.wikipedia.org/wiki/Message_passing

docs/index.rst

@@ -396,7 +396,7 @@ tasks spawned via multiple RPC calls to an actor can modify
 
 
         # a per process cache
-        _actor_cache: Dict[str, bool] = {}
+        _actor_cache: dict[str, bool] = {}
 
 
         def ping_endpoints(endpoints: List[str]):

examples/advanced_faults/ipc_failure_during_stream.py

@@ -0,0 +1,151 @@
+'''
+Complex edge case where during real-time streaming the IPC tranport
+channels are wiped out (purposely in this example though it could have
+been an outage) and we want to ensure that despite being in debug mode
+(or not) the user can sent SIGINT once they notice the hang and the
+actor tree will eventually be cancelled without leaving any zombies.
+
+'''
+import trio
+from tractor import (
+    open_nursery,
+    context,
+    Context,
+    MsgStream,
+)
+
+
+async def break_channel_silently_then_error(
+    stream: MsgStream,
+):
+    async for msg in stream:
+        await stream.send(msg)
+
+        # XXX: close the channel right after an error is raised
+        # purposely breaking the IPC transport to make sure the parent
+        # doesn't get stuck in debug or hang on the connection join.
+        # this more or less simulates an infinite msg-receive hang on
+        # the other end.
+        await stream._ctx.chan.send(None)
+        assert 0
+
+
+async def close_stream_and_error(
+    stream: MsgStream,
+):
+    async for msg in stream:
+        await stream.send(msg)
+
+        # wipe out channel right before raising
+        await stream._ctx.chan.send(None)
+        await stream.aclose()
+        assert 0
+
+
+@context
+async def recv_and_spawn_net_killers(
+
+    ctx: Context,
+    break_ipc_after: bool | int = False,
+
+) -> None:
+    '''
+    Receive stream msgs and spawn some IPC killers mid-stream.
+
+    '''
+    await ctx.started()
+    async with (
+        ctx.open_stream() as stream,
+        trio.open_nursery() as n,
+    ):
+        async for i in stream:
+            print(f'child echoing {i}')
+            await stream.send(i)
+            if (
+                break_ipc_after
+                and i > break_ipc_after
+            ):
+                '#################################\n'
+                'Simulating child-side IPC BREAK!\n'
+                '#################################'
+                n.start_soon(break_channel_silently_then_error, stream)
+                n.start_soon(close_stream_and_error, stream)
+
+
+async def main(
+    debug_mode: bool = False,
+    start_method: str = 'trio',
+
+    # by default we break the parent IPC first (if configured to break
+    # at all), but this can be changed so the child does first (even if
+    # both are set to break).
+    break_parent_ipc_after: int | bool = False,
+    break_child_ipc_after: int | bool = False,
+
+) -> None:
+
+    async with (
+        open_nursery(
+            start_method=start_method,
+
+            # NOTE: even debugger is used we shouldn't get
+            # a hang since it never engages due to broken IPC
+            debug_mode=debug_mode,
+            loglevel='warning',
+
+        ) as an,
+    ):
+        portal = await an.start_actor(
+            'chitty_hijo',
+            enable_modules=[__name__],
+        )
+
+        async with portal.open_context(
+            recv_and_spawn_net_killers,
+            break_ipc_after=break_child_ipc_after,
+
+        ) as (ctx, sent):
+            async with ctx.open_stream() as stream:
+                for i in range(1000):
+
+                    if (
+                        break_parent_ipc_after
+                        and i > break_parent_ipc_after
+                    ):
+                        print(
+                            '#################################\n'
+                            'Simulating parent-side IPC BREAK!\n'
+                            '#################################'
+                        )
+                        await stream._ctx.chan.send(None)
+
+                    # it actually breaks right here in the
+                    # mp_spawn/forkserver backends and thus the zombie
+                    # reaper never even kicks in?
+                    print(f'parent sending {i}')
+                    await stream.send(i)
+
+                    with trio.move_on_after(2) as cs:
+
+                        # NOTE: in the parent side IPC failure case this
+                        # will raise an ``EndOfChannel`` after the child
+                        # is killed and sends a stop msg back to it's
+                        # caller/this-parent.
+                        rx = await stream.receive()
+
+                        print(f"I'm a happy user and echoed to me is {rx}")
+
+                    if cs.cancelled_caught:
+                        # pretend to be a user seeing no streaming action
+                        # thinking it's a hang, and then hitting ctl-c..
+                        print("YOO i'm a user anddd thingz hangin..")
+
+                print(
+                    "YOO i'm mad send side dun but thingz hangin..\n"
+                    'MASHING CTlR-C Ctl-c..'
+                )
+                raise KeyboardInterrupt
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/multi_daemon_subactors.py

@@ -27,6 +27,17 @@ async def main():
 
         # retreive results
         async with p0.open_stream_from(breakpoint_forever) as stream:
+
+            # triggers the first name error
+            try:
+                await p1.run(name_error)
+            except tractor.RemoteActorError as rae:
+                assert rae.type is NameError
+
+            async for i in stream:
+
+                # a second time try the failing subactor and this tie
+                # let error propagate up to the parent/nursery.
                 await p1.run(name_error)
 
 

examples/debugging/multi_nested_subactors_error_up_through_nurseries.py

@@ -12,18 +12,31 @@ async def breakpoint_forever():
     while True:
         await tractor.breakpoint()
 
+        # NOTE: if the test never sent 'q'/'quit' commands
+        # on the pdb repl, without this checkpoint line the
+        # repl would spin in this actor forever.
+        # await trio.sleep(0)
+
 
 async def spawn_until(depth=0):
     """"A nested nursery that triggers another ``NameError``.
     """
     async with tractor.open_nursery() as n:
         if depth < 1:
-            # await n.run_in_actor('breakpoint_forever', breakpoint_forever)
+
-            await n.run_in_actor(
+            await n.run_in_actor(breakpoint_forever)
+
+            p = await n.run_in_actor(
                 name_error,
                 name='name_error'
             )
+            await trio.sleep(0.5)
+            # rx and propagate error from child
+            await p.result()
+
         else:
+            # recusrive call to spawn another process branching layer of
+            # the tree
             depth -= 1
             await n.run_in_actor(
                 spawn_until,
@@ -53,6 +66,7 @@ async def main():
     """
     async with tractor.open_nursery(
         debug_mode=True,
+        # loglevel='cancel',
     ) as n:
 
         # spawn both actors
@@ -67,8 +81,16 @@ async def main():
             name='spawner1',
         )
 
+        # TODO: test this case as well where the parent don't see
+        # the sub-actor errors by default and instead expect a user
+        # ctrl-c to kill the root.
+        with trio.move_on_after(3):
+            await trio.sleep_forever()
+
         # gah still an issue here.
         await portal.result()
+
+        # should never get here
         await portal1.result()
 
 

examples/debugging/open_ctx_modnofound.py

@@ -0,0 +1,40 @@
+import trio
+import tractor
+
+
+@tractor.context
+async def just_sleep(
+
+    ctx: tractor.Context,
+    **kwargs,
+
+) -> None:
+    '''
+    Start and sleep.
+
+    '''
+    await ctx.started()
+    await trio.sleep_forever()
+
+
+async def main() -> None:
+
+    async with tractor.open_nursery(
+        debug_mode=True,
+    ) as n:
+        portal = await n.start_actor(
+            'ctx_child',
+
+            # XXX: we don't enable the current module in order
+            # to trigger `ModuleNotFound`.
+            enable_modules=[],
+        )
+
+        async with portal.open_context(
+            just_sleep,  # taken from pytest parameterization
+        ) as (ctx, sent):
+            raise KeyboardInterrupt
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/restore_builtin_breakpoint.py

@@ -0,0 +1,24 @@
+import os
+import sys
+
+import trio
+import tractor
+
+
+async def main() -> None:
+    async with tractor.open_nursery(debug_mode=True) as an:
+
+        assert os.environ['PYTHONBREAKPOINT'] == 'tractor._debug._set_trace'
+
+        # TODO: an assert that verifies the hook has indeed been, hooked
+        # XD
+        assert sys.breakpointhook is not tractor._debug._set_trace
+
+        breakpoint()
+
+    # TODO: an assert that verifies the hook is unhooked..
+    assert sys.breakpointhook
+    breakpoint()
+
+if __name__ == '__main__':
+    trio.run(main)

examples/infected_asyncio_echo_server.py

@@ -13,6 +13,7 @@ import tractor
 async def aio_echo_server(
     to_trio: trio.MemorySendChannel,
     from_trio: asyncio.Queue,
+
 ) -> None:
 
     # a first message must be sent **from** this ``asyncio``

examples/integration/open_context_and_sleep.py

@@ -0,0 +1,49 @@
+import trio
+import click
+import tractor
+import pydantic
+# from multiprocessing import shared_memory
+
+
+@tractor.context
+async def just_sleep(
+
+    ctx: tractor.Context,
+    **kwargs,
+
+) -> None:
+    '''
+    Test a small ping-pong 2-way streaming server.
+
+    '''
+    await ctx.started()
+    await trio.sleep_forever()
+
+
+async def main() -> None:
+
+    proc = await trio.open_process( (
+        'python',
+        '-c',
+        'import trio; trio.run(trio.sleep_forever)',
+    ))
+    await proc.wait()
+    # await trio.sleep_forever()
+    # async with tractor.open_nursery() as n:
+
+    #     portal = await n.start_actor(
+    #         'rpc_server',
+    #         enable_modules=[__name__],
+    #     )
+
+    #     async with portal.open_context(
+    #         just_sleep,  # taken from pytest parameterization
+    #     ) as (ctx, sent):
+    #         await trio.sleep_forever()
+
+
+
+if __name__ == '__main__':
+    import time
+    # time.sleep(999)
+    trio.run(main)

examples/parallelism/concurrent_actors_primes.py

@@ -9,7 +9,7 @@ is ``tractor``'s channels.
 
 """
 from contextlib import asynccontextmanager
-from typing import List, Callable
+from typing import Callable
 import itertools
 import math
 import time
@@ -71,8 +71,8 @@ async def worker_pool(workers=4):
 
         async def _map(
             worker_func: Callable[[int], bool],
-            sequence: List[int]
+            sequence: list[int]
-        ) -> List[bool]:
+        ) -> list[bool]:
 
             # define an async (local) task to collect results from workers
             async def send_result(func, value, portal):

nooz/248.misc.rst

@@ -1,8 +0,0 @@
-Adjust the `tractor._spawn.soft_wait()` strategy to avoid sending an
-actor cancel request (via `Portal.cancel_actor()`) if either the child
-process is detected as having terminated or the IPC channel is detected
-to be closed.
-
-This ensures (even) more deterministic inter-actor cancellation by
-avoiding the timeout condition where possible when a whild never
-sucessfully spawned, crashed, or became un-contactable over IPC.

nooz/295.misc.rst

@@ -1,3 +0,0 @@
-Add an experimental `tractor.msg.NamespacePath` type for passing Python
-objects by "reference" through a ``str``-subtype message and using the
-new ``pkgutil.resolve_name()`` for reference loading.

nooz/298.misc.rst

@@ -1,2 +0,0 @@
-Add a new `tractor.experimental` subpackage for staging new high level
-APIs and subystems that we might eventually make built-ins.

nooz/300.misc.rst

@@ -1,3 +0,0 @@
-Update to and pin latest `msgpack` (1.0.3) and `msgspec` (0.4.0)
-both of which required adjustments for backwards imcompatible API
-tweaks.

nooz/322.trivial.rst

@@ -0,0 +1,16 @@
+Strictly support Python 3.10+, start runtime machinery reorg
+
+Since we want to push forward using the new `match:` syntax for our
+internal RPC-msg loops, we officially drop 3.9 support for the next
+release which should coincide well with the first release of 3.11.
+
+This patch set also officially removes the ``tractor.run()`` API (which
+has been deprecated for some time) as well as starts an initial re-org
+of the internal runtime core by:
+- renaming ``tractor._actor`` -> ``._runtime``
+- moving the ``._runtime.ActorActor._process_messages()`` and
+  ``._async_main()`` to be module level singleton-task-functions since
+  they are only started once for each connection and actor spawn
+  respectively; this internal API thus looks more similar to (at the
+  time of writing) the ``trio``-internals in ``trio._core._run``.
+- officially remove ``tractor.run()``, now deprecated for some time.

nooz/324.bugfix.rst

@@ -0,0 +1,4 @@
+Only set `._debug.Lock.local_pdb_complete` if has been created.
+
+This can be triggered by a very rare race condition (and thus we have no
+working test yet) but it is known to exist in (a) consumer project(s).

nooz/333.feature.rst

@@ -0,0 +1,25 @@
+Add support for ``trio >= 0.22`` and support for the new Python 3.11
+``[Base]ExceptionGroup`` from `pep 654`_ via the backported
+`exceptiongroup`_ package and some final fixes to the debug mode
+subsystem.
+
+This port ended up driving some (hopefully) final fixes to our debugger
+subsystem including the solution to all lingering stdstreams locking
+race-conditions and deadlock scenarios. This includes extending the
+debugger tests suite as well as cancellation and ``asyncio`` mode cases.
+Some of the notable details:
+
+- always reverting to the ``trio`` SIGINT handler when leaving debug
+  mode.
+- bypassing child attempts to acquire the debug lock when detected
+  to be amdist actor-runtime-cancellation.
+- allowing the root actor to cancel local but IPC-stale subactor
+  requests-tasks for the debug lock when in a "no IPC peers" state.
+
+Further we refined our ``ActorNursery`` semantics to be more similar to
+``trio`` in the sense that parent task errors are always packed into the
+actor-nursery emitted exception group and adjusted all tests and
+examples accordingly.
+
+.. _pep 654: https://peps.python.org/pep-0654/#handling-exception-groups
+.. _exceptiongroup: https://github.com/python-trio/exceptiongroup

nooz/335.trivial.rst

@@ -0,0 +1,5 @@
+Establish an explicit "backend spawning" method table; use it from CI
+
+More clearly lays out the current set of (3) backends: ``['trio',
+'mp_spawn', 'mp_forkserver']`` and adjusts the ``._spawn.py`` internals
+as well as the test suite to accommodate.

nooz/336.trivial.rst

@@ -0,0 +1,4 @@
+Add ``key: Callable[..., Hashable]`` support to ``.trionics.maybe_open_context()``
+
+Gives users finer grained control over cache hit behaviour using
+a callable which receives the input ``kwargs: dict``.

nooz/337.feature.rst

@@ -0,0 +1,41 @@
+Add support for debug-lock blocking using a ``._debug.Lock._blocked:
+set[tuple]`` and add ids when no-more IPC connections with the
+root actor are detected.
+
+This is an enhancement which (mostly) solves a lingering debugger
+locking race case we needed to handle:
+
+- child crashes acquires TTY lock in root and attaches to ``pdb``
+- child IPC goes down such that all channels to the root are broken
+  / non-functional.
+- root is stuck thinking the child is still in debug even though it
+  can't be contacted and the child actor machinery hasn't been
+  cancelled by its parent.
+- root get's stuck in deadlock with child since it won't send a cancel
+  request until the child is finished debugging (to avoid clobbering
+  a child that is actually using the debugger), but the child can't
+  unlock the debugger bc IPC is down and it can't contact the root.
+
+To avoid this scenario add debug lock blocking list via
+`._debug.Lock._blocked: set[tuple]` which holds actor uids for any actor
+that is detected by the root as having no transport channel connections
+(of which at least one should exist if this sub-actor at some point
+acquired the debug lock). The root consequently checks this list for any
+actor that tries to (re)acquire the lock and blocks with
+a ``ContextCancelled``. Further, when a debug condition is tested in
+``._runtime._invoke``, the context's ``._enter_debugger_on_cancel`` is
+set to `False` if the actor was put on the block list then all
+post-mortem / crash handling will be bypassed for that task.
+
+In theory this approach to block list management may cause problems
+where some nested child actor acquires and releases the lock multiple
+times and it gets stuck on the block list after the first use? If this
+turns out to be an issue we can try changing the strat so blocks are
+only added when the root has zero IPC peers left?
+
+Further, this adds a root-locking-task side cancel scope,
+``Lock._root_local_task_cs_in_debug``, which can be ``.cancel()``-ed by the root
+runtime when a stale lock is detected during the IPC channel testing.
+However, right now we're NOT using this since it seems to cause test
+failures likely due to causing pre-mature cancellation and maybe needs
+a bit more experimenting?

nooz/343.trivial.rst

@@ -0,0 +1,19 @@
+Rework our ``.trionics.BroadcastReceiver`` internals to avoid method
+recursion and approach a design and interface closer to ``trio``'s
+``MemoryReceiveChannel``.
+
+The details of the internal changes include:
+
+- implementing a ``BroadcastReceiver.receive_nowait()`` and using it
+  within the async ``.receive()`` thus avoiding recursion from
+  ``.receive()``.
+- failing over to an internal ``._receive_from_underlying()`` when the
+  ``_nowait()`` call raises ``trio.WouldBlock``
+- adding ``BroadcastState.statistics()`` for debugging and testing both
+  internals and by users.
+- add an internal ``BroadcastReceiver._raise_on_lag: bool`` which can be
+  set to avoid ``Lagged`` raising for possible use cases where a user
+  wants to choose between a [cheap or nasty
+  pattern](https://zguide.zeromq.org/docs/chapter7/#The-Cheap-or-Nasty-Pattern)
+  the the particular stream (we use this in ``piker``'s dark clearing
+  engine to avoid fast feeds breaking during HFT periods).

nooz/344.bugfix.rst

@@ -0,0 +1,11 @@
+Always ``list``-cast the ``mngrs`` input to
+``.trionics.gather_contexts()`` and ensure its size otherwise raise
+a ``ValueError``.
+
+Turns out that trying to pass an inline-style generator comprehension
+doesn't seem to work inside the ``async with`` expression? Further, in
+such a case we can get a hang waiting on the all-entered event
+completion when the internal mngrs iteration is a noop. Instead we
+always greedily check a size and error on empty input; the lazy
+iteration of a generator input is not beneficial anyway since we're
+entering all manager instances in concurrent tasks.

nooz/346.bugfix.rst

@@ -0,0 +1,15 @@
+Fixes to ensure IPC (channel) breakage doesn't result in hung actor
+trees; the zombie reaping and general supervision machinery will always
+clean up and terminate.
+
+This includes not only the (mostly minor) fixes to solve these cases but
+also a new extensive test suite in `test_advanced_faults.py` with an
+accompanying highly configurable example module-script in
+`examples/advanced_faults/ipc_failure_during_stream.py`. Tests ensure we
+never get hang or zombies despite operating in debug mode and attempt to
+simulate all possible IPC transport failure cases for a local-host actor
+tree.
+
+Further we simplify `Context.open_stream.__aexit__()` to just call
+`MsgStream.aclose()` directly more or less avoiding a pure duplicate
+code path.

nooz/349.trivial.rst

@@ -0,0 +1,10 @@
+Always redraw the `pdbpp` prompt on `SIGINT` during REPL use.
+
+There was recent changes todo with Python 3.10 that required us to pin
+to a specific commit in `pdbpp` which have recently been fixed minus
+this last issue with `SIGINT` shielding: not clobbering or not
+showing the `(Pdb++)` prompt on ctlr-c by the user. This repairs all
+that by firstly removing the standard KBI intercepting of the std lib's
+`pdb.Pdb._cmdloop()` as well as ensuring that only the actor with REPL
+control ever reports `SIGINT` handler log msgs and prompt redraws. With
+this we move back to using pypi `pdbpp` release.

nooz/356.trivial.rst

@@ -0,0 +1,7 @@
+Drop `trio.Process.aclose()` usage, copy into our spawning code.
+
+The details are laid out in https://github.com/goodboy/tractor/issues/330.
+`trio` changed is process running quite some time ago, this just copies
+out the small bit we needed (from the old `.aclose()`) for hard kills
+where a soft runtime cancel request fails and our "zombie killer"
+implementation kicks in.

nooz/358.feature.rst

@@ -0,0 +1,15 @@
+Switch to using the fork & fix of `pdb++`, `pdbp`:
+https://github.com/mdmintz/pdbp
+
+Allows us to sidestep a variety of issues that aren't being maintained
+in the upstream project thanks to the hard work of @mdmintz!
+
+We also include some default settings adjustments as per recent
+development on the fork:
+
+- sticky mode is still turned on by default but now activates when
+  a using the `ll` repl command.
+- turn off line truncation by default to avoid inter-line gaps when
+  resizing the terimnal during use.
+- when using the backtrace cmd either by `w` or `bt`, the config
+  automatically switches to non-sticky mode.

pyproject.toml

@@ -0,0 +1,28 @@
+[tool.towncrier]
+package = "tractor"
+filename = "NEWS.rst"
+directory = "nooz/"
+version = "0.1.0a6"
+title_format = "tractor {version} ({project_date})"
+template = "nooz/_template.rst"
+all_bullets = true
+
+  [[tool.towncrier.type]]
+  directory = "feature"
+  name = "Features"
+  showcontent = true
+
+  [[tool.towncrier.type]]
+  directory = "bugfix"
+  name = "Bug Fixes"
+  showcontent = true
+
+  [[tool.towncrier.type]]
+  directory = "doc"
+  name = "Improved Documentation"
+  showcontent = true
+
+  [[tool.towncrier.type]]
+  directory = "trivial"
+  name = "Trivial/Internal Changes"
+  showcontent = true

requirements-test.txt

@@ -1,7 +1,8 @@
 pytest
 pytest-trio
-pdbpp
+pytest-timeout
-mypy<0.920
+pdbp
-trio_typing<0.7.0
+mypy
+trio_typing
 pexpect
 towncrier

setup.py

@@ -1,21 +1,22 @@
 #!/usr/bin/env python
 #
-# tractor: a trionic actor model built on `multiprocessing` and `trio`
+# tractor: structured concurrent "actors".
 #
-# Copyright (C) 2018-2020  Tyler Goodlet
+# Copyright 2018-eternity Tyler Goodlet.
 
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from setuptools import setup
 
 with open('docs/README.rst', encoding='utf-8') as f:
@@ -24,55 +25,62 @@ with open('docs/README.rst', encoding='utf-8') as f:
 
 setup(
     name="tractor",
-    version='0.1.0a5.dev',  # alpha zone
+    version='0.1.0a6dev0',  # alpha zone
-    description='structured concurrrent "actors"',
+    description='structured concurrrent `trio`-"actors"',
     long_description=readme,
-    license='GPLv3',
+    license='AGPLv3',
     author='Tyler Goodlet',
     maintainer='Tyler Goodlet',
-    maintainer_email='jgbt@protonmail.com',
+    maintainer_email='goodboy_foss@protonmail.com',
     url='https://github.com/goodboy/tractor',
     platforms=['linux', 'windows'],
     packages=[
         'tractor',
         'tractor.experimental',
         'tractor.trionics',
-        'tractor.testing',
     ],
     install_requires=[
 
         # trio related
-        'trio>0.8',
+        # proper range spec:
+        # https://packaging.python.org/en/latest/discussions/install-requires-vs-requirements/#id5
+        'trio >= 0.22',
         'async_generator',
         'trio_typing',
+        'exceptiongroup',
 
         # tooling
         'tricycle',
         'trio_typing',
-
-        # tooling
         'colorlog',
         'wrapt',
-        'pdbpp',
 
-        # serialization
+        # IPC serialization
-        'msgpack>=1.0.3',
+        'msgspec',
+
+        # debug mode REPL
+        'pdbp',
+
+        # pip ref docs on these specs:
+        # https://pip.pypa.io/en/stable/reference/requirement-specifiers/#examples
+        # and pep:
+        # https://peps.python.org/pep-0440/#version-specifiers
+
+        # windows deps workaround for ``pdbpp``
+        # https://github.com/pdbpp/pdbpp/issues/498
+        # https://github.com/pdbpp/fancycompleter/issues/37
+        'pyreadline3 ; platform_system == "Windows"',
 
     ],
-    extras_require={
-
-        # serialization
-        'msgspec': ['msgspec >= "0.4.0"'],
-
-    },
     tests_require=['pytest'],
-    python_requires=">=3.9",
+    python_requires=">=3.10",
     keywords=[
         'trio',
-        "async",
+        'async',
-        "concurrency",
+        'concurrency',
-        "actor model",
+        'structured concurrency',
-        "distributed",
+        'actor model',
+        'distributed',
         'multiprocessing'
     ],
     classifiers=[
@@ -80,11 +88,10 @@ setup(
         "Operating System :: POSIX :: Linux",
         "Operating System :: Microsoft :: Windows",
         "Framework :: Trio",
-        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+        "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.9",
         "Intended Audience :: Science/Research",
         "Intended Audience :: Developers",
         "Topic :: System :: Distributed Computing",

tests/conftest.py

@@ -7,16 +7,91 @@ import os
 import random
 import signal
 import platform
+import pathlib
 import time
+import inspect
+from functools import partial, wraps
 
 import pytest
+import trio
 import tractor
 
-# export for tests
-from tractor.testing import tractor_test  # noqa
-
-
 pytest_plugins = ['pytester']
+
+
+def tractor_test(fn):
+    """
+    Use:
+
+    @tractor_test
+    async def test_whatever():
+        await ...
+
+    If fixtures:
+
+        - ``arb_addr`` (a socket addr tuple where arbiter is listening)
+        - ``loglevel`` (logging level passed to tractor internals)
+        - ``start_method`` (subprocess spawning backend)
+
+    are defined in the `pytest` fixture space they will be automatically
+    injected to tests declaring these funcargs.
+    """
+    @wraps(fn)
+    def wrapper(
+        *args,
+        loglevel=None,
+        arb_addr=None,
+        start_method=None,
+        **kwargs
+    ):
+        # __tracebackhide__ = True
+
+        if 'arb_addr' in inspect.signature(fn).parameters:
+            # injects test suite fixture value to test as well
+            # as `run()`
+            kwargs['arb_addr'] = arb_addr
+
+        if 'loglevel' in inspect.signature(fn).parameters:
+            # allows test suites to define a 'loglevel' fixture
+            # that activates the internal logging
+            kwargs['loglevel'] = loglevel
+
+        if start_method is None:
+            if platform.system() == "Windows":
+                start_method = 'trio'
+
+        if 'start_method' in inspect.signature(fn).parameters:
+            # set of subprocess spawning backends
+            kwargs['start_method'] = start_method
+
+        if kwargs:
+
+            # use explicit root actor start
+
+            async def _main():
+                async with tractor.open_root_actor(
+                    # **kwargs,
+                    arbiter_addr=arb_addr,
+                    loglevel=loglevel,
+                    start_method=start_method,
+
+                    # TODO: only enable when pytest is passed --pdb
+                    # debug_mode=True,
+
+                ):
+                    await fn(*args, **kwargs)
+
+            main = _main
+
+        else:
+            # use implicit root actor start
+            main = partial(fn, *args, **kwargs)
+
+        return trio.run(main)
+
+    return wrapper
+
+
 _arb_addr = '127.0.0.1', random.randint(1000, 9999)
 
 
@@ -39,14 +114,21 @@ no_windows = pytest.mark.skipif(
 )
 
 
-def repodir():
+def repodir() -> pathlib.Path:
-    """Return the abspath to the repo directory.
+    '''
-    """
+    Return the abspath to the repo directory.
-    dirname = os.path.dirname
+
-    dirpath = os.path.abspath(
+    '''
-        dirname(dirname(os.path.realpath(__file__)))
+    # 2 parents up to step up through tests/<repo_dir>
-        )
+    return pathlib.Path(__file__).parent.parent.absolute()
-    return dirpath
+
+
+def examples_dir() -> pathlib.Path:
+    '''
+    Return the abspath to the examples directory as `pathlib.Path`.
+
+    '''
+    return repodir() / 'examples'
 
 
 def pytest_addoption(parser):
@@ -64,10 +146,6 @@ def pytest_addoption(parser):
 
 def pytest_configure(config):
     backend = config.option.spawn_backend
-
-    if backend == 'mp':
-        tractor._spawn.try_set_start_method('spawn')
-    elif backend == 'trio':
     tractor._spawn.try_set_start_method(backend)
 
 
@@ -81,15 +159,18 @@ def loglevel(request):
 
 
 @pytest.fixture(scope='session')
-def spawn_backend(request):
+def spawn_backend(request) -> str:
     return request.config.option.spawn_backend
 
 
+_ci_env: bool = os.environ.get('CI', False)
+
+
 @pytest.fixture(scope='session')
 def ci_env() -> bool:
     """Detect CI envoirment.
     """
-    return os.environ.get('TRAVIS', False) or os.environ.get('CI', False)
+    return _ci_env
 
 
 @pytest.fixture(scope='session')
@@ -99,24 +180,24 @@ def arb_addr():
 
 def pytest_generate_tests(metafunc):
     spawn_backend = metafunc.config.option.spawn_backend
+
     if not spawn_backend:
         # XXX some weird windows bug with `pytest`?
-        spawn_backend = 'mp'
+        spawn_backend = 'trio'
-    assert spawn_backend in ('mp', 'trio')
 
+    # TODO: maybe just use the literal `._spawn.SpawnMethodKey`?
+    assert spawn_backend in (
+        'mp_spawn',
+        'mp_forkserver',
+        'trio',
+    )
+
+    # NOTE: used to be used to dyanmically parametrize tests for when
+    # you just passed --spawn-backend=`mp` on the cli, but now we expect
+    # that cli input to be manually specified, BUT, maybe we'll do
+    # something like this again in the future?
     if 'start_method' in metafunc.fixturenames:
-        if spawn_backend == 'mp':
+        metafunc.parametrize("start_method", [spawn_backend], scope='module')
-            from multiprocessing import get_all_start_methods
-            methods = get_all_start_methods()
-            if 'fork' in methods:
-                # fork not available on windows, so check before
-                # removing XXX: the fork method is in general
-                # incompatible with trio's global scheduler state
-                methods.remove('fork')
-        elif spawn_backend == 'trio':
-            methods = ['trio']
-
-        metafunc.parametrize("start_method", methods, scope='module')
 
 
 def sig_prog(proc, sig):
@@ -132,16 +213,22 @@ def sig_prog(proc, sig):
 
 
 @pytest.fixture
-def daemon(loglevel, testdir, arb_addr):
+def daemon(
-    """Run a daemon actor as a "remote arbiter".
+    loglevel: str,
-    """
+    testdir,
+    arb_addr: tuple[str, int],
+):
+    '''
+    Run a daemon actor as a "remote arbiter".
+
+    '''
     if loglevel in ('trace', 'debug'):
         # too much logging will lock up the subproc (smh)
         loglevel = 'info'
 
     cmdargs = [
         sys.executable, '-c',
-        "import tractor; tractor.run_daemon([], arbiter_addr={}, loglevel={})"
+        "import tractor; tractor.run_daemon([], registry_addr={}, loglevel={})"
         .format(
             arb_addr,
             "'{}'".format(loglevel) if loglevel else None)

tests/test_advanced_faults.py

@@ -0,0 +1,193 @@
+'''
+Sketchy network blackoutz, ugly byzantine gens, puedes eschuchar la
+cancelacion?..
+
+'''
+from functools import partial
+
+import pytest
+from _pytest.pathlib import import_path
+import trio
+import tractor
+
+from conftest import (
+    examples_dir,
+)
+
+
+@pytest.mark.parametrize(
+    'debug_mode',
+    [False, True],
+    ids=['no_debug_mode', 'debug_mode'],
+)
+@pytest.mark.parametrize(
+    'ipc_break',
+    [
+        # no breaks
+        {
+            'break_parent_ipc_after': False,
+            'break_child_ipc_after': False,
+        },
+
+        # only parent breaks
+        {
+            'break_parent_ipc_after': 500,
+            'break_child_ipc_after': False,
+        },
+
+        # only child breaks
+        {
+            'break_parent_ipc_after': False,
+            'break_child_ipc_after': 500,
+        },
+
+        # both: break parent first
+        {
+            'break_parent_ipc_after': 500,
+            'break_child_ipc_after': 800,
+        },
+        # both: break child first
+        {
+            'break_parent_ipc_after': 800,
+            'break_child_ipc_after': 500,
+        },
+
+    ],
+    ids=[
+        'no_break',
+        'break_parent',
+        'break_child',
+        'break_both_parent_first',
+        'break_both_child_first',
+    ],
+)
+def test_ipc_channel_break_during_stream(
+    debug_mode: bool,
+    spawn_backend: str,
+    ipc_break: dict | None,
+):
+    '''
+    Ensure we can have an IPC channel break its connection during
+    streaming and it's still possible for the (simulated) user to kill
+    the actor tree using SIGINT.
+
+    We also verify the type of connection error expected in the parent
+    depending on which side if the IPC breaks first.
+
+    '''
+    if spawn_backend != 'trio':
+        if debug_mode:
+            pytest.skip('`debug_mode` only supported on `trio` spawner')
+
+        # non-`trio` spawners should never hit the hang condition that
+        # requires the user to do ctl-c to cancel the actor tree.
+        expect_final_exc = trio.ClosedResourceError
+
+    mod = import_path(
+        examples_dir() / 'advanced_faults' / 'ipc_failure_during_stream.py',
+        root=examples_dir(),
+    )
+
+    expect_final_exc = KeyboardInterrupt
+
+    # when ONLY the child breaks we expect the parent to get a closed
+    # resource error on the next `MsgStream.receive()` and then fail out
+    # and cancel the child from there.
+    if (
+
+        # only child breaks
+        (
+            ipc_break['break_child_ipc_after']
+            and ipc_break['break_parent_ipc_after'] is False
+        )
+
+        # both break but, parent breaks first
+        or (
+            ipc_break['break_child_ipc_after'] is not False
+            and (
+                ipc_break['break_parent_ipc_after']
+                > ipc_break['break_child_ipc_after']
+            )
+        )
+
+    ):
+        expect_final_exc = trio.ClosedResourceError
+
+    # when the parent IPC side dies (even if the child's does as well
+    # but the child fails BEFORE the parent) we expect the channel to be
+    # sent a stop msg from the child at some point which will signal the
+    # parent that the stream has been terminated.
+    # NOTE: when the parent breaks "after" the child you get this same
+    # case as well, the child breaks the IPC channel with a stop msg
+    # before any closure takes place.
+    elif (
+        # only parent breaks
+        (
+            ipc_break['break_parent_ipc_after']
+            and ipc_break['break_child_ipc_after'] is False
+        )
+
+        # both break but, child breaks first
+        or (
+            ipc_break['break_parent_ipc_after'] is not False
+            and (
+                ipc_break['break_child_ipc_after']
+                > ipc_break['break_parent_ipc_after']
+            )
+        )
+    ):
+        expect_final_exc = trio.EndOfChannel
+
+    with pytest.raises(expect_final_exc):
+        trio.run(
+            partial(
+                mod.main,
+                debug_mode=debug_mode,
+                start_method=spawn_backend,
+                **ipc_break,
+            )
+        )
+
+
+@tractor.context
+async def break_ipc_after_started(
+    ctx: tractor.Context,
+) -> None:
+    await ctx.started()
+    async with ctx.open_stream() as stream:
+        await stream.aclose()
+        await trio.sleep(0.2)
+        await ctx.chan.send(None)
+        print('child broke IPC and terminating')
+
+
+def test_stream_closed_right_after_ipc_break_and_zombie_lord_engages():
+    '''
+    Verify that is a subactor's IPC goes down just after bringing up a stream
+    the parent can trigger a SIGINT and the child will be reaped out-of-IPC by
+    the localhost process supervision machinery: aka "zombie lord".
+
+    '''
+    async def main():
+        async with tractor.open_nursery() as n:
+            portal = await n.start_actor(
+                'ipc_breaker',
+                enable_modules=[__name__],
+            )
+
+            with trio.move_on_after(1):
+                async with (
+                    portal.open_context(
+                        break_ipc_after_started
+                    ) as (ctx, sent),
+                ):
+                    async with ctx.open_stream():
+                        await trio.sleep(0.5)
+
+                    print('parent waiting on context')
+
+            print('parent exited context')
+            raise KeyboardInterrupt
+
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(main)

tests/test_advanced_streaming.py

@@ -5,7 +5,6 @@ Advanced streaming patterns using bidirectional streams and contexts.
 from collections import Counter
 import itertools
 import platform
-from typing import Set, Dict, List
 
 import trio
 import tractor
@@ -15,7 +14,7 @@ def is_win():
     return platform.system() == 'Windows'
 
 
-_registry: Dict[str, Set[tractor.ReceiveMsgStream]] = {
+_registry: dict[str, set[tractor.MsgStream]] = {
     'even': set(),
     'odd': set(),
 }
@@ -77,7 +76,7 @@ async def subscribe(
 
 async def consumer(
 
-    subs: List[str],
+    subs: list[str],
 
 ) -> None:
 

tests/test_cancellation.py

@@ -8,6 +8,10 @@ import platform
 import time
 from itertools import repeat
 
+from exceptiongroup import (
+    BaseExceptionGroup,
+    ExceptionGroup,
+)
 import pytest
 import trio
 import tractor
@@ -56,29 +60,49 @@ def test_remote_error(arb_addr, args_err):
             arbiter_addr=arb_addr,
         ) as nursery:
 
+            # on a remote type error caused by bad input args
+            # this should raise directly which means we **don't** get
+            # an exception group outside the nursery since the error
+            # here and the far end task error are one in the same?
             portal = await nursery.run_in_actor(
                 assert_err, name='errorer', **args
             )
 
             # get result(s) from main task
             try:
+                # this means the root actor will also raise a local
+                # parent task error and thus an eg will propagate out
+                # of this actor nursery.
                 await portal.result()
             except tractor.RemoteActorError as err:
                 assert err.type == errtype
                 print("Look Maa that actor failed hard, hehh")
                 raise
 
+    # ensure boxed errors
+    if args:
         with pytest.raises(tractor.RemoteActorError) as excinfo:
             trio.run(main)
 
-    # ensure boxed error is correct
         assert excinfo.value.type == errtype
 
+    else:
+        # the root task will also error on the `.result()` call
+        # so we expect an error from there AND the child.
+        with pytest.raises(BaseExceptionGroup) as excinfo:
+            trio.run(main)
+
+        # ensure boxed errors
+        for exc in excinfo.value.exceptions:
+            assert exc.type == errtype
+
 
 def test_multierror(arb_addr):
-    """Verify we raise a ``trio.MultiError`` out of a nursery where
+    '''
+    Verify we raise a ``BaseExceptionGroup`` out of a nursery where
     more then one actor errors.
-    """
+
+    '''
     async def main():
         async with tractor.open_nursery(
             arbiter_addr=arb_addr,
@@ -95,10 +119,10 @@ def test_multierror(arb_addr):
                 print("Look Maa that first actor failed hard, hehh")
                 raise
 
-        # here we should get a `trio.MultiError` containing exceptions
+        # here we should get a ``BaseExceptionGroup`` containing exceptions
         # from both subactors
 
-    with pytest.raises(trio.MultiError):
+    with pytest.raises(BaseExceptionGroup):
         trio.run(main)
 
 
@@ -107,7 +131,7 @@ def test_multierror(arb_addr):
     'num_subactors', range(25, 26),
 )
 def test_multierror_fast_nursery(arb_addr, start_method, num_subactors, delay):
-    """Verify we raise a ``trio.MultiError`` out of a nursery where
+    """Verify we raise a ``BaseExceptionGroup`` out of a nursery where
     more then one actor errors and also with a delay before failure
     to test failure during an ongoing spawning.
     """
@@ -123,10 +147,11 @@ def test_multierror_fast_nursery(arb_addr, start_method, num_subactors, delay):
                     delay=delay
                 )
 
-    with pytest.raises(trio.MultiError) as exc_info:
+    # with pytest.raises(trio.MultiError) as exc_info:
+    with pytest.raises(BaseExceptionGroup) as exc_info:
         trio.run(main)
 
-    assert exc_info.type == tractor.MultiError
+    assert exc_info.type == ExceptionGroup
     err = exc_info.value
     exceptions = err.exceptions
 
@@ -214,8 +239,8 @@ async def test_cancel_infinite_streamer(start_method):
     [
         # daemon actors sit idle while single task actors error out
         (1, tractor.RemoteActorError, AssertionError, (assert_err, {}), None),
-        (2, tractor.MultiError, AssertionError, (assert_err, {}), None),
+        (2, BaseExceptionGroup, AssertionError, (assert_err, {}), None),
-        (3, tractor.MultiError, AssertionError, (assert_err, {}), None),
+        (3, BaseExceptionGroup, AssertionError, (assert_err, {}), None),
 
         # 1 daemon actor errors out while single task actors sleep forever
         (3, tractor.RemoteActorError, AssertionError, (sleep_forever, {}),
@@ -226,7 +251,7 @@ async def test_cancel_infinite_streamer(start_method):
          (do_nuthin, {}), (assert_err, {'delay': 1}, True)),
         # daemon complete quickly delay while single task
         # actors error after brief delay
-        (3, tractor.MultiError, AssertionError,
+        (3, BaseExceptionGroup, AssertionError,
          (assert_err, {'delay': 1}), (do_nuthin, {}, False)),
     ],
     ids=[
@@ -293,7 +318,7 @@ async def test_some_cancels_all(num_actors_and_errs, start_method, loglevel):
         # should error here with a ``RemoteActorError`` or ``MultiError``
 
     except first_err as err:
-        if isinstance(err, tractor.MultiError):
+        if isinstance(err, BaseExceptionGroup):
             assert len(err.exceptions) == num_actors
             for exc in err.exceptions:
                 if isinstance(exc, tractor.RemoteActorError):
@@ -337,7 +362,7 @@ async def spawn_and_error(breadth, depth) -> None:
 @tractor_test
 async def test_nested_multierrors(loglevel, start_method):
     '''
-    Test that failed actor sets are wrapped in `trio.MultiError`s. This
+    Test that failed actor sets are wrapped in `BaseExceptionGroup`s. This
     test goes only 2 nurseries deep but we should eventually have tests
     for arbitrary n-depth actor trees.
 
@@ -365,7 +390,7 @@ async def test_nested_multierrors(loglevel, start_method):
                         breadth=subactor_breadth,
                         depth=depth,
                     )
-        except trio.MultiError as err:
+        except BaseExceptionGroup as err:
             assert len(err.exceptions) == subactor_breadth
             for subexc in err.exceptions:
 
@@ -383,10 +408,10 @@ async def test_nested_multierrors(loglevel, start_method):
                         assert subexc.type in (
                             tractor.RemoteActorError,
                             trio.Cancelled,
-                            trio.MultiError
+                            BaseExceptionGroup,
                         )
 
-                    elif isinstance(subexc, trio.MultiError):
+                    elif isinstance(subexc, BaseExceptionGroup):
                         for subsub in subexc.exceptions:
 
                             if subsub in (tractor.RemoteActorError,):
@@ -394,7 +419,7 @@ async def test_nested_multierrors(loglevel, start_method):
 
                             assert type(subsub) in (
                                 trio.Cancelled,
-                                trio.MultiError,
+                                BaseExceptionGroup,
                             )
                 else:
                     assert isinstance(subexc, tractor.RemoteActorError)
@@ -406,13 +431,13 @@ async def test_nested_multierrors(loglevel, start_method):
                     if is_win():
                         if isinstance(subexc, tractor.RemoteActorError):
                             assert subexc.type in (
-                                trio.MultiError,
+                                BaseExceptionGroup,
                                 tractor.RemoteActorError
                             )
                         else:
-                            assert isinstance(subexc, trio.MultiError)
+                            assert isinstance(subexc, BaseExceptionGroup)
                     else:
-                        assert subexc.type is trio.MultiError
+                        assert subexc.type is ExceptionGroup
                 else:
                     assert subexc.type in (
                         tractor.RemoteActorError,
@@ -435,7 +460,7 @@ def test_cancel_via_SIGINT(
         with trio.fail_after(2):
             async with tractor.open_nursery() as tn:
                 await tn.start_actor('sucka')
-                if spawn_backend == 'mp':
+                if 'mp' in spawn_backend:
                     time.sleep(0.1)
                 os.kill(pid, signal.SIGINT)
                 await trio.sleep_forever()
@@ -474,7 +499,7 @@ def test_cancel_via_SIGINT_other_task(
         with trio.fail_after(timeout):
             async with trio.open_nursery() as n:
                 await n.start(spawn_and_sleep_forever)
-                if spawn_backend == 'mp':
+                if 'mp' in spawn_backend:
                     time.sleep(0.1)
                 os.kill(pid, signal.SIGINT)
 

tests/test_child_manages_service_nursery.py

@@ -0,0 +1,173 @@
+'''
+Test a service style daemon that maintains a nursery for spawning
+"remote async tasks" including both spawning other long living
+sub-sub-actor daemons.
+
+'''
+from typing import Optional
+import asyncio
+from contextlib import asynccontextmanager as acm
+
+import pytest
+import trio
+from trio_typing import TaskStatus
+import tractor
+from tractor import RemoteActorError
+from async_generator import aclosing
+
+
+async def aio_streamer(
+    from_trio: asyncio.Queue,
+    to_trio: trio.abc.SendChannel,
+) -> trio.abc.ReceiveChannel:
+
+    # required first msg to sync caller
+    to_trio.send_nowait(None)
+
+    from itertools import cycle
+    for i in cycle(range(10)):
+        to_trio.send_nowait(i)
+        await asyncio.sleep(0.01)
+
+
+async def trio_streamer():
+    from itertools import cycle
+    for i in cycle(range(10)):
+        yield i
+        await trio.sleep(0.01)
+
+
+async def trio_sleep_and_err(delay: float = 0.5):
+    await trio.sleep(delay)
+    # name error
+    doggy()  # noqa
+
+
+_cached_stream: Optional[
+    trio.abc.ReceiveChannel
+] = None
+
+
+@acm
+async def wrapper_mngr(
+):
+    from tractor.trionics import broadcast_receiver
+    global _cached_stream
+    in_aio = tractor.current_actor().is_infected_aio()
+
+    if in_aio:
+        if _cached_stream:
+
+            from_aio = _cached_stream
+
+            # if we already have a cached feed deliver a rx side clone
+            # to consumer
+            async with broadcast_receiver(from_aio, 6) as from_aio:
+                yield from_aio
+                return
+        else:
+            async with tractor.to_asyncio.open_channel_from(
+                aio_streamer,
+            ) as (first, from_aio):
+                assert not first
+
+                # cache it so next task uses broadcast receiver
+                _cached_stream = from_aio
+
+                yield from_aio
+    else:
+        async with aclosing(trio_streamer()) as stream:
+            # cache it so next task uses broadcast receiver
+            _cached_stream = stream
+            yield stream
+
+
+_nursery: trio.Nursery = None
+
+
+@tractor.context
+async def trio_main(
+    ctx: tractor.Context,
+):
+    # sync
+    await ctx.started()
+
+    # stash a "service nursery" as "actor local" (aka a Python global)
+    global _nursery
+    n = _nursery
+    assert n
+
+    async def consume_stream():
+        async with wrapper_mngr() as stream:
+            async for msg in stream:
+                print(msg)
+
+    # run 2 tasks to ensure broadcaster chan use
+    n.start_soon(consume_stream)
+    n.start_soon(consume_stream)
+
+    n.start_soon(trio_sleep_and_err)
+
+    await trio.sleep_forever()
+
+
+@tractor.context
+async def open_actor_local_nursery(
+    ctx: tractor.Context,
+):
+    global _nursery
+    async with trio.open_nursery() as n:
+        _nursery = n
+        await ctx.started()
+        await trio.sleep(10)
+        # await trio.sleep(1)
+
+        # XXX: this causes the hang since
+        # the caller does not unblock from its own
+        # ``trio.sleep_forever()``.
+
+        # TODO: we need to test a simple ctx task starting remote tasks
+        # that error and then blocking on a ``Nursery.start()`` which
+        # never yields back.. aka a scenario where the
+        # ``tractor.context`` task IS NOT in the service n's cancel
+        # scope.
+        n.cancel_scope.cancel()
+
+
+@pytest.mark.parametrize(
+    'asyncio_mode',
+    [True, False],
+    ids='asyncio_mode={}'.format,
+)
+def test_actor_managed_trio_nursery_task_error_cancels_aio(
+    asyncio_mode: bool,
+    arb_addr
+):
+    '''
+    Verify that a ``trio`` nursery created managed in a child actor
+    correctly relays errors to the parent actor when one of its spawned
+    tasks errors even when running in infected asyncio mode and using
+    broadcast receivers for multi-task-per-actor subscription.
+
+    '''
+    async def main():
+
+        # cancel the nursery shortly after boot
+        async with tractor.open_nursery() as n:
+            p = await n.start_actor(
+                'nursery_mngr',
+                infect_asyncio=asyncio_mode,
+                enable_modules=[__name__],
+            )
+            async with (
+                p.open_context(open_actor_local_nursery) as (ctx, first),
+                p.open_context(trio_main) as (ctx, first),
+            ):
+                await trio.sleep_forever()
+
+    with pytest.raises(RemoteActorError) as excinfo:
+        trio.run(main)
+
+    # verify boxed error
+    err = excinfo.value
+    assert isinstance(err.type(), NameError)

tests/test_clustering.py

@@ -1,5 +1,6 @@
 import itertools
 
+import pytest
 import trio
 import tractor
 from tractor import open_actor_cluster
@@ -11,26 +12,72 @@ from conftest import tractor_test
 MESSAGE = 'tractoring at full speed'
 
 
+def test_empty_mngrs_input_raises() -> None:
+
+    async def main():
+        with trio.fail_after(1):
+            async with (
+                open_actor_cluster(
+                    modules=[__name__],
+
+                    # NOTE: ensure we can passthrough runtime opts
+                    loglevel='info',
+                    # debug_mode=True,
+
+                ) as portals,
+
+                gather_contexts(
+                    # NOTE: it's the use of inline-generator syntax
+                    # here that causes the empty input.
+                    mngrs=(
+                        p.open_context(worker) for p in portals.values()
+                    ),
+                ),
+            ):
+                assert 0
+
+    with pytest.raises(ValueError):
+        trio.run(main)
+
+
 @tractor.context
-async def worker(ctx: tractor.Context) -> None:
+async def worker(
+    ctx: tractor.Context,
+
+) -> None:
+
     await ctx.started()
-    async with ctx.open_stream(backpressure=True) as stream:
+
+    async with ctx.open_stream(
+        backpressure=True,
+    ) as stream:
+
+        # TODO: this with the below assert causes a hang bug?
+        # with trio.move_on_after(1):
+
         async for msg in stream:
             # do something with msg
             print(msg)
             assert msg == MESSAGE
 
+        # TODO: does this ever cause a hang
+        # assert 0
+
 
 @tractor_test
 async def test_streaming_to_actor_cluster() -> None:
+
     async with (
         open_actor_cluster(modules=[__name__]) as portals,
+
         gather_contexts(
             mngrs=[p.open_context(worker) for p in portals.values()],
         ) as contexts,
+
         gather_contexts(
             mngrs=[ctx[0].open_stream() for ctx in contexts],
         ) as streams,
+
     ):
         with trio.move_on_after(1):
             for stream in itertools.cycle(streams):

tests/test_context_stream_semantics.py

@@ -265,6 +265,7 @@ async def test_callee_closes_ctx_after_stream_open():
             enable_modules=[__name__],
         )
 
+        with trio.fail_after(2):
             async with portal.open_context(
                 close_ctx_immediately,
 
@@ -297,6 +298,7 @@ async def test_callee_closes_ctx_after_stream_open():
                 # of a stream to the context (at least until a time of
                 # if/when we decide that's a good idea?)
                 try:
+                    with trio.fail_after(0.5):
                         async with ctx.open_stream() as stream:
                             pass
                 except trio.ClosedResourceError:
@@ -569,7 +571,7 @@ def test_one_end_stream_not_opened(overrun_by):
 
     '''
     overrunner, buf_size_increase, entrypoint = overrun_by
-    from tractor._actor import Actor
+    from tractor._runtime import Actor
     buf_size = buf_size_increase + Actor.msg_buffer_size
 
     async def main():

tests/test_debugger.py

@@ -1,5 +1,5 @@
 """
-That native debug better work!
+That "native" debug mode better work!
 
 All these tests can be understood (somewhat) by running the equivalent
 `examples/debugging/` scripts manually.
@@ -10,15 +10,25 @@ TODO:
     - wonder if any of it'll work on OS X?
 
 """
-import time
+import itertools
 from os import path
+from typing import Optional
 import platform
+import pathlib
+import sys
+import time
 
 import pytest
 import pexpect
+from pexpect.exceptions import (
+    TIMEOUT,
+    EOF,
+)
 
-from conftest import repodir
+from conftest import (
-
+    examples_dir,
+    _ci_env,
+)
 
 # TODO: The next great debugger audit could be done by you!
 # - recurrent entry to breakpoint() from single actor *after* and an
@@ -37,19 +47,31 @@ if platform.system() == 'Windows':
     )
 
 
-def examples_dir():
-    """Return the abspath to the examples directory.
-    """
-    return path.join(repodir(), 'examples', 'debugging/')
-
-
 def mk_cmd(ex_name: str) -> str:
-    """Generate a command suitable to pass to ``pexpect.spawn()``.
+    '''
-    """
+    Generate a command suitable to pass to ``pexpect.spawn()``.
-    return ' '.join(
+
-        ['python',
+    '''
-         path.join(examples_dir(), f'{ex_name}.py')]
+    script_path: pathlib.Path = examples_dir() / 'debugging' / f'{ex_name}.py'
-    )
+    return ' '.join(['python', str(script_path)])
+
+
+# TODO: was trying to this xfail style but some weird bug i see in CI
+# that's happening at collect time.. pretty soon gonna dump actions i'm
+# thinkin...
+# in CI we skip tests which >= depth 1 actor trees due to there
+# still being an oustanding issue with relaying the debug-mode-state
+# through intermediary parents.
+has_nested_actors = pytest.mark.has_nested_actors
+# .xfail(
+#     os.environ.get('CI', False),
+#     reason=(
+#         'This test uses nested actors and fails in CI\n'
+#         'The test seems to run fine locally but until we solve the '
+#         'following issue this CI test will be xfail:\n'
+#         'https://github.com/goodboy/tractor/issues/320'
+#     )
+# )
 
 
 @pytest.fixture
@@ -73,6 +95,83 @@ def spawn(
     return _spawn
 
 
+PROMPT = r"\(Pdb\+\)"
+
+
+def expect(
+    child,
+
+    # prompt by default
+    patt: str = PROMPT,
+
+    **kwargs,
+
+) -> None:
+    '''
+    Expect wrapper that prints last seen console
+    data before failing.
+
+    '''
+    try:
+        child.expect(
+            patt,
+            **kwargs,
+        )
+    except TIMEOUT:
+        before = str(child.before.decode())
+        print(before)
+        raise
+
+
+def assert_before(
+    child,
+    patts: list[str],
+
+) -> None:
+
+    before = str(child.before.decode())
+
+    for patt in patts:
+        try:
+            assert patt in before
+        except AssertionError:
+            print(before)
+            raise
+
+
+@pytest.fixture(
+    params=[False, True],
+    ids='ctl-c={}'.format,
+)
+def ctlc(
+    request,
+    ci_env: bool,
+
+) -> bool:
+
+    use_ctlc = request.param
+
+    node = request.node
+    markers = node.own_markers
+    for mark in markers:
+        if mark.name == 'has_nested_actors':
+            pytest.skip(
+                f'Test {node} has nested actors and fails with Ctrl-C.\n'
+                f'The test can sometimes run fine locally but until'
+                ' we solve' 'this issue this CI test will be xfail:\n'
+                'https://github.com/goodboy/tractor/issues/320'
+            )
+
+    if use_ctlc:
+        # XXX: disable pygments highlighting for auto-tests
+        # since some envs (like actions CI) will struggle
+        # the the added color-char encoding..
+        from tractor._debug import TractorConfig
+        TractorConfig.use_pygements = False
+
+    yield use_ctlc
+
+
 @pytest.mark.parametrize(
     'user_in_out',
     [
@@ -82,14 +181,16 @@ def spawn(
     ids=lambda item: f'{item[0]} -> {item[1]}',
 )
 def test_root_actor_error(spawn, user_in_out):
-    """Demonstrate crash handler entering pdbpp from basic error in root actor.
+    '''
-    """
+    Demonstrate crash handler entering pdb from basic error in root actor.
+
+    '''
     user_input, expect_err_str = user_in_out
 
     child = spawn('root_actor_error')
 
-    # scan for the pdbpp prompt
+    # scan for the prompt
-    child.expect(r"\(Pdb\+\+\)")
+    expect(child, PROMPT)
 
     before = str(child.before.decode())
 
@@ -101,7 +202,7 @@ def test_root_actor_error(spawn, user_in_out):
     child.sendline(user_input)
 
     # process should exit
-    child.expect(pexpect.EOF)
+    expect(child, EOF)
     assert expect_err_str in str(child.before)
 
 
@@ -119,8 +220,8 @@ def test_root_actor_bp(spawn, user_in_out):
     user_input, expect_err_str = user_in_out
     child = spawn('root_actor_breakpoint')
 
-    # scan for the pdbpp prompt
+    # scan for the prompt
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
 
     assert 'Error' not in str(child.before)
 
@@ -137,56 +238,129 @@ def test_root_actor_bp(spawn, user_in_out):
         assert expect_err_str in str(child.before)
 
 
-def test_root_actor_bp_forever(spawn):
+def do_ctlc(
+    child,
+    count: int = 3,
+    delay: float = 0.1,
+    patt: Optional[str] = None,
+
+    # expect repl UX to reprint the prompt after every
+    # ctrl-c send.
+    # XXX: no idea but, in CI this never seems to work even on 3.10 so
+    # needs some further investigation potentially...
+    expect_prompt: bool = not _ci_env,
+
+) -> None:
+
+    # make sure ctl-c sends don't do anything but repeat output
+    for _ in range(count):
+        time.sleep(delay)
+        child.sendcontrol('c')
+
+        # TODO: figure out why this makes CI fail..
+        # if you run this test manually it works just fine..
+        if expect_prompt:
+            before = str(child.before.decode())
+            time.sleep(delay)
+            child.expect(PROMPT)
+            time.sleep(delay)
+
+            if patt:
+                # should see the last line on console
+                assert patt in before
+
+
+def test_root_actor_bp_forever(
+    spawn,
+    ctlc: bool,
+):
     "Re-enter a breakpoint from the root actor-task."
     child = spawn('root_actor_breakpoint_forever')
 
     # do some "next" commands to demonstrate recurrent breakpoint
     # entries
     for _ in range(10):
-        child.sendline('next')
-        child.expect(r"\(Pdb\+\+\)")
 
-    # do one continue which should trigger a new task to lock the tty
+        child.expect(PROMPT)
+
+        if ctlc:
+            do_ctlc(child)
+
+        child.sendline('next')
+
+    # do one continue which should trigger a
+    # new task to lock the tty
     child.sendline('continue')
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
+
+    # seems that if we hit ctrl-c too fast the
+    # sigint guard machinery might not kick in..
+    time.sleep(0.001)
+
+    if ctlc:
+        do_ctlc(child)
 
     # XXX: this previously caused a bug!
     child.sendline('n')
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
 
     child.sendline('n')
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
+
+    # quit out of the loop
+    child.sendline('q')
+    child.expect(pexpect.EOF)
 
 
-def test_subactor_error(spawn):
+@pytest.mark.parametrize(
-    "Single subactor raising an error"
+    'do_next',
+    (True, False),
+    ids='do_next={}'.format,
+)
+def test_subactor_error(
+    spawn,
+    ctlc: bool,
+    do_next: bool,
+):
+    '''
+    Single subactor raising an error
 
+    '''
     child = spawn('subactor_error')
 
-    # scan for the pdbpp prompt
+    # scan for the prompt
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
 
     before = str(child.before.decode())
     assert "Attaching to pdb in crashed actor: ('name_error'" in before
 
-    # send user command
+    if do_next:
-    # (in this case it's the same for 'continue' vs. 'quit')
+        child.sendline('n')
+
+    else:
+        # make sure ctl-c sends don't do anything but repeat output
+        if ctlc:
+            do_ctlc(
+                child,
+            )
+
+        # send user command and (in this case it's the same for 'continue'
+        # vs. 'quit') the debugger should enter a second time in the nursery
+        # creating actor
         child.sendline('continue')
 
-    # the debugger should enter a second time in the nursery
+    child.expect(PROMPT)
-    # creating actor
-
-    child.expect(r"\(Pdb\+\+\)")
-
     before = str(child.before.decode())
 
     # root actor gets debugger engaged
     assert "Attaching to pdb in crashed actor: ('root'" in before
-
     # error is a remote error propagated from the subactor
     assert "RemoteActorError: ('name_error'" in before
 
+    # another round
+    if ctlc:
+        do_ctlc(child)
+
     child.sendline('c')
     child.expect('\r\n')
 
@@ -194,13 +368,16 @@ def test_subactor_error(spawn):
     child.expect(pexpect.EOF)
 
 
-def test_subactor_breakpoint(spawn):
+def test_subactor_breakpoint(
+    spawn,
+    ctlc: bool,
+):
     "Single subactor with an infinite breakpoint loop"
 
     child = spawn('subactor_breakpoint')
 
-    # scan for the pdbpp prompt
+    # scan for the prompt
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
 
     before = str(child.before.decode())
     assert "Attaching pdb to actor: ('breakpoint_forever'" in before
@@ -209,25 +386,34 @@ def test_subactor_breakpoint(spawn):
     # entries
     for _ in range(10):
         child.sendline('next')
-        child.expect(r"\(Pdb\+\+\)")
+        child.expect(PROMPT)
+
+        if ctlc:
+            do_ctlc(child)
 
     # now run some "continues" to show re-entries
     for _ in range(5):
         child.sendline('continue')
-        child.expect(r"\(Pdb\+\+\)")
+        child.expect(PROMPT)
         before = str(child.before.decode())
         assert "Attaching pdb to actor: ('breakpoint_forever'" in before
 
+        if ctlc:
+            do_ctlc(child)
+
     # finally quit the loop
     child.sendline('q')
 
     # child process should exit but parent will capture pdb.BdbQuit
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
 
     before = str(child.before.decode())
     assert "RemoteActorError: ('breakpoint_forever'" in before
     assert 'bdb.BdbQuit' in before
 
+    if ctlc:
+        do_ctlc(child)
+
     # quit the parent
     child.sendline('c')
 
@@ -239,110 +425,159 @@ def test_subactor_breakpoint(spawn):
     assert 'bdb.BdbQuit' in before
 
 
-def test_multi_subactors(spawn):
+@has_nested_actors
-    """
+def test_multi_subactors(
-    Multiple subactors, both erroring and breakpointing as well as
+    spawn,
-    a nested subactor erroring.
+    ctlc: bool,
-    """
+):
+    '''
+    Multiple subactors, both erroring and
+    breakpointing as well as a nested subactor erroring.
+
+    '''
     child = spawn(r'multi_subactors')
 
-    # scan for the pdbpp prompt
+    # scan for the prompt
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
 
     before = str(child.before.decode())
     assert "Attaching pdb to actor: ('breakpoint_forever'" in before
 
+    if ctlc:
+        do_ctlc(child)
+
     # do some "next" commands to demonstrate recurrent breakpoint
     # entries
     for _ in range(10):
         child.sendline('next')
-        child.expect(r"\(Pdb\+\+\)")
+        child.expect(PROMPT)
+
+        if ctlc:
+            do_ctlc(child)
 
     # continue to next error
     child.sendline('c')
 
     # first name_error failure
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
     before = str(child.before.decode())
     assert "Attaching to pdb in crashed actor: ('name_error'" in before
     assert "NameError" in before
 
+    if ctlc:
+        do_ctlc(child)
+
     # continue again
     child.sendline('c')
 
     # 2nd name_error failure
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
-    before = str(child.before.decode())
+
-    assert "Attaching to pdb in crashed actor: ('name_error_1'" in before
+    # TODO: will we ever get the race where this crash will show up?
-    assert "NameError" in before
+    # blocklist strat now prevents this crash
+    # assert_before(child, [
+    #     "Attaching to pdb in crashed actor: ('name_error_1'",
+    #     "NameError",
+    # ])
+
+    if ctlc:
+        do_ctlc(child)
 
     # breakpoint loop should re-engage
     child.sendline('c')
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
     before = str(child.before.decode())
     assert "Attaching pdb to actor: ('breakpoint_forever'" in before
 
+    if ctlc:
+        do_ctlc(child)
+
     # wait for spawn error to show up
     spawn_err = "Attaching to pdb in crashed actor: ('spawn_error'"
-    while spawn_err not in before:
+    start = time.time()
+    while (
+        spawn_err not in before
+        and (time.time() - start) < 3  # timeout eventually
+    ):
         child.sendline('c')
         time.sleep(0.1)
-        child.expect(r"\(Pdb\+\+\)")
+        child.expect(PROMPT)
         before = str(child.before.decode())
 
+        if ctlc:
+            do_ctlc(child)
+
     # 2nd depth nursery should trigger
-    # child.sendline('c')
+    # (XXX: this below if guard is technically a hack that makes the
-    # child.expect(r"\(Pdb\+\+\)")
+    # nested case seem to work locally on linux but ideally in the long
-    # before = str(child.before.decode())
+    # run this can be dropped.)
-    assert spawn_err in before
+    if not ctlc:
-    assert "RemoteActorError: ('name_error_1'" in before
+        assert_before(child, [
+            spawn_err,
+            "RemoteActorError: ('name_error_1'",
+        ])
 
     # now run some "continues" to show re-entries
     for _ in range(5):
         child.sendline('c')
-        child.expect(r"\(Pdb\+\+\)")
+        child.expect(PROMPT)
 
     # quit the loop and expect parent to attach
     child.sendline('q')
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
     before = str(child.before.decode())
+
+    assert_before(child, [
         # debugger attaches to root
-    assert "Attaching to pdb in crashed actor: ('root'" in before
+        "Attaching to pdb in crashed actor: ('root'",
+
         # expect a multierror with exceptions for each sub-actor
-    assert "RemoteActorError: ('breakpoint_forever'" in before
+        "RemoteActorError: ('breakpoint_forever'",
-    assert "RemoteActorError: ('name_error'" in before
+        "RemoteActorError: ('name_error'",
-    assert "RemoteActorError: ('spawn_error'" in before
+        "RemoteActorError: ('spawn_error'",
-    assert "RemoteActorError: ('name_error_1'" in before
+        "RemoteActorError: ('name_error_1'",
-    assert 'bdb.BdbQuit' in before
+        'bdb.BdbQuit',
+    ])
+
+    if ctlc:
+        do_ctlc(child)
 
     # process should exit
     child.sendline('c')
     child.expect(pexpect.EOF)
+
     # repeat of previous multierror for final output
-    before = str(child.before.decode())
+    assert_before(child, [
-    assert "RemoteActorError: ('breakpoint_forever'" in before
+        "RemoteActorError: ('breakpoint_forever'",
-    assert "RemoteActorError: ('name_error'" in before
+        "RemoteActorError: ('name_error'",
-    assert "RemoteActorError: ('spawn_error'" in before
+        "RemoteActorError: ('spawn_error'",
-    assert "RemoteActorError: ('name_error_1'" in before
+        "RemoteActorError: ('name_error_1'",
-    assert 'bdb.BdbQuit' in before
+        'bdb.BdbQuit',
+    ])
 
 
-def test_multi_daemon_subactors(spawn, loglevel):
+def test_multi_daemon_subactors(
-    """Multiple daemon subactors, both erroring and breakpointing within a
+    spawn,
+    loglevel: str,
+    ctlc: bool
+):
+    '''
+    Multiple daemon subactors, both erroring and breakpointing within a
     stream.
-    """
+
+    '''
     child = spawn('multi_daemon_subactors')
 
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
 
-    # there is a race for which subactor will acquire
+    # there can be a race for which subactor will acquire
-    # the root's tty lock first
+    # the root's tty lock first so anticipate either crash
-
+    # message on the first entry.
-    before = str(child.before.decode())
 
     bp_forever_msg = "Attaching pdb to actor: ('bp_forever'"
-    name_error_msg = "NameError"
+    name_error_msg = "NameError: name 'doggypants' is not defined"
 
+    before = str(child.before.decode())
     if bp_forever_msg in before:
         next_msg = name_error_msg
 
@@ -352,6 +587,9 @@ def test_multi_daemon_subactors(spawn, loglevel):
     else:
         raise ValueError("Neither log msg was found !?")
 
+    if ctlc:
+        do_ctlc(child)
+
     # NOTE: previously since we did not have clobber prevention
     # in the root actor this final resume could result in the debugger
     # tearing down since both child actors would be cancelled and it was
@@ -360,10 +598,8 @@ def test_multi_daemon_subactors(spawn, loglevel):
     # second entry by `bp_forever`.
 
     child.sendline('c')
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
-    before = str(child.before.decode())
+    assert_before(child, [next_msg])
-
-    assert next_msg in before
 
     # XXX: hooray the root clobbering the child here was fixed!
     # IMO, this demonstrates the true power of SC system design.
@@ -371,7 +607,7 @@ def test_multi_daemon_subactors(spawn, loglevel):
     # now the root actor won't clobber the bp_forever child
     # during it's first access to the debug lock, but will instead
     # wait for the lock to release, by the edge triggered
-    # ``_debug._no_remote_has_tty`` event before sending cancel messages
+    # ``_debug.Lock.no_remote_has_tty`` event before sending cancel messages
     # (via portals) to its underlings B)
 
     # at some point here there should have been some warning msg from
@@ -379,32 +615,61 @@ def test_multi_daemon_subactors(spawn, loglevel):
     # it seems unreliable in testing here to gnab it:
     # assert "in use by child ('bp_forever'," in before
 
-    # wait for final error in root
+    if ctlc:
-    while True:
+        do_ctlc(child)
+
+    # expect another breakpoint actor entry
+    child.sendline('c')
+    child.expect(PROMPT)
+
+    try:
+        assert_before(child, [bp_forever_msg])
+    except AssertionError:
+        assert_before(child, [name_error_msg])
+
+    else:
+        if ctlc:
+            do_ctlc(child)
+
+        # should crash with the 2nd name error (simulates
+        # a retry) and then the root eventually (boxed) errors
+        # after 1 or more further bp actor entries.
 
         child.sendline('c')
-        child.expect(r"\(Pdb\+\+\)")
+        child.expect(PROMPT)
-        before = str(child.before.decode())
+        assert_before(child, [name_error_msg])
-        try:
 
-            # root error should be packed as remote error
+    # wait for final error in root
-            assert "_exceptions.RemoteActorError: ('name_error'" in before
+    # where it crashs with boxed error
+    while True:
+        try:
+            child.sendline('c')
+            child.expect(PROMPT)
+            assert_before(
+                child,
+                [bp_forever_msg]
+            )
+        except AssertionError:
             break
 
-        except AssertionError:
+    assert_before(
-            assert bp_forever_msg in before
+        child,
+        [
+            # boxed error raised in root task
+            "Attaching to pdb in crashed actor: ('root'",
+            "_exceptions.RemoteActorError: ('name_error'",
+        ]
+    )
 
-    try:
     child.sendline('c')
     child.expect(pexpect.EOF)
 
-    except pexpect.exceptions.TIMEOUT:
-        # Failed to exit using continue..?
-        child.sendline('q')
-        child.expect(pexpect.EOF)
 
-
+@has_nested_actors
-def test_multi_subactors_root_errors(spawn):
+def test_multi_subactors_root_errors(
+    spawn,
+    ctlc: bool
+):
     '''
     Multiple subactors, both erroring and breakpointing as well as
     a nested subactor erroring.
@@ -412,50 +677,87 @@ def test_multi_subactors_root_errors(spawn):
     '''
     child = spawn('multi_subactor_root_errors')
 
-    # scan for the pdbpp prompt
+    # scan for the prompt
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
 
     # at most one subactor should attach before the root is cancelled
     before = str(child.before.decode())
     assert "NameError: name 'doggypants' is not defined" in before
 
+    if ctlc:
+        do_ctlc(child)
+
     # continue again to catch 2nd name error from
     # actor 'name_error_1' (which is 2nd depth).
     child.sendline('c')
-    child.expect(r"\(Pdb\+\+\)")
+
+    # due to block list strat from #337, this will no longer
+    # propagate before the root errors and cancels the spawner sub-tree.
+    child.expect(PROMPT)
+
+    # only if the blocking condition doesn't kick in fast enough
     before = str(child.before.decode())
-    assert "Attaching to pdb in crashed actor: ('name_error_1'" in before
+    if "Debug lock blocked for ['name_error_1'" not in before:
-    assert "NameError" in before
+
+        assert_before(child, [
+            "Attaching to pdb in crashed actor: ('name_error_1'",
+            "NameError",
+        ])
+
+        if ctlc:
+            do_ctlc(child)
 
         child.sendline('c')
-    child.expect(r"\(Pdb\+\+\)")
+        child.expect(PROMPT)
+
+    # check if the spawner crashed or was blocked from debug
+    # and if this intermediary attached check the boxed error
     before = str(child.before.decode())
-    assert "Attaching to pdb in crashed actor: ('spawn_error'" in before
+    if "Attaching to pdb in crashed actor: ('spawn_error'" in before:
-    # boxed error from previous step
+
-    assert "RemoteActorError: ('name_error_1'" in before
+        assert_before(child, [
-    assert "NameError" in before
+            # boxed error from spawner's child
+            "RemoteActorError: ('name_error_1'",
+            "NameError",
+        ])
+
+        if ctlc:
+            do_ctlc(child)
 
         child.sendline('c')
-    child.expect(r"\(Pdb\+\+\)")
+        child.expect(PROMPT)
-    before = str(child.before.decode())
-    assert "Attaching to pdb in crashed actor: ('root'" in before
-    # boxed error from first level failure
-    assert "RemoteActorError: ('name_error'" in before
-    assert "NameError" in before
 
-    # warnings assert we probably don't need
+    # expect a root actor crash
-    # assert "Cancelling nursery in ('spawn_error'," in before
+    assert_before(child, [
+        "RemoteActorError: ('name_error'",
+        "NameError",
+
+        # error from root actor and root task that created top level nursery
+        "Attaching to pdb in crashed actor: ('root'",
+        "AssertionError",
+    ])
 
-    # continue again
     child.sendline('c')
     child.expect(pexpect.EOF)
 
-    before = str(child.before.decode())
+    assert_before(child, [
-    # error from root actor and root task that created top level nursery
+        # "Attaching to pdb in crashed actor: ('root'",
-    assert "AssertionError" in before
+        # boxed error from previous step
+        "RemoteActorError: ('name_error'",
+        "NameError",
+        "AssertionError",
+        'assert 0',
+    ])
 
 
-def test_multi_nested_subactors_error_through_nurseries(spawn):
+@has_nested_actors
+def test_multi_nested_subactors_error_through_nurseries(
+    spawn,
+
+    # TODO: address debugger issue for nested tree:
+    # https://github.com/goodboy/tractor/issues/320
+    # ctlc: bool,
+):
     """Verify deeply nested actors that error trigger debugger entries
     at each actor nurserly (level) all the way up the tree.
 
@@ -470,55 +772,70 @@ def test_multi_nested_subactors_error_through_nurseries(spawn):
 
     timed_out_early: bool = False
 
-    for i in range(12):
+    for send_char in itertools.cycle(['c', 'q']):
         try:
-            child.expect(r"\(Pdb\+\+\)")
+            child.expect(PROMPT)
-            child.sendline('c')
+            child.sendline(send_char)
-            time.sleep(0.1)
+            time.sleep(0.01)
 
-        except pexpect.exceptions.EOF:
+        except EOF:
-
-            # race conditions on how fast the continue is sent?
-            print(f"Failed early on {i}?")
-            timed_out_early = True
             break
-    else:
-        child.expect(pexpect.EOF)
 
-    if not timed_out_early:
+    assert_before(child, [
-        before = str(child.before.decode())
+
-        assert "NameError" in before
+        # boxed source errors
+        "NameError: name 'doggypants' is not defined",
+        "tractor._exceptions.RemoteActorError: ('name_error'",
+        "bdb.BdbQuit",
+
+        # first level subtrees
+        "tractor._exceptions.RemoteActorError: ('spawner0'",
+        # "tractor._exceptions.RemoteActorError: ('spawner1'",
+
+        # propagation of errors up through nested subtrees
+        "tractor._exceptions.RemoteActorError: ('spawn_until_0'",
+        "tractor._exceptions.RemoteActorError: ('spawn_until_1'",
+        "tractor._exceptions.RemoteActorError: ('spawn_until_2'",
+    ])
 
 
+@pytest.mark.timeout(15)
+@has_nested_actors
 def test_root_nursery_cancels_before_child_releases_tty_lock(
     spawn,
-    start_method
+    start_method,
+    ctlc: bool,
 ):
-    """Test that when the root sends a cancel message before a nested
+    '''
-    child has unblocked (which can happen when it has the tty lock and
+    Test that when the root sends a cancel message before a nested child
-    is engaged in pdb) it is indeed cancelled after exiting the debugger.
+    has unblocked (which can happen when it has the tty lock and is
-    """
+    engaged in pdb) it is indeed cancelled after exiting the debugger.
+
+    '''
     timed_out_early = False
 
     child = spawn('root_cancelled_but_child_is_in_tty_lock')
 
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
 
     before = str(child.before.decode())
     assert "NameError: name 'doggypants' is not defined" in before
     assert "tractor._exceptions.RemoteActorError: ('name_error'" not in before
     time.sleep(0.5)
 
+    if ctlc:
+        do_ctlc(child)
+
     child.sendline('c')
 
     for i in range(4):
         time.sleep(0.5)
         try:
-            child.expect(r"\(Pdb\+\+\)")
+            child.expect(PROMPT)
 
         except (
-            pexpect.exceptions.EOF,
+            EOF,
-            pexpect.exceptions.TIMEOUT,
+            TIMEOUT,
         ):
             # races all over..
 
@@ -533,26 +850,37 @@ def test_root_nursery_cancels_before_child_releases_tty_lock(
         before = str(child.before.decode())
         assert "NameError: name 'doggypants' is not defined" in before
 
-        child.sendline('c')
+        if ctlc:
+            do_ctlc(child)
 
-    while True:
-        try:
-            child.expect(pexpect.EOF)
-            break
-        except pexpect.exceptions.TIMEOUT:
         child.sendline('c')
+        time.sleep(0.1)
+
+    for i in range(3):
+        try:
+            child.expect(pexpect.EOF, timeout=0.5)
+            break
+        except TIMEOUT:
+            child.sendline('c')
+            time.sleep(0.1)
             print('child was able to grab tty lock again?')
+    else:
+        print('giving up on child releasing, sending `quit` cmd')
+        child.sendline('q')
+        expect(child, EOF)
 
     if not timed_out_early:
-
         before = str(child.before.decode())
-        assert "tractor._exceptions.RemoteActorError: ('spawner0'" in before
+        assert_before(child, [
-        assert "tractor._exceptions.RemoteActorError: ('name_error'" in before
+            "tractor._exceptions.RemoteActorError: ('spawner0'",
-        assert "NameError: name 'doggypants' is not defined" in before
+            "tractor._exceptions.RemoteActorError: ('name_error'",
+            "NameError: name 'doggypants' is not defined",
+        ])
 
 
 def test_root_cancels_child_context_during_startup(
     spawn,
+    ctlc: bool,
 ):
     '''Verify a fast fail in the root doesn't lock up the child reaping
     and all while using the new context api.
@@ -560,26 +888,33 @@ def test_root_cancels_child_context_during_startup(
     '''
     child = spawn('fast_error_in_root_after_spawn')
 
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
 
     before = str(child.before.decode())
     assert "AssertionError" in before
 
+    if ctlc:
+        do_ctlc(child)
+
     child.sendline('c')
     child.expect(pexpect.EOF)
 
 
 def test_different_debug_mode_per_actor(
     spawn,
+    ctlc: bool,
 ):
     child = spawn('per_actor_debug')
-    child.expect(r"\(Pdb\+\+\)")
+    child.expect(PROMPT)
 
     # only one actor should enter the debugger
     before = str(child.before.decode())
     assert "Attaching to pdb in crashed actor: ('debugged_boi'" in before
     assert "RuntimeError" in before
 
+    if ctlc:
+        do_ctlc(child)
+
     child.sendline('c')
     child.expect(pexpect.EOF)
 

tests/test_docs_examples.py

@@ -12,17 +12,17 @@ import shutil
 
 import pytest
 
-from conftest import repodir
+from conftest import (
-
+    examples_dir,
-
+)
-def examples_dir():
-    """Return the abspath to the examples directory.
-    """
-    return os.path.join(repodir(), 'examples')
 
 
 @pytest.fixture
-def run_example_in_subproc(loglevel, testdir, arb_addr):
+def run_example_in_subproc(
+    loglevel: str,
+    testdir,
+    arb_addr: tuple[str, int],
+):
 
     @contextmanager
     def run(script_code):
@@ -32,8 +32,8 @@ def run_example_in_subproc(loglevel, testdir, arb_addr):
             # on windows we need to create a special __main__.py which will
             # be executed with ``python -m <modulename>`` on windows..
             shutil.copyfile(
-                os.path.join(examples_dir(), '__main__.py'),
+                examples_dir() / '__main__.py',
-                os.path.join(str(testdir), '__main__.py')
+                str(testdir / '__main__.py'),
             )
 
             # drop the ``if __name__ == '__main__'`` guard onwards from
@@ -81,11 +81,15 @@ def run_example_in_subproc(loglevel, testdir, arb_addr):
     'example_script',
 
     # walk yields: (dirpath, dirnames, filenames)
-    [(p[0], f) for p in os.walk(examples_dir()) for f in p[2]
+    [
+        (p[0], f) for p in os.walk(examples_dir()) for f in p[2]
 
         if '__' not in f
         and f[0] != '_'
-        and 'debugging' not in p[0]],
+        and 'debugging' not in p[0]
+        and 'integration' not in p[0]
+        and 'advanced_faults' not in p[0]
+    ],
 
     ids=lambda t: t[1],
 )
@@ -113,9 +117,19 @@ def test_example(run_example_in_subproc, example_script):
             # print(f'STDOUT: {out}')
 
             # if we get some gnarly output let's aggregate and raise
+            if err:
                 errmsg = err.decode()
                 errlines = errmsg.splitlines()
-            if err and 'Error' in errlines[-1]:
+                last_error = errlines[-1]
+                if (
+                    'Error' in last_error
+
+                    # XXX: currently we print this to console, but maybe
+                    # shouldn't eventually once we figure out what's
+                    # a better way to be explicit about aio side
+                    # cancels?
+                    and 'asyncio.exceptions.CancelledError' not in last_error
+                ):
                     raise Exception(errmsg)
 
             assert proc.returncode == 0

tests/test_infected_asyncio.py

@@ -2,20 +2,36 @@
 The hipster way to force SC onto the stdlib's "async": 'infection mode'.
 
 '''
-from typing import Optional, Iterable
+from typing import Optional, Iterable, Union
 import asyncio
 import builtins
+import itertools
 import importlib
 
+from exceptiongroup import BaseExceptionGroup
 import pytest
 import trio
 import tractor
-from tractor import to_asyncio
+from tractor import (
-from tractor import RemoteActorError
+    to_asyncio,
+    RemoteActorError,
+)
+from tractor.trionics import BroadcastReceiver
 
 
-async def sleep_and_err():
+async def sleep_and_err(
-    await asyncio.sleep(0.1)
+    sleep_for: float = 0.1,
+
+    # just signature placeholders for compat with
+    # ``to_asyncio.open_channel_from()``
+    to_trio: Optional[trio.MemorySendChannel] = None,
+    from_trio: Optional[asyncio.Queue] = None,
+
+):
+    if to_trio:
+        to_trio.send_nowait('start')
+
+    await asyncio.sleep(sleep_for)
     assert 0
 
 
@@ -144,6 +160,81 @@ def test_trio_cancels_aio(arb_addr):
     trio.run(main)
 
 
+@tractor.context
+async def trio_ctx(
+    ctx: tractor.Context,
+):
+
+    await ctx.started('start')
+
+    # this will block until the ``asyncio`` task sends a "first"
+    # message.
+    with trio.fail_after(2):
+        async with (
+            trio.open_nursery() as n,
+
+            tractor.to_asyncio.open_channel_from(
+                sleep_and_err,
+            ) as (first, chan),
+        ):
+
+            assert first == 'start'
+
+            # spawn another asyncio task for the cuck of it.
+            n.start_soon(
+                tractor.to_asyncio.run_task,
+                sleep_forever,
+            )
+            await trio.sleep_forever()
+
+
+@pytest.mark.parametrize(
+    'parent_cancels', [False, True],
+    ids='parent_actor_cancels_child={}'.format
+)
+def test_context_spawns_aio_task_that_errors(
+    arb_addr,
+    parent_cancels: bool,
+):
+    '''
+    Verify that spawning a task via an intertask channel ctx mngr that
+    errors correctly propagates the error back from the `asyncio`-side
+    task.
+
+    '''
+    async def main():
+
+        with trio.fail_after(2):
+            async with tractor.open_nursery() as n:
+                p = await n.start_actor(
+                    'aio_daemon',
+                    enable_modules=[__name__],
+                    infect_asyncio=True,
+                    # debug_mode=True,
+                    loglevel='cancel',
+                )
+                async with p.open_context(
+                    trio_ctx,
+                ) as (ctx, first):
+
+                    assert first == 'start'
+
+                    if parent_cancels:
+                        await p.cancel_actor()
+
+                    await trio.sleep_forever()
+
+    with pytest.raises(RemoteActorError) as excinfo:
+        trio.run(main)
+
+    err = excinfo.value
+    assert isinstance(err, RemoteActorError)
+    if parent_cancels:
+        assert err.type == trio.Cancelled
+    else:
+        assert err.type == AssertionError
+
+
 async def aio_cancel():
     ''''
     Cancel urself boi.
@@ -217,6 +308,7 @@ async def stream_from_aio(
     exit_early: bool = False,
     raise_err: bool = False,
     aio_raise_err: bool = False,
+    fan_out: bool = False,
 
 ) -> None:
     seq = range(100)
@@ -234,6 +326,12 @@ async def stream_from_aio(
 
             assert first is True
 
+            async def consume(
+                chan: Union[
+                    to_asyncio.LinkedTaskChannel,
+                    BroadcastReceiver,
+                ],
+            ):
                 async for value in chan:
                     print(f'trio received {value}')
                     pulled.append(value)
@@ -243,6 +341,23 @@ async def stream_from_aio(
                             raise Exception
                         elif exit_early:
                             break
+
+            if fan_out:
+                # start second task that get's the same stream value set.
+                async with (
+
+                    # NOTE: this has to come first to avoid
+                    # the channel being closed before the nursery
+                    # tasks are joined..
+                    chan.subscribe() as br,
+
+                    trio.open_nursery() as n,
+                ):
+                    n.start_soon(consume, br)
+                    await consume(chan)
+
+            else:
+                await consume(chan)
     finally:
 
         if (
@@ -250,19 +365,33 @@ async def stream_from_aio(
             not exit_early and
             not aio_raise_err
         ):
+            if fan_out:
+                # we get double the pulled values in the
+                # ``.subscribe()`` fan out case.
+                doubled = list(itertools.chain(*zip(expect, expect)))
+                expect = doubled[:len(pulled)]
+                assert list(sorted(pulled)) == expect
+
+            else:
                 assert pulled == expect
         else:
+            assert not fan_out
             assert pulled == expect[:51]
 
         print('trio guest mode task completed!')
 
 
-def test_basic_interloop_channel_stream(arb_addr):
+@pytest.mark.parametrize(
+    'fan_out', [False, True],
+    ids='fan_out_w_chan_subscribe={}'.format
+)
+def test_basic_interloop_channel_stream(arb_addr, fan_out):
     async def main():
         async with tractor.open_nursery() as n:
             portal = await n.run_in_actor(
                 stream_from_aio,
                 infect_asyncio=True,
+                fan_out=fan_out,
             )
             await portal.result()
 
@@ -281,11 +410,12 @@ def test_trio_error_cancels_intertask_chan(arb_addr):
             # should trigger remote actor error
             await portal.result()
 
-    with pytest.raises(RemoteActorError) as excinfo:
+    with pytest.raises(BaseExceptionGroup) as excinfo:
         trio.run(main)
 
-    # ensure boxed error is correct
+    # ensure boxed errors
-    assert excinfo.value.type == Exception
+    for exc in excinfo.value.exceptions:
+        assert exc.type == Exception
 
 
 def test_trio_closes_early_and_channel_exits(arb_addr):
@@ -314,11 +444,12 @@ def test_aio_errors_and_channel_propagates_and_closes(arb_addr):
             # should trigger remote actor error
             await portal.result()
 
-    with pytest.raises(RemoteActorError) as excinfo:
+    with pytest.raises(BaseExceptionGroup) as excinfo:
         trio.run(main)
 
-    # ensure boxed error is correct
+    # ensure boxed errors
-    assert excinfo.value.type == Exception
+    for exc in excinfo.value.exceptions:
+        assert exc.type == Exception
 
 
 @tractor.context
@@ -345,6 +476,8 @@ async def trio_to_aio_echo_server(
                 print('breaking aio echo loop')
                 break
 
+        print('exiting asyncio task')
+
     async with to_asyncio.open_channel_from(
         aio_echo_server,
     ) as (first, chan):

tests/test_legacy_one_way_streaming.py

@@ -7,9 +7,10 @@ import platform
 
 import trio
 import tractor
-from tractor.testing import tractor_test
 import pytest
 
+from conftest import tractor_test
+
 
 def test_must_define_ctx():
 
@@ -250,7 +251,7 @@ def test_a_quadruple_example(time_quad_ex, ci_env, spawn_backend):
 
     results, diff = time_quad_ex
     assert results
-    this_fast = 6 if platform.system() in ('Windows', 'Darwin') else 2.666
+    this_fast = 6 if platform.system() in ('Windows', 'Darwin') else 3
     assert diff < this_fast
 
 

tests/test_local.py

@@ -11,7 +11,7 @@ from conftest import tractor_test
 
 
 @pytest.mark.trio
-async def test_no_arbitter():
+async def test_no_runtime():
     """An arbitter must be established before any nurseries
     can be created.
 
@@ -19,17 +19,10 @@ async def test_no_arbitter():
     some point?)
     """
     with pytest.raises(RuntimeError) :
-        with tractor.open_nursery():
+        async with tractor.find_actor('doggy'):
             pass
 
 
-def test_no_main():
-    """An async function **must** be passed to ``tractor.run()``.
-    """
-    with pytest.raises(TypeError):
-        tractor.run(None)
-
-
 @tractor_test
 async def test_self_is_registered(arb_addr):
     "Verify waiting on the arbiter to register itself using the standard api."

tests/test_pubsub.py

@@ -4,9 +4,10 @@ from itertools import cycle
 import pytest
 import trio
 import tractor
-from tractor.testing import tractor_test
 from tractor.experimental import msgpub
 
+from conftest import tractor_test
+
 
 def test_type_checks():
 

tests/test_runtime.py

@@ -0,0 +1,73 @@
+"""
+Verifying internal runtime state and undocumented extras.
+
+"""
+import os
+
+import pytest
+import trio
+import tractor
+
+from conftest import tractor_test
+
+
+_file_path: str = ''
+
+
+def unlink_file():
+    print('Removing tmp file!')
+    os.remove(_file_path)
+
+
+async def crash_and_clean_tmpdir(
+    tmp_file_path: str,
+    error: bool = True,
+):
+    global _file_path
+    _file_path = tmp_file_path
+
+    actor = tractor.current_actor()
+    actor.lifetime_stack.callback(unlink_file)
+
+    assert os.path.isfile(tmp_file_path)
+    await trio.sleep(0.1)
+    if error:
+        assert 0
+    else:
+        actor.cancel_soon()
+
+
+@pytest.mark.parametrize(
+    'error_in_child',
+    [True, False],
+)
+@tractor_test
+async def test_lifetime_stack_wipes_tmpfile(
+    tmp_path,
+    error_in_child: bool,
+):
+    child_tmp_file = tmp_path / "child.txt"
+    child_tmp_file.touch()
+    assert child_tmp_file.exists()
+    path = str(child_tmp_file)
+
+    try:
+        with trio.move_on_after(0.5):
+            async with tractor.open_nursery() as n:
+                    await (  # inlined portal
+                        await n.run_in_actor(
+                            crash_and_clean_tmpdir,
+                            tmp_file_path=path,
+                            error=error_in_child,
+                        )
+                    ).result()
+
+    except (
+        tractor.RemoteActorError,
+        tractor.BaseExceptionGroup,
+    ):
+        pass
+
+    # tmp file should have been wiped by
+    # teardown stack.
+    assert not child_tmp_file.exists()

tests/test_spawning.py

@@ -1,7 +1,8 @@
 """
 Spawning basics
+
 """
-from typing import Dict, Tuple, Optional
+from typing import Optional
 
 import pytest
 import trio
@@ -14,8 +15,8 @@ data_to_pass_down = {'doggy': 10, 'kitty': 4}
 
 async def spawn(
     is_arbiter: bool,
-    data: Dict,
+    data: dict,
-    arb_addr: Tuple[str, int],
+    arb_addr: tuple[str, int],
 ):
     namespaces = [__name__]
 
@@ -141,7 +142,7 @@ def test_loglevel_propagated_to_subactor(
     capfd,
     arb_addr,
 ):
-    if start_method == 'forkserver':
+    if start_method == 'mp_forkserver':
         pytest.skip(
             "a bug with `capfd` seems to make forkserver capture not work?")
 
@@ -150,13 +151,13 @@ def test_loglevel_propagated_to_subactor(
     async def main():
         async with tractor.open_nursery(
             name='arbiter',
-            loglevel=level,
             start_method=start_method,
             arbiter_addr=arb_addr,
 
         ) as tn:
             await tn.run_in_actor(
                 check_loglevel,
+                loglevel=level,
                 level=level,
             )
 

tests/test_task_broadcasting.py

@@ -6,13 +6,16 @@ from contextlib import asynccontextmanager
 from functools import partial
 from itertools import cycle
 import time
-from typing import Optional, List, Tuple
+from typing import Optional
 
 import pytest
 import trio
 from trio.lowlevel import current_task
 import tractor
-from tractor.trionics import broadcast_receiver, Lagged
+from tractor.trionics import (
+    broadcast_receiver,
+    Lagged,
+)
 
 
 @tractor.context
@@ -37,7 +40,7 @@ async def echo_sequences(
 
 async def ensure_sequence(
 
-    stream: tractor.ReceiveMsgStream,
+    stream: tractor.MsgStream,
     sequence: list,
     delay: Optional[float] = None,
 
@@ -62,8 +65,8 @@ async def ensure_sequence(
 @asynccontextmanager
 async def open_sequence_streamer(
 
-    sequence: List[int],
+    sequence: list[int],
-    arb_addr: Tuple[str, int],
+    arb_addr: tuple[str, int],
     start_method: str,
 
 ) -> tractor.MsgStream:
@@ -211,7 +214,8 @@ def test_faster_task_to_recv_is_cancelled_by_slower(
     arb_addr,
     start_method,
 ):
-    '''Ensure that if a faster task consuming from a stream is cancelled
+    '''
+    Ensure that if a faster task consuming from a stream is cancelled
     the slower task can continue to receive all expected values.
 
     '''
@@ -460,3 +464,51 @@ def test_first_recver_is_cancelled():
                     assert value == 1
 
     trio.run(main)
+
+
+def test_no_raise_on_lag():
+    '''
+    Run a simple 2-task broadcast where one task is slow but configured
+    so that it does not raise `Lagged` on overruns using
+    `raise_on_lasg=False` and verify that the task does not raise.
+
+    '''
+    size = 100
+    tx, rx = trio.open_memory_channel(size)
+    brx = broadcast_receiver(rx, size)
+
+    async def slow():
+        async with brx.subscribe(
+            raise_on_lag=False,
+        ) as br:
+            async for msg in br:
+                print(f'slow task got: {msg}')
+                await trio.sleep(0.1)
+
+    async def fast():
+        async with brx.subscribe() as br:
+            async for msg in br:
+                print(f'fast task got: {msg}')
+
+    async def main():
+        async with (
+            tractor.open_root_actor(
+                # NOTE: so we see the warning msg emitted by the bcaster
+                # internals when the no raise flag is set.
+                loglevel='warning',
+            ),
+            trio.open_nursery() as n,
+        ):
+            n.start_soon(slow)
+            n.start_soon(fast)
+
+            for i in range(1000):
+                await tx.send(i)
+
+            # simulate user nailing ctl-c after realizing
+            # there's a lag in the slow task.
+            await trio.sleep(1)
+            raise KeyboardInterrupt
+
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(main)

tests/test_trioisms.py

@@ -0,0 +1,82 @@
+'''
+Reminders for oddities in `trio` that we need to stay aware of and/or
+want to see changed.
+
+'''
+import pytest
+import trio
+from trio_typing import TaskStatus
+
+
+@pytest.mark.parametrize(
+    'use_start_soon', [
+        pytest.param(
+            True,
+            marks=pytest.mark.xfail(reason="see python-trio/trio#2258")
+        ),
+        False,
+    ]
+)
+def test_stashed_child_nursery(use_start_soon):
+
+    _child_nursery = None
+
+    async def waits_on_signal(
+        ev: trio.Event(),
+        task_status: TaskStatus[trio.Nursery] = trio.TASK_STATUS_IGNORED,
+    ):
+        '''
+        Do some stuf, then signal other tasks, then yield back to "starter".
+
+        '''
+        await ev.wait()
+        task_status.started()
+
+    async def mk_child_nursery(
+        task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
+    ):
+        '''
+        Allocate a child sub-nursery and stash it as a global.
+
+        '''
+        nonlocal _child_nursery
+
+        async with trio.open_nursery() as cn:
+            _child_nursery = cn
+            task_status.started(cn)
+
+            # block until cancelled by parent.
+            await trio.sleep_forever()
+
+    async def sleep_and_err(
+        ev: trio.Event,
+        task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
+    ):
+        await trio.sleep(0.5)
+        doggy()  # noqa
+        ev.set()
+        task_status.started()
+
+    async def main():
+
+        async with (
+            trio.open_nursery() as pn,
+        ):
+            cn = await pn.start(mk_child_nursery)
+            assert cn
+
+            ev = trio.Event()
+
+            if use_start_soon:
+                # this causes inf hang
+                cn.start_soon(sleep_and_err, ev)
+
+            else:
+                # this does not.
+                await cn.start(sleep_and_err, ev)
+
+            with trio.fail_after(1):
+                await cn.start(waits_on_signal, ev)
+
+    with pytest.raises(NameError):
+        trio.run(main)

towncrier.toml

@@ -1,7 +0,0 @@
-[tool.towncrier]
-package = "tractor"
-filename = "NEWS.rst"
-directory = "nooz/"
-title_format = "tractor {version} ({project_date})"
-version = "0.1.0a4"
-template = "nooz/_template.rst"

tractor/__init__.py

@@ -18,39 +18,57 @@
 tractor: structured concurrent "actors".
 
 """
-from trio import MultiError
+from exceptiongroup import BaseExceptionGroup
 
 from ._clustering import open_actor_cluster
 from ._ipc import Channel
 from ._streaming import (
     Context,
-    ReceiveMsgStream,
     MsgStream,
     stream,
     context,
 )
-from ._discovery import get_arbiter, find_actor, wait_for_actor
+from ._discovery import (
+    get_arbiter,
+    find_actor,
+    wait_for_actor,
+    query_actor,
+)
 from ._supervise import open_nursery
-from ._state import current_actor, is_root_process
+from ._state import (
+    current_actor,
+    is_root_process,
+)
 from ._exceptions import (
     RemoteActorError,
     ModuleNotExposed,
     ContextCancelled,
 )
-from ._debug import breakpoint, post_mortem
+from ._debug import (
+    breakpoint,
+    post_mortem,
+)
 from . import msg
-from ._root import run, run_daemon, open_root_actor
+from ._root import (
+    run_daemon,
+    open_root_actor,
+)
 from ._portal import Portal
+from ._runtime import Actor
 
 
 __all__ = [
+    'Actor',
     'Channel',
     'Context',
-    'ModuleNotExposed',
-    'MultiError',
-    'RemoteActorError',
     'ContextCancelled',
+    'ModuleNotExposed',
+    'MsgStream',
+    'BaseExceptionGroup',
+    'Portal',
+    'RemoteActorError',
     'breakpoint',
+    'context',
     'current_actor',
     'find_actor',
     'get_arbiter',
@@ -59,14 +77,10 @@ __all__ = [
     'open_actor_cluster',
     'open_nursery',
     'open_root_actor',
-    'Portal',
     'post_mortem',
-    'run',
+    'query_actor',
     'run_daemon',
     'stream',
-    'context',
-    'ReceiveMsgStream',
-    'MsgStream',
     'to_asyncio',
     'wait_for_actor',
 ]

tractor/_child.py

@@ -24,7 +24,7 @@ import argparse
 
 from ast import literal_eval
 
-from ._actor import Actor
+from ._runtime import Actor
 from ._entry import _trio_main
 
 

tractor/_clustering.py

@@ -32,9 +32,12 @@ import tractor
 async def open_actor_cluster(
     modules: list[str],
     count: int = cpu_count(),
-    names: Optional[list[str]] = None,
+    names: list[str] | None = None,
-    start_method: Optional[str] = None,
     hard_kill: bool = False,
+
+    # passed through verbatim to ``open_root_actor()``
+    **runtime_kwargs,
+
 ) -> AsyncGenerator[
     dict[str, tractor.Portal],
     None,
@@ -49,7 +52,9 @@ async def open_actor_cluster(
         raise ValueError(
             'Number of names is {len(names)} but count it {count}')
 
-    async with tractor.open_nursery(start_method=start_method) as an:
+    async with tractor.open_nursery(
+        **runtime_kwargs,
+    ) as an:
         async with trio.open_nursery() as n:
             uid = tractor.current_actor().uid
 

tractor/_discovery.py

@@ -18,9 +18,12 @@
 Actor discovery API.
 
 """
-import typing
+from typing import (
-from typing import Tuple, Optional, Union
+    Optional,
-from async_generator import asynccontextmanager
+    Union,
+    AsyncGenerator,
+)
+from contextlib import asynccontextmanager as acm
 
 from ._ipc import _connect_chan, Channel
 from ._portal import (
@@ -31,13 +34,13 @@ from ._portal import (
 from ._state import current_actor, _runtime_vars
 
 
-@asynccontextmanager
+@acm
 async def get_arbiter(
 
     host: str,
     port: int,
 
-) -> typing.AsyncGenerator[Union[Portal, LocalPortal], None]:
+) -> AsyncGenerator[Union[Portal, LocalPortal], None]:
     '''Return a portal instance connected to a local or remote
     arbiter.
     '''
@@ -58,10 +61,10 @@ async def get_arbiter(
                 yield arb_portal
 
 
-@asynccontextmanager
+@acm
 async def get_root(
     **kwargs,
-) -> typing.AsyncGenerator[Portal, None]:
+) -> AsyncGenerator[Portal, None]:
 
     host, port = _runtime_vars['_root_mailbox']
     assert host is not None
@@ -71,28 +74,56 @@ async def get_root(
             yield portal
 
 
-@asynccontextmanager
+@acm
-async def find_actor(
+async def query_actor(
     name: str,
-    arbiter_sockaddr: Tuple[str, int] = None
+    arbiter_sockaddr: Optional[tuple[str, int]] = None,
-) -> typing.AsyncGenerator[Optional[Portal], None]:
-    """Ask the arbiter to find actor(s) by name.
 
-    Returns a connected portal to the last registered matching actor
+) -> AsyncGenerator[tuple[str, int], None]:
-    known to the arbiter.
+    '''
-    """
+    Simple address lookup for a given actor name.
+
+    Returns the (socket) address or ``None``.
+
+    '''
     actor = current_actor()
-    async with get_arbiter(*arbiter_sockaddr or actor._arb_addr) as arb_portal:
+    async with get_arbiter(
+        *arbiter_sockaddr or actor._arb_addr
+    ) as arb_portal:
 
-        sockaddr = await arb_portal.run_from_ns('self', 'find_actor', name=name)
+        sockaddr = await arb_portal.run_from_ns(
+            'self',
+            'find_actor',
+            name=name,
+        )
 
         # TODO: return portals to all available actors - for now just
         # the last one that registered
         if name == 'arbiter' and actor.is_arbiter:
             raise RuntimeError("The current actor is the arbiter")
 
-        elif sockaddr:
+        yield sockaddr if sockaddr else None
 
+
+@acm
+async def find_actor(
+    name: str,
+    arbiter_sockaddr: tuple[str, int] | None = None
+
+) -> AsyncGenerator[Optional[Portal], None]:
+    '''
+    Ask the arbiter to find actor(s) by name.
+
+    Returns a connected portal to the last registered matching actor
+    known to the arbiter.
+
+    '''
+    async with query_actor(
+        name=name,
+        arbiter_sockaddr=arbiter_sockaddr,
+    ) as sockaddr:
+
+        if sockaddr:
             async with _connect_chan(*sockaddr) as chan:
                 async with open_portal(chan) as portal:
                     yield portal
@@ -100,20 +131,25 @@ async def find_actor(
             yield None
 
 
-@asynccontextmanager
+@acm
 async def wait_for_actor(
     name: str,
-    arbiter_sockaddr: Tuple[str, int] = None
+    arbiter_sockaddr: tuple[str, int] | None = None
-) -> typing.AsyncGenerator[Portal, None]:
+) -> AsyncGenerator[Portal, None]:
     """Wait on an actor to register with the arbiter.
 
     A portal to the first registered actor is returned.
     """
     actor = current_actor()
 
-    async with get_arbiter(*arbiter_sockaddr or actor._arb_addr) as arb_portal:
+    async with get_arbiter(
-
+        *arbiter_sockaddr or actor._arb_addr,
-        sockaddrs = await arb_portal.run_from_ns('self', 'wait_for_actor', name=name)
+    ) as arb_portal:
+        sockaddrs = await arb_portal.run_from_ns(
+            'self',
+            'wait_for_actor',
+            name=name,
+        )
         sockaddr = sockaddrs[-1]
 
         async with _connect_chan(*sockaddr) as chan:

tractor/_entry.py

@@ -18,15 +18,28 @@
 Sub-process entry points.
 
 """
+from __future__ import annotations
 from functools import partial
-from typing import Tuple, Any
+from typing import (
-import signal
+    Any,
+    TYPE_CHECKING,
+)
 
 import trio  # type: ignore
 
-from .log import get_console_log, get_logger
+from .log import (
+    get_console_log,
+    get_logger,
+)
 from . import _state
 from .to_asyncio import run_as_asyncio_guest
+from ._runtime import (
+    async_main,
+    Actor,
+)
+
+if TYPE_CHECKING:
+    from ._spawn import SpawnMethodKey
 
 
 log = get_logger(__name__)
@@ -34,11 +47,11 @@ log = get_logger(__name__)
 
 def _mp_main(
 
-    actor: 'Actor',  # type: ignore
+    actor: Actor,  # type: ignore
-    accept_addr: Tuple[str, int],
+    accept_addr: tuple[str, int],
-    forkserver_info: Tuple[Any, Any, Any, Any, Any],
+    forkserver_info: tuple[Any, Any, Any, Any, Any],
-    start_method: str,
+    start_method: SpawnMethodKey,
-    parent_addr: Tuple[str, int] = None,
+    parent_addr: tuple[str, int] | None = None,
     infect_asyncio: bool = False,
 
 ) -> None:
@@ -63,7 +76,8 @@ def _mp_main(
 
     log.debug(f"parent_addr is {parent_addr}")
     trio_main = partial(
-        actor._async_main,
+        async_main,
+        actor,
         accept_addr,
         parent_addr=parent_addr
     )
@@ -82,9 +96,9 @@ def _mp_main(
 
 def _trio_main(
 
-    actor: 'Actor',  # type: ignore
+    actor: Actor,  # type: ignore
     *,
-    parent_addr: Tuple[str, int] = None,
+    parent_addr: tuple[str, int] | None = None,
     infect_asyncio: bool = False,
 
 ) -> None:
@@ -106,7 +120,8 @@ def _trio_main(
 
     log.debug(f"parent_addr is {parent_addr}")
     trio_main = partial(
-        actor._async_main,
+        async_main,
+        actor,
         parent_addr=parent_addr
     )
 

tractor/_exceptions.py

@@ -18,11 +18,16 @@
 Our classy exception set.
 
 """
-from typing import Dict, Any, Optional, Type
+from typing import (
+    Any,
+    Optional,
+    Type,
+)
 import importlib
 import builtins
 import traceback
 
+import exceptiongroup as eg
 import trio
 
 
@@ -48,9 +53,6 @@ class RemoteActorError(Exception):
         self.type = suberror_type
         self.msgdata = msgdata
 
-    # TODO: a trio.MultiError.catch like context manager
-    # for catching underlying remote errors of a particular type
-
 
 class InternalActorError(RemoteActorError):
     """Remote internal ``tractor`` error indicating
@@ -95,7 +97,7 @@ def pack_error(
     exc: BaseException,
     tb=None,
 
-) -> Dict[str, Any]:
+) -> dict[str, Any]:
     """Create an "error message" for tranmission over
     a channel (aka the wire).
     """
@@ -114,15 +116,17 @@ def pack_error(
 
 def unpack_error(
 
-    msg: Dict[str, Any],
+    msg: dict[str, Any],
     chan=None,
     err_type=RemoteActorError
 
 ) -> Exception:
-    """Unpack an 'error' message from the wire
+    '''
+    Unpack an 'error' message from the wire
     into a local ``RemoteActorError``.
 
-    """
+    '''
+    __tracebackhide__ = True
     error = msg['error']
 
     tb_str = error.get('tb_str', '')
@@ -135,7 +139,12 @@ def unpack_error(
         suberror_type = trio.Cancelled
 
     else:  # try to lookup a suitable local error type
-        for ns in [builtins, _this_mod, trio]:
+        for ns in [
+            builtins,
+            _this_mod,
+            eg,
+            trio,
+        ]:
             try:
                 suberror_type = getattr(ns, type_name)
                 break
@@ -154,12 +163,15 @@ def unpack_error(
 
 
 def is_multi_cancelled(exc: BaseException) -> bool:
-    """Predicate to determine if a ``trio.MultiError`` contains only
+    '''
-    ``trio.Cancelled`` sub-exceptions (and is likely the result of
+    Predicate to determine if a possible ``eg.BaseExceptionGroup`` contains
+    only ``trio.Cancelled`` sub-exceptions (and is likely the result of
     cancelling a collection of subtasks.
 
-    """
+    '''
-    return not trio.MultiError.filter(
+    if isinstance(exc, eg.BaseExceptionGroup):
-        lambda exc: exc if not isinstance(exc, trio.Cancelled) else None,
+        return exc.subgroup(
-        exc,
+            lambda exc: isinstance(exc, trio.Cancelled)
-    )
+        ) is not None
+
+    return False

tractor/_ipc.py

@@ -22,14 +22,21 @@ from __future__ import annotations
 import platform
 import struct
 import typing
-from collections.abc import AsyncGenerator, AsyncIterator
+from collections.abc import (
+    AsyncGenerator,
+    AsyncIterator,
+)
 from typing import (
-    Any, Tuple, Optional,
+    Any,
-    Type, Protocol, TypeVar,
+    runtime_checkable,
+    Optional,
+    Protocol,
+    Type,
+    TypeVar,
 )
 
 from tricycle import BufferedReceiveStream
-import msgpack
+import msgspec
 import trio
 from async_generator import asynccontextmanager
 
@@ -42,7 +49,7 @@ _is_windows = platform.system() == 'Windows'
 log = get_logger(__name__)
 
 
-def get_stream_addrs(stream: trio.SocketStream) -> Tuple:
+def get_stream_addrs(stream: trio.SocketStream) -> tuple:
     # should both be IP sockets
     lsockname = stream.socket.getsockname()
     rsockname = stream.socket.getpeername()
@@ -60,6 +67,7 @@ MsgType = TypeVar("MsgType")
 # - https://jcristharif.com/msgspec/usage.html#structs
 
 
+@runtime_checkable
 class MsgTransport(Protocol[MsgType]):
 
     stream: trio.SocketStream
@@ -87,23 +95,27 @@ class MsgTransport(Protocol[MsgType]):
         ...
 
     @property
-    def laddr(self) -> Tuple[str, int]:
+    def laddr(self) -> tuple[str, int]:
         ...
 
     @property
-    def raddr(self) -> Tuple[str, int]:
+    def raddr(self) -> tuple[str, int]:
         ...
 
 
-class MsgpackTCPStream:
+# TODO: not sure why we have to inherit here, but it seems to be an
+# issue with ``get_msg_transport()`` returning a ``Type[Protocol]``;
+# probably should make a `mypy` issue?
+class MsgpackTCPStream(MsgTransport):
     '''
     A ``trio.SocketStream`` delivering ``msgpack`` formatted data
-    using ``msgpack-python``.
+    using the ``msgspec`` codec lib.
 
     '''
     def __init__(
         self,
         stream: trio.SocketStream,
+        prefix_size: int = 4,
 
     ) -> None:
 
@@ -120,105 +132,6 @@ class MsgpackTCPStream:
         # public i guess?
         self.drained: list[dict] = []
 
-    async def _iter_packets(self) -> AsyncGenerator[dict, None]:
-        '''
-        Yield packets from the underlying stream.
-
-        '''
-        unpacker = msgpack.Unpacker(
-            raw=False,
-        )
-        while True:
-            try:
-                data = await self.stream.receive_some(2**10)
-
-            except trio.BrokenResourceError as err:
-                msg = err.args[0]
-
-                # XXX: handle connection-reset-by-peer the same as a EOF.
-                # we're currently remapping this since we allow
-                # a quick connect then drop for root actors when
-                # checking to see if there exists an "arbiter"
-                # on the chosen sockaddr (``_root.py:108`` or thereabouts)
-                if (
-                    # nix
-                    '[Errno 104]' in msg or
-
-                    # on windows it seems there are a variety of errors
-                    # to handle..
-                    _is_windows
-                ):
-                    raise TransportClosed(
-                        f'{self} was broken with {msg}'
-                    )
-
-                else:
-                    raise
-
-            log.transport(f"received {data}")  # type: ignore
-
-            if data == b'':
-                raise TransportClosed(
-                    f'transport {self} was already closed prior to read'
-                )
-
-            unpacker.feed(data)
-            for packet in unpacker:
-                yield packet
-
-    @property
-    def laddr(self) -> Tuple[Any, ...]:
-        return self._laddr
-
-    @property
-    def raddr(self) -> Tuple[Any, ...]:
-        return self._raddr
-
-    async def send(self, msg: Any) -> None:
-        async with self._send_lock:
-            return await self.stream.send_all(
-                msgpack.dumps(msg, use_bin_type=True)
-            )
-
-    async def recv(self) -> Any:
-        return await self._agen.asend(None)
-
-    async def drain(self) -> AsyncIterator[dict]:
-        '''
-        Drain the stream's remaining messages sent from
-        the far end until the connection is closed by
-        the peer.
-
-        '''
-        try:
-            async for msg in self._iter_packets():
-                self.drained.append(msg)
-        except TransportClosed:
-            for msg in self.drained:
-                yield msg
-
-    def __aiter__(self):
-        return self._agen
-
-    def connected(self) -> bool:
-        return self.stream.socket.fileno() != -1
-
-
-class MsgspecTCPStream(MsgpackTCPStream):
-    '''
-    A ``trio.SocketStream`` delivering ``msgpack`` formatted data
-    using ``msgspec``.
-
-    '''
-    def __init__(
-        self,
-        stream: trio.SocketStream,
-        prefix_size: int = 4,
-
-    ) -> None:
-        import msgspec
-
-        super().__init__(stream)
         self.recv_stream = BufferedReceiveStream(transport_stream=stream)
         self.prefix_size = prefix_size
 
@@ -231,7 +144,7 @@ class MsgspecTCPStream(MsgpackTCPStream):
 
         '''
         import msgspec  # noqa
-        last_decode_failed: bool = False
+        decodes_failed: int = 0
 
         while True:
             try:
@@ -239,6 +152,7 @@ class MsgspecTCPStream(MsgpackTCPStream):
 
             except (
                 ValueError,
+                ConnectionResetError,
 
                 # not sure entirely why we need this but without it we
                 # seem to be getting racy failures here on
@@ -264,15 +178,24 @@ class MsgspecTCPStream(MsgpackTCPStream):
             try:
                 yield self.decode(msg_bytes)
             except (
-                msgspec.DecodingError,
+                msgspec.DecodeError,
                 UnicodeDecodeError,
             ):
-                if not last_decode_failed:
+                if decodes_failed < 4:
                     # ignore decoding errors for now and assume they have to
                     # do with a channel drop - hope that receiving from the
                     # channel will raise an expected error and bubble up.
-                    log.error('`msgspec` failed to decode!?')
+                    try:
-                    last_decode_failed = True
+                        msg_str: str | bytes = msg_bytes.decode()
+                    except UnicodeDecodeError:
+                        msg_str = msg_bytes
+
+                    log.error(
+                        '`msgspec` failed to decode!?\n'
+                        'dumping bytes:\n'
+                        f'{msg_str!r}'
+                    )
+                    decodes_failed += 1
                 else:
                     raise
 
@@ -287,16 +210,46 @@ class MsgspecTCPStream(MsgpackTCPStream):
 
             return await self.stream.send_all(size + bytes_data)
 
+    @property
+    def laddr(self) -> tuple[str, int]:
+        return self._laddr
+
+    @property
+    def raddr(self) -> tuple[str, int]:
+        return self._raddr
+
+    async def recv(self) -> Any:
+        return await self._agen.asend(None)
+
+    async def drain(self) -> AsyncIterator[dict]:
+        '''
+        Drain the stream's remaining messages sent from
+        the far end until the connection is closed by
+        the peer.
+
+        '''
+        try:
+            async for msg in self._iter_packets():
+                self.drained.append(msg)
+        except TransportClosed:
+            for msg in self.drained:
+                yield msg
+
+    def __aiter__(self):
+        return self._agen
+
+    def connected(self) -> bool:
+        return self.stream.socket.fileno() != -1
+
 
 def get_msg_transport(
 
-    key: Tuple[str, str],
+    key: tuple[str, str],
 
 ) -> Type[MsgTransport]:
 
     return {
         ('msgpack', 'tcp'): MsgpackTCPStream,
-        ('msgspec', 'tcp'): MsgspecTCPStream,
     }[key]
 
 
@@ -305,16 +258,18 @@ class Channel:
     An inter-process channel for communication between (remote) actors.
 
     Wraps a ``MsgStream``: transport + encoding IPC connection.
+
     Currently we only support ``trio.SocketStream`` for transport
-    (aka TCP).
+    (aka TCP) and the ``msgpack`` interchange format via the ``msgspec``
+    codec libary.
 
     '''
     def __init__(
 
         self,
-        destaddr: Optional[Tuple[str, int]],
+        destaddr: Optional[tuple[str, int]],
 
-        msg_transport_type_key: Tuple[str, str] = ('msgpack', 'tcp'),
+        msg_transport_type_key: tuple[str, str] = ('msgpack', 'tcp'),
 
         # TODO: optional reconnection support?
         # auto_reconnect: bool = False,
@@ -325,14 +280,6 @@ class Channel:
         # self._recon_seq = on_reconnect
         # self._autorecon = auto_reconnect
 
-        # TODO: maybe expose this through the nursery api?
-        try:
-            # if installed load the msgspec transport since it's faster
-            import msgspec  # noqa
-            msg_transport_type_key = ('msgspec', 'tcp')
-        except ImportError:
-            pass
-
         self._destaddr = destaddr
         self._transport_key = msg_transport_type_key
 
@@ -342,7 +289,7 @@ class Channel:
         self.msgstream: Optional[MsgTransport] = None
 
         # set after handshake - always uid of far end
-        self.uid: Optional[Tuple[str, str]] = None
+        self.uid: Optional[tuple[str, str]] = None
 
         self._agen = self._aiter_recv()
         self._exc: Optional[Exception] = None  # set if far end actor errors
@@ -370,7 +317,7 @@ class Channel:
     def set_msg_transport(
         self,
         stream: trio.SocketStream,
-        type_key: Optional[Tuple[str, str]] = None,
+        type_key: Optional[tuple[str, str]] = None,
 
     ) -> MsgTransport:
         type_key = type_key or self._transport_key
@@ -385,16 +332,16 @@ class Channel:
         return object.__repr__(self)
 
     @property
-    def laddr(self) -> Optional[Tuple[str, int]]:
+    def laddr(self) -> Optional[tuple[str, int]]:
         return self.msgstream.laddr if self.msgstream else None
 
     @property
-    def raddr(self) -> Optional[Tuple[str, int]]:
+    def raddr(self) -> Optional[tuple[str, int]]:
         return self.msgstream.raddr if self.msgstream else None
 
     async def connect(
         self,
-        destaddr: Tuple[Any, ...] = None,
+        destaddr: tuple[Any, ...] | None = None,
         **kwargs
 
     ) -> MsgTransport:

tractor/_mp_fixup_main.py

@@ -18,9 +18,9 @@
 Helpers pulled mostly verbatim from ``multiprocessing.spawn``
 to aid with "fixing up" the ``__main__`` module in subprocesses.
 
-These helpers are needed for any spawing backend that doesn't already handle this.
+These helpers are needed for any spawing backend that doesn't already
-For example when using ``trio_run_in_process`` it is needed but obviously not when
+handle this. For example when using ``trio_run_in_process`` it is needed
-we're already using ``multiprocessing``.
+but obviously not when we're already using ``multiprocessing``.
 
 """
 import os
@@ -28,13 +28,12 @@ import sys
 import platform
 import types
 import runpy
-from typing import Dict
 
 
 ORIGINAL_DIR = os.path.abspath(os.getcwd())
 
 
-def _mp_figure_out_main() -> Dict[str, str]:
+def _mp_figure_out_main() -> dict[str, str]:
     """Taken from ``multiprocessing.spawn.get_preparation_data()``.
 
     Retrieve parent actor `__main__` module data.

tractor/_portal.py

@@ -35,6 +35,7 @@ import warnings
 import trio
 from async_generator import asynccontextmanager
 
+from .trionics import maybe_open_nursery
 from ._state import current_actor
 from ._ipc import Channel
 from .log import get_logger
@@ -44,43 +45,27 @@ from ._exceptions import (
     NoResult,
     ContextCancelled,
 )
-from ._streaming import Context, ReceiveMsgStream
+from ._streaming import (
+    Context,
+    MsgStream,
+)
 
 
 log = get_logger(__name__)
 
 
-@asynccontextmanager
-async def maybe_open_nursery(
-    nursery: trio.Nursery = None,
-    shield: bool = False,
-) -> AsyncGenerator[trio.Nursery, Any]:
-    '''
-    Create a new nursery if None provided.
-
-    Blocks on exit as expected if no input nursery is provided.
-
-    '''
-    if nursery is not None:
-        yield nursery
-    else:
-        async with trio.open_nursery() as nursery:
-            nursery.cancel_scope.shield = shield
-            yield nursery
-
-
 def _unwrap_msg(
-
     msg: dict[str, Any],
     channel: Channel
 
 ) -> Any:
+    __tracebackhide__ = True
     try:
         return msg['return']
     except KeyError:
         # internal error should never get here
         assert msg.get('cid'), "Received internal error at portal?"
-        raise unpack_error(msg, channel)
+        raise unpack_error(msg, channel) from None
 
 
 class MessagingError(Exception):
@@ -119,7 +104,7 @@ class Portal:
         # it is expected that ``result()`` will be awaited at some
         # point.
         self._expect_result: Optional[Context] = None
-        self._streams: set[ReceiveMsgStream] = set()
+        self._streams: set[MsgStream] = set()
         self.actor = current_actor()
 
     async def _submit_for_result(
@@ -154,6 +139,7 @@ class Portal:
         Return the result(s) from the remote actor's "main" task.
 
         '''
+        # __tracebackhide__ = True
         # Check for non-rpc errors slapped on the
         # channel for which we always raise
         exc = self.channel._exc
@@ -203,7 +189,7 @@ class Portal:
 
     async def cancel_actor(
         self,
-        timeout: float = None,
+        timeout: float | None = None,
 
     ) -> bool:
         '''
@@ -333,7 +319,7 @@ class Portal:
         async_gen_func: Callable,  # typing: ignore
         **kwargs,
 
-    ) -> AsyncGenerator[ReceiveMsgStream, None]:
+    ) -> AsyncGenerator[MsgStream, None]:
 
         if not inspect.isasyncgenfunction(async_gen_func):
             if not (
@@ -358,7 +344,7 @@ class Portal:
 
         try:
             # deliver receive only stream
-            async with ReceiveMsgStream(
+            async with MsgStream(
                 ctx, ctx._recv_chan,
             ) as rchan:
                 self._streams.add(rchan)
@@ -478,7 +464,6 @@ class Portal:
             # sure it's worth being pedantic:
             # Exception,
             # trio.Cancelled,
-            # trio.MultiError,
             # KeyboardInterrupt,
 
         ) as err:
@@ -515,6 +500,10 @@ class Portal:
                     f'actor: {uid}'
                 )
                 result = await ctx.result()
+                log.runtime(
+                    f'Context {fn_name} returned '
+                    f'value from callee `{result}`'
+                )
 
             # though it should be impossible for any tasks
             # operating *in* this scope to have survived
@@ -536,14 +525,22 @@ class Portal:
                         f'task:{cid}\n'
                         f'actor:{uid}'
                     )
-            else:
+            # XXX: (MEGA IMPORTANT) if this is a root opened process we
-                log.runtime(
+            # wait for any immediate child in debug before popping the
-                    f'Context {fn_name} returned '
+            # context from the runtime msg loop otherwise inside
-                    f'value from callee `{result}`'
+            # ``Actor._push_result()`` the msg will be discarded and in
-                )
+            # the case where that msg is global debugger unlock (via
+            # a "stop" msg for a stream), this can result in a deadlock
+            # where the root is waiting on the lock to clear but the
+            # child has already cleared it and clobbered IPC.
+            from ._debug import maybe_wait_for_debugger
+            await maybe_wait_for_debugger()
 
             # remove the context from runtime tracking
-            self.actor._contexts.pop((self.channel.uid, ctx.cid))
+            self.actor._contexts.pop(
+                (self.channel.uid, ctx.cid),
+                None,
+            )
 
 
 @dataclass
@@ -600,9 +597,11 @@ async def open_portal(
 
         msg_loop_cs: Optional[trio.CancelScope] = None
         if start_msg_loop:
+            from ._runtime import process_messages
             msg_loop_cs = await nursery.start(
                 partial(
-                    actor._process_messages,
+                    process_messages,
+                    actor,
                     channel,
                     # if the local task is cancelled we want to keep
                     # the msg loop running until our block ends

tractor/_root.py

@@ -22,14 +22,21 @@ from contextlib import asynccontextmanager
 from functools import partial
 import importlib
 import logging
+import signal
+import sys
 import os
-from typing import Tuple, Optional, List, Any
 import typing
 import warnings
 
+
+from exceptiongroup import BaseExceptionGroup
 import trio
 
-from ._actor import Actor, Arbiter
+from ._runtime import (
+    Actor,
+    Arbiter,
+    async_main,
+)
 from . import _debug
 from . import _spawn
 from . import _state
@@ -49,37 +56,45 @@ logger = log.get_logger('tractor')
 @asynccontextmanager
 async def open_root_actor(
 
+    *,
     # defaults are above
-    arbiter_addr: Optional[Tuple[str, int]] = (
+    arbiter_addr: tuple[str, int] | None = None,
-        _default_arbiter_host,
-        _default_arbiter_port,
-    ),
 
-    name: Optional[str] = 'root',
+    # defaults are above
+    registry_addr: tuple[str, int] | None = None,
+
+    name: str | None = 'root',
 
     # either the `multiprocessing` start method:
     # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
     # OR `trio` (the new default).
-    start_method: Optional[str] = None,
+    start_method: _spawn.SpawnMethodKey | None = None,
 
     # enables the multi-process debugger support
     debug_mode: bool = False,
 
     # internal logging
-    loglevel: Optional[str] = None,
+    loglevel: str | None = None,
 
-    enable_modules: Optional[List] = None,
+    enable_modules: list | None = None,
-    rpc_module_paths: Optional[List] = None,
+    rpc_module_paths: list | None = None,
 
 ) -> typing.Any:
-    """Async entry point for ``tractor``.
+    '''
+    Runtime init entry point for ``tractor``.
 
-    """
+    '''
     # Override the global debugger hook to make it play nice with
-    # ``trio``, see:
+    # ``trio``, see much discussion in:
     # https://github.com/python-trio/trio/issues/1155#issuecomment-742964018
+    builtin_bp_handler = sys.breakpointhook
+    orig_bp_path: str | None = os.environ.get('PYTHONBREAKPOINT', None)
     os.environ['PYTHONBREAKPOINT'] = 'tractor._debug._set_trace'
 
+    # attempt to retreive ``trio``'s sigint handler and stash it
+    # on our debugger lock state.
+    _debug.Lock._trio_handler = signal.getsignal(signal.SIGINT)
+
     # mark top most level process as root actor
     _state._runtime_vars['_is_root'] = True
 
@@ -98,18 +113,24 @@ async def open_root_actor(
     if start_method is not None:
         _spawn.try_set_start_method(start_method)
 
-    arbiter_addr = (host, port) = arbiter_addr or (
+    if arbiter_addr is not None:
+        warnings.warn(
+            '`arbiter_addr` is now deprecated and has been renamed to'
+            '`registry_addr`.\nUse that instead..',
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+    registry_addr = (host, port) = (
+        registry_addr
+        or arbiter_addr
+        or (
             _default_arbiter_host,
             _default_arbiter_port,
         )
+    )
 
-    if loglevel is None:
+    loglevel = (loglevel or log._default_loglevel).upper()
-        loglevel = log.get_loglevel()
-    else:
-        log._default_loglevel = loglevel
-        log.get_console_log(loglevel)
-
-    assert loglevel
 
     if debug_mode and _spawn._spawn_method == 'trio':
         _state._runtime_vars['_debug_mode'] = True
@@ -124,7 +145,7 @@ async def open_root_actor(
             logging.getLevelName(
                 # lul, need the upper case for the -> int map?
                 # sweet "dynamic function behaviour" stdlib...
-                loglevel.upper()
+                loglevel,
             ) > logging.getLevelName('PDB')
         ):
             loglevel = 'PDB'
@@ -134,20 +155,25 @@ async def open_root_actor(
             "Debug mode is only supported for the `trio` backend!"
         )
 
-    # make a temporary connection to see if an arbiter exists
+    log.get_console_log(loglevel)
-    arbiter_found = False
 
     try:
+        # make a temporary connection to see if an arbiter exists,
+        # if one can't be made quickly we assume none exists.
+        arbiter_found = False
+
         # TODO: this connect-and-bail forces us to have to carefully
         # rewrap TCP 104-connection-reset errors as EOF so as to avoid
         # propagating cancel-causing errors to the channel-msg loop
         # machinery.  Likely it would be better to eventually have
         # a "discovery" protocol with basic handshake instead.
+        with trio.move_on_after(1):
             async with _connect_chan(host, port):
                 arbiter_found = True
 
     except OSError:
-        logger.warning(f"No actor could be found @ {host}:{port}")
+        # TODO: make this a "discovery" log level?
+        logger.warning(f"No actor registry found @ {host}:{port}")
 
     # create a local actor and start up its main routine/task
     if arbiter_found:
@@ -157,7 +183,7 @@ async def open_root_actor(
 
         actor = Actor(
             name or 'anonymous',
-            arbiter_addr=arbiter_addr,
+            arbiter_addr=registry_addr,
             loglevel=loglevel,
             enable_modules=enable_modules,
         )
@@ -173,7 +199,7 @@ async def open_root_actor(
 
         actor = Arbiter(
             name or 'arbiter',
-            arbiter_addr=arbiter_addr,
+            arbiter_addr=registry_addr,
             loglevel=loglevel,
             enable_modules=enable_modules,
         )
@@ -189,13 +215,14 @@ async def open_root_actor(
         # start the actor runtime in a new task
         async with trio.open_nursery() as nursery:
 
-            # ``Actor._async_main()`` creates an internal nursery and
+            # ``_runtime.async_main()`` creates an internal nursery and
             # thus blocks here until the entire underlying actor tree has
             # terminated thereby conducting structured concurrency.
 
             await nursery.start(
                 partial(
-                    actor._async_main,
+                    async_main,
+                    actor,
                     accept_addr=(host, port),
                     parent_addr=None
                 )
@@ -203,7 +230,10 @@ async def open_root_actor(
             try:
                 yield actor
 
-            except (Exception, trio.MultiError) as err:
+            except (
+                Exception,
+                BaseExceptionGroup,
+            ) as err:
 
                 entered = await _debug._maybe_enter_pm(err)
 
@@ -216,7 +246,8 @@ async def open_root_actor(
             finally:
                 # NOTE: not sure if we'll ever need this but it's
                 # possibly better for even more determinism?
-                # logger.cancel(f'Waiting on {len(nurseries)} nurseries in root..')
+                # logger.cancel(
+                #     f'Waiting on {len(nurseries)} nurseries in root..')
                 # nurseries = actor._actoruid2nursery.values()
                 # async with trio.open_nursery() as tempn:
                 #     for an in nurseries:
@@ -226,64 +257,40 @@ async def open_root_actor(
                 await actor.cancel()
     finally:
         _state._current_actor = None
+
+        # restore breakpoint hook state
+        sys.breakpointhook = builtin_bp_handler
+        if orig_bp_path is not None:
+            os.environ['PYTHONBREAKPOINT'] = orig_bp_path
+        else:
+            # clear env back to having no entry
+            os.environ.pop('PYTHONBREAKPOINT')
+
         logger.runtime("Root actor terminated")
 
 
-def run(
-
-    # target
-    async_fn: typing.Callable[..., typing.Awaitable],
-    *args,
-
-    # runtime kwargs
-    name: Optional[str] = 'root',
-    arbiter_addr: Tuple[str, int] = (
-        _default_arbiter_host,
-        _default_arbiter_port,
-    ),
-
-    start_method: Optional[str] = None,
-    debug_mode: bool = False,
-    **kwargs,
-
-) -> Any:
-    """Run a trio-actor async function in process.
-
-    This is tractor's main entry and the start point for any async actor.
-    """
-    async def _main():
-
-        async with open_root_actor(
-            arbiter_addr=arbiter_addr,
-            name=name,
-            start_method=start_method,
-            debug_mode=debug_mode,
-            **kwargs,
-        ):
-
-            return await async_fn(*args)
-
-    warnings.warn(
-        "`tractor.run()` is now deprecated. `tractor` now"
-        " implicitly starts the root actor on first actor nursery"
-        " use. If you want to start the root actor manually, use"
-        " `tractor.open_root_actor()`.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    return trio.run(_main)
-
-
 def run_daemon(
     enable_modules: list[str],
+
+    # runtime kwargs
+    name: str | None = 'root',
+    registry_addr: tuple[str, int] = (
+        _default_arbiter_host,
+        _default_arbiter_port,
+    ),
+
+    start_method: str | None = None,
+    debug_mode: bool = False,
     **kwargs
+
 ) -> None:
     '''
-    Spawn daemon actor which will respond to RPC.
+    Spawn daemon actor which will respond to RPC; the main task simply
+    starts the runtime and then sleeps forever.
 
-    This is a convenience wrapper around
+    This is a very minimal convenience wrapper around starting
-    ``tractor.run(trio.sleep(float('inf')))`` such that the first actor spawned
+    a "run-until-cancelled" root actor which can be started with a set
-    is meant to run forever responding to RPC requests.
+    of enabled modules for RPC request handling.
 
     '''
     kwargs['enable_modules'] = list(enable_modules)
@@ -291,4 +298,15 @@ def run_daemon(
     for path in enable_modules:
         importlib.import_module(path)
 
-    return run(partial(trio.sleep, float('inf')), **kwargs)
+    async def _main():
+
+        async with open_root_actor(
+            registry_addr=registry_addr,
+            name=name,
+            start_method=start_method,
+            debug_mode=debug_mode,
+            **kwargs,
+        ):
+            return await trio.sleep_forever()
+
+    return trio.run(_main)

tractor/_spawn.py

@@ -18,30 +18,22 @@
 Machinery for actor process spawning using multiple backends.
 
 """
+from __future__ import annotations
 import sys
-import multiprocessing as mp
 import platform
 from typing import (
-    Any, Dict, Optional, Callable,
+    Any,
+    Awaitable,
+    Literal,
+    Callable,
     TypeVar,
+    TYPE_CHECKING,
 )
-from collections.abc import Awaitable
 
+from exceptiongroup import BaseExceptionGroup
 import trio
 from trio_typing import TaskStatus
 
-try:
-    from multiprocessing import semaphore_tracker  # type: ignore
-    resource_tracker = semaphore_tracker
-    resource_tracker._resource_tracker = resource_tracker._semaphore_tracker
-except ImportError:
-    # 3.8 introduces a more general version that also tracks shared mems
-    from multiprocessing import resource_tracker  # type: ignore
-
-from multiprocessing import forkserver  # type: ignore
-from typing import Tuple
-
-from . import _forkserver_override
 from ._debug import (
     maybe_wait_for_debugger,
     acquire_debug_lock,
@@ -52,24 +44,33 @@ from ._state import (
     is_root_process,
     debug_mode,
 )
-
 from .log import get_logger
 from ._portal import Portal
-from ._actor import Actor
+from ._runtime import Actor
 from ._entry import _mp_main
 from ._exceptions import ActorFailure
 
 
-log = get_logger('tractor')
+if TYPE_CHECKING:
+    from ._supervise import ActorNursery
+    import multiprocessing as mp
     ProcessType = TypeVar('ProcessType', mp.Process, trio.Process)
 
+log = get_logger('tractor')
+
 # placeholder for an mp start context if so using that backend
-_ctx: Optional[mp.context.BaseContext] = None
+_ctx: mp.context.BaseContext | None = None
-_spawn_method: str = "trio"
+SpawnMethodKey = Literal[
+    'trio',  # supported on all platforms
+    'mp_spawn',
+    'mp_forkserver',  # posix only
+]
+_spawn_method: SpawnMethodKey = 'trio'
 
 
 if platform.system() == 'Windows':
 
+    import multiprocessing as mp
     _ctx = mp.get_context("spawn")
 
     async def proc_waiter(proc: mp.Process) -> None:
@@ -81,7 +82,10 @@ else:
         await trio.lowlevel.wait_readable(proc.sentinel)
 
 
-def try_set_start_method(name: str) -> Optional[mp.context.BaseContext]:
+def try_set_start_method(
+    key: SpawnMethodKey
+
+) -> mp.context.BaseContext | None:
     '''
     Attempt to set the method for process starting, aka the "actor
     spawning backend".
@@ -92,30 +96,34 @@ def try_set_start_method(name: str) -> Optional[mp.context.BaseContext]:
     ``subprocess.Popen``.
 
     '''
+    import multiprocessing as mp
     global _ctx
     global _spawn_method
 
-    methods = mp.get_all_start_methods()
+    mp_methods = mp.get_all_start_methods()
-    if 'fork' in methods:
+    if 'fork' in mp_methods:
         # forking is incompatible with ``trio``s global task tree
-        methods.remove('fork')
+        mp_methods.remove('fork')
 
-    # supported on all platforms
+    match key:
-    methods += ['trio']
+        case 'mp_forkserver':
-
+            from . import _forkserver_override
-    if name not in methods:
-        raise ValueError(
-            f"Spawn method `{name}` is invalid please choose one of {methods}"
-        )
-    elif name == 'forkserver':
             _forkserver_override.override_stdlib()
-        _ctx = mp.get_context(name)
+            _ctx = mp.get_context('forkserver')
-    elif name == 'trio':
-        _ctx = None
-    else:
-        _ctx = mp.get_context(name)
 
-    _spawn_method = name
+        case 'mp_spawn':
+            _ctx = mp.get_context('spawn')
+
+        case 'trio':
+            _ctx = None
+
+        case _:
+            raise ValueError(
+                f'Spawn method `{key}` is invalid!\n'
+                f'Please choose one of {SpawnMethodKey}'
+            )
+
+    _spawn_method = key
     return _ctx
 
 
@@ -131,6 +139,7 @@ async def exhaust_portal(
     If the main task is an async generator do our best to consume
     what's left of it.
     '''
+    __tracebackhide__ = True
     try:
         log.debug(f"Waiting on final result from {actor.uid}")
 
@@ -138,8 +147,11 @@ async def exhaust_portal(
         # always be established and shutdown using a context manager api
         final = await portal.result()
 
-    except (Exception, trio.MultiError) as err:
+    except (
-        # we reraise in the parent task via a ``trio.MultiError``
+        Exception,
+        BaseExceptionGroup,
+    ) as err:
+        # we reraise in the parent task via a ``BaseExceptionGroup``
         return err
     except trio.Cancelled as err:
         # lol, of course we need this too ;P
@@ -155,7 +167,7 @@ async def cancel_on_completion(
 
     portal: Portal,
     actor: Actor,
-    errors: Dict[Tuple[str, str], Exception],
+    errors: dict[tuple[str, str], Exception],
 
 ) -> None:
     '''
@@ -167,7 +179,7 @@ async def cancel_on_completion(
     '''
     # if this call errors we store the exception for later
     # in ``errors`` which will be reraised inside
-    # a MultiError and we still send out a cancel request
+    # an exception group and we still send out a cancel request
     result = await exhaust_portal(portal, actor)
     if isinstance(result, Exception):
         errors[actor.uid] = result
@@ -187,16 +199,37 @@ async def cancel_on_completion(
 async def do_hard_kill(
     proc: trio.Process,
     terminate_after: int = 3,
+
 ) -> None:
     # NOTE: this timeout used to do nothing since we were shielding
     # the ``.wait()`` inside ``new_proc()`` which will pretty much
     # never release until the process exits, now it acts as
     # a hard-kill time ultimatum.
+    log.debug(f"Terminating {proc}")
     with trio.move_on_after(terminate_after) as cs:
 
-        # NOTE: This ``__aexit__()`` shields internally.
+        # NOTE: code below was copied verbatim from the now deprecated
-        async with proc:  # calls ``trio.Process.aclose()``
+        # (in 0.20.0) ``trio._subrocess.Process.aclose()``, orig doc
-            log.debug(f"Terminating {proc}")
+        # string:
+        #
+        # Close any pipes we have to the process (both input and output)
+        # and wait for it to exit. If cancelled, kills the process and
+        # waits for it to finish exiting before propagating the
+        # cancellation.
+        with trio.CancelScope(shield=True):
+            if proc.stdin is not None:
+                await proc.stdin.aclose()
+            if proc.stdout is not None:
+                await proc.stdout.aclose()
+            if proc.stderr is not None:
+                await proc.stderr.aclose()
+        try:
+            await proc.wait()
+        finally:
+            if proc.returncode is None:
+                proc.kill()
+                with trio.CancelScope(shield=True):
+                    await proc.wait()
 
     if cs.cancelled_caught:
         # XXX: should pretty much never get here unless we have
@@ -247,29 +280,65 @@ async def soft_wait(
 
             if proc.poll() is None:  # type: ignore
                 log.warning(
-                    f'Process still alive after cancel request:\n{uid}')
+                    'Actor still alive after cancel request:\n'
+                    f'{uid}'
+                )
 
                 n.cancel_scope.cancel()
         raise
 
 
 async def new_proc(
-
     name: str,
-    actor_nursery: 'ActorNursery',  # type: ignore  # noqa
+    actor_nursery: ActorNursery,
     subactor: Actor,
-    errors: Dict[Tuple[str, str], Exception],
+    errors: dict[tuple[str, str], Exception],
 
     # passed through to actor main
-    bind_addr: Tuple[str, int],
+    bind_addr: tuple[str, int],
-    parent_addr: Tuple[str, int],
+    parent_addr: tuple[str, int],
-    _runtime_vars: Dict[str, Any],  # serialized and sent to _child
+    _runtime_vars: dict[str, Any],  # serialized and sent to _child
 
     *,
 
     infect_asyncio: bool = False,
     task_status: TaskStatus[Portal] = trio.TASK_STATUS_IGNORED
 
+) -> None:
+
+    # lookup backend spawning target
+    target = _methods[_spawn_method]
+
+    # mark the new actor with the global spawn method
+    subactor._spawn_method = _spawn_method
+
+    await target(
+        name,
+        actor_nursery,
+        subactor,
+        errors,
+        bind_addr,
+        parent_addr,
+        _runtime_vars,  # run time vars
+        infect_asyncio=infect_asyncio,
+        task_status=task_status,
+    )
+
+
+async def trio_proc(
+    name: str,
+    actor_nursery: ActorNursery,
+    subactor: Actor,
+    errors: dict[tuple[str, str], Exception],
+
+    # passed through to actor main
+    bind_addr: tuple[str, int],
+    parent_addr: tuple[str, int],
+    _runtime_vars: dict[str, Any],  # serialized and sent to _child
+    *,
+    infect_asyncio: bool = False,
+    task_status: TaskStatus[Portal] = trio.TASK_STATUS_IGNORED
+
 ) -> None:
     '''
     Create a new ``Process`` using a "spawn method" as (configured using
@@ -279,11 +348,6 @@ async def new_proc(
     here is to be considered the core supervision strategy.
 
     '''
-    # mark the new actor with the global spawn method
-    subactor._spawn_method = _spawn_method
-    uid = subactor.uid
-
-    if _spawn_method == 'trio':
     spawn_cmd = [
         sys.executable,
         "-m",
@@ -295,7 +359,7 @@ async def new_proc(
         # the OS; it otherwise can be passed via the parent channel if
         # we prefer in the future (for privacy).
         "--uid",
-            str(uid),
+        str(subactor.uid),
         # Address the child must connect to on startup
         "--parent_addr",
         str(parent_addr)
@@ -311,17 +375,19 @@ async def new_proc(
         spawn_cmd.append("--asyncio")
 
     cancelled_during_spawn: bool = False
-        proc: Optional[trio.Process] = None
+    proc: trio.Process | None = None
     try:
         try:
-                proc = await trio.open_process(spawn_cmd)
+            # TODO: needs ``trio_typing`` patch?
+            proc = await trio.lowlevel.open_process(spawn_cmd)
 
             log.runtime(f"Started {proc}")
 
             # wait for actor to spawn and connect back to us
             # channel should have handshake completed by the
             # local actor by the time we get a ref to it
-                event, chan = await actor_nursery._actor.wait_for_peer(uid)
+            event, chan = await actor_nursery._actor.wait_for_peer(
+                subactor.uid)
 
         except trio.Cancelled:
             cancelled_during_spawn = True
@@ -334,15 +400,21 @@ async def new_proc(
                         await maybe_wait_for_debugger()
 
                     elif proc is not None:
-                            async with acquire_debug_lock(uid):
+                        async with acquire_debug_lock(subactor.uid):
                             # soft wait on the proc to terminate
                             with trio.move_on_after(0.5):
                                 await proc.wait()
             raise
 
+        # a sub-proc ref **must** exist now
+        assert proc
+
         portal = Portal(chan)
         actor_nursery._children[subactor.uid] = (
-                subactor, proc, portal)
+            subactor,
+            proc,
+            portal,
+        )
 
         # send additional init params
         await chan.send({
@@ -362,13 +434,18 @@ async def new_proc(
         task_status.started(portal)
 
         # wait for ActorNursery.wait() to be called
-            n_exited = actor_nursery._join_procs
         with trio.CancelScope(shield=True):
-                await n_exited.wait()
+            await actor_nursery._join_procs.wait()
 
         async with trio.open_nursery() as nursery:
+            if portal in actor_nursery._cancel_after_result_on_exit:
+                nursery.start_soon(
+                    cancel_on_completion,
+                    portal,
+                    subactor,
+                    errors
+                )
 
-                async def soft_wait_and_maybe_cancel_ria_task():
             # This is a "soft" (cancellable) join/reap which
             # will remote cancel the actor on a ``trio.Cancelled``
             # condition.
@@ -378,66 +455,39 @@ async def new_proc(
                 portal
             )
 
-                    if n_exited.is_set():
             # cancel result waiter that may have been spawned in
             # tandem if not done already
             log.warning(
                 "Cancelling existing result waiter task for "
-                            f"{subactor.uid}"
+                f"{subactor.uid}")
-                        )
             nursery.cancel_scope.cancel()
 
-                    else:
-                        log.warning(
-                            f'Process for actor {uid} terminated before'
-                            'nursery exit. ' 'This may mean an IPC'
-                            'connection failed!'
-                        )
-
-                nursery.start_soon(soft_wait_and_maybe_cancel_ria_task)
-
-                # TODO: when we finally remove the `.run_in_actor()` api
-                # we should be able to entirely drop these 2 blocking calls:
-                # - we don't need to wait on nursery exit to capture
-                #   process-spawn-machinery level errors (and propagate them).
-                # - we don't need to wait on final results from ria portals
-                #   since this will be done in some higher level wrapper API.
-
-                # XXX: interestingly we can't put this here bc it causes
-                # the pub-sub tests to fail? wth.. should probably drop
-                # those XD
-                # wait for ActorNursery.wait() to be called
-                # with trio.CancelScope(shield=True):
-                #     await n_exited.wait()
-
-                if portal in actor_nursery._cancel_after_result_on_exit:
-                    nursery.start_soon(
-                        cancel_on_completion,
-                        portal,
-                        subactor,
-                        errors
-                    )
-
     finally:
-            # The "hard" reap since no actor zombies are allowed!
+        # XXX NOTE XXX: The "hard" reap since no actor zombies are
-            # XXX: do this **after** cancellation/tearfown to avoid
+        # allowed! Do this **after** cancellation/teardown to avoid
         # killing the process too early.
         if proc:
-                log.cancel(f'Hard reap sequence starting for {uid}')
+            log.cancel(f'Hard reap sequence starting for {subactor.uid}')
             with trio.CancelScope(shield=True):
 
                 # don't clobber an ongoing pdb
                 if cancelled_during_spawn:
                     # Try again to avoid TTY clobbering.
-                        async with acquire_debug_lock(uid):
+                    async with acquire_debug_lock(subactor.uid):
                         with trio.move_on_after(0.5):
                             await proc.wait()
 
                 if is_root_process():
-
+                    # TODO: solve the following issue where we need
+                    # to do a similar wait like this but in an
+                    # "intermediary" parent actor that itself isn't
+                    # in debug but has a child that is, and we need
+                    # to hold off on relaying SIGINT until that child
+                    # is complete.
+                    # https://github.com/goodboy/tractor/issues/320
                     await maybe_wait_for_debugger(
                         child_in_debug=_runtime_vars.get(
-                                '_debug_mode', False)
+                            '_debug_mode', False),
                     )
 
                 if proc.poll() is None:
@@ -453,43 +503,36 @@ async def new_proc(
             # subactor
             actor_nursery._children.pop(subactor.uid)
 
-    else:
-        # `multiprocessing`
-        # async with trio.open_nursery() as nursery:
-        await mp_new_proc(
-            name=name,
-            actor_nursery=actor_nursery,
-            subactor=subactor,
-            errors=errors,
-
-            # passed through to actor main
-            bind_addr=bind_addr,
-            parent_addr=parent_addr,
-            _runtime_vars=_runtime_vars,
-            infect_asyncio=infect_asyncio,
-            task_status=task_status,
-        )
-
-
-async def mp_new_proc(
 
+async def mp_proc(
     name: str,
-    actor_nursery: 'ActorNursery',  # type: ignore  # noqa
+    actor_nursery: ActorNursery,  # type: ignore  # noqa
     subactor: Actor,
-    errors: Dict[Tuple[str, str], Exception],
+    errors: dict[tuple[str, str], Exception],
     # passed through to actor main
-    bind_addr: Tuple[str, int],
+    bind_addr: tuple[str, int],
-    parent_addr: Tuple[str, int],
+    parent_addr: tuple[str, int],
-    _runtime_vars: Dict[str, Any],  # serialized and sent to _child
+    _runtime_vars: dict[str, Any],  # serialized and sent to _child
     *,
     infect_asyncio: bool = False,
     task_status: TaskStatus[Portal] = trio.TASK_STATUS_IGNORED
 
 ) -> None:
 
+    # uggh zone
+    try:
+        from multiprocessing import semaphore_tracker  # type: ignore
+        resource_tracker = semaphore_tracker
+        resource_tracker._resource_tracker = resource_tracker._semaphore_tracker  # noqa
+    except ImportError:
+        # 3.8 introduces a more general version that also tracks shared mems
+        from multiprocessing import resource_tracker  # type: ignore
+
     assert _ctx
     start_method = _ctx.get_start_method()
     if start_method == 'forkserver':
+
+        from multiprocessing import forkserver  # type: ignore
         # XXX do our hackery on the stdlib to avoid multiple
         # forkservers (one at each subproc layer).
         fs = forkserver._forkserver
@@ -501,23 +544,24 @@ async def mp_new_proc(
             # forkserver.set_forkserver_preload(enable_modules)
             forkserver.ensure_running()
             fs_info = (
-                fs._forkserver_address,
+                fs._forkserver_address,  # type: ignore  # noqa
-                fs._forkserver_alive_fd,
+                fs._forkserver_alive_fd,  # type: ignore  # noqa
                 getattr(fs, '_forkserver_pid', None),
                 getattr(
                     resource_tracker._resource_tracker, '_pid', None),
                 resource_tracker._resource_tracker._fd,
             )
-        else:
+        else:  # request to forkerserver to fork a new child
             assert curr_actor._forkserver_info
             fs_info = (
-                fs._forkserver_address,
+                fs._forkserver_address,  # type: ignore  # noqa
-                fs._forkserver_alive_fd,
+                fs._forkserver_alive_fd,  # type: ignore  # noqa
-                fs._forkserver_pid,
+                fs._forkserver_pid,  # type: ignore  # noqa
                 resource_tracker._resource_tracker._pid,
                 resource_tracker._resource_tracker._fd,
              ) = curr_actor._forkserver_info
     else:
+        # spawn method
         fs_info = (None, None, None, None, None)
 
     proc: mp.Process = _ctx.Process(  # type: ignore
@@ -526,7 +570,7 @@ async def mp_new_proc(
             subactor,
             bind_addr,
             fs_info,
-            start_method,
+            _spawn_method,
             parent_addr,
             infect_asyncio,
         ),
@@ -620,4 +664,16 @@ async def mp_new_proc(
         log.debug(f"Joined {proc}")
 
         # pop child entry to indicate we are no longer managing subactor
-        subactor, proc, portal = actor_nursery._children.pop(subactor.uid)
+        actor_nursery._children.pop(subactor.uid)
+
+        # TODO: prolly report to ``mypy`` how this causes all sorts of
+        # false errors..
+        # subactor, proc, portal = actor_nursery._children.pop(subactor.uid)
+
+
+# proc spawning backend target map
+_methods: dict[SpawnMethodKey, Callable] = {
+    'trio': trio_proc,
+    'mp_spawn': mp_proc,
+    'mp_forkserver': mp_proc,
+}

tractor/_state.py

@@ -18,9 +18,10 @@
 Per process state
 
 """
-from typing import Optional, Dict, Any
+from typing import (
-from collections.abc import Mapping
+    Optional,
-import multiprocessing as mp
+    Any,
+)
 
 import trio
 
@@ -28,7 +29,7 @@ from ._exceptions import NoRuntime
 
 
 _current_actor: Optional['Actor'] = None  # type: ignore # noqa
-_runtime_vars: Dict[str, Any] = {
+_runtime_vars: dict[str, Any] = {
     '_debug_mode': False,
     '_is_root': False,
     '_root_mailbox': (None, None)
@@ -44,33 +45,10 @@ def current_actor(err_on_no_runtime: bool = True) -> 'Actor':  # type: ignore #
     return _current_actor
 
 
-_conc_name_getters = {
-    'task': trio.lowlevel.current_task,
-    'actor': current_actor
-}
-
-
-class ActorContextInfo(Mapping):
-    "Dyanmic lookup for local actor and task names"
-    _context_keys = ('task', 'actor')
-
-    def __len__(self):
-        return len(self._context_keys)
-
-    def __iter__(self):
-        return iter(self._context_keys)
-
-    def __getitem__(self, key: str) -> str:
-        try:
-            return _conc_name_getters[key]().name  # type: ignore
-        except RuntimeError:
-            # no local actor/task context initialized yet
-            return f'no {key} context'
-
-
 def is_main_process() -> bool:
     """Bool determining if this actor is running in the top-most process.
     """
+    import multiprocessing as mp
     return mp.current_process().name == 'MainProcess'
 
 

tractor/_streaming.py

@@ -23,8 +23,10 @@ import inspect
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from typing import (
-    Any, Optional, Callable,
+    Any,
-    AsyncGenerator, Dict,
+    Optional,
+    Callable,
+    AsyncGenerator,
     AsyncIterator
 )
 
@@ -48,12 +50,13 @@ log = get_logger(__name__)
 # - use __slots__ on ``Context``?
 
 
-class ReceiveMsgStream(trio.abc.ReceiveChannel):
+class MsgStream(trio.abc.Channel):
     '''
-    A IPC message stream for receiving logically sequenced values over
+    A bidirectional message stream for receiving logically sequenced
-    an inter-actor ``Channel``. This is the type returned to a local
+    values over an inter-actor IPC ``Channel``.
-    task which entered either ``Portal.open_stream_from()`` or
+
-    ``Context.open_stream()``.
+    This is the type returned to a local task which entered either
+    ``Portal.open_stream_from()`` or ``Context.open_stream()``.
 
     Termination rules:
 
@@ -95,6 +98,9 @@ class ReceiveMsgStream(trio.abc.ReceiveChannel):
         if self._eoc:
             raise trio.EndOfChannel
 
+        if self._closed:
+            raise trio.ClosedResourceError('This stream was closed')
+
         try:
             msg = await self._rx_chan.receive()
             return msg['yield']
@@ -108,6 +114,9 @@ class ReceiveMsgStream(trio.abc.ReceiveChannel):
             # - 'error'
             # possibly just handle msg['stop'] here!
 
+            if self._closed:
+                raise trio.ClosedResourceError('This stream was closed')
+
             if msg.get('stop') or self._eoc:
                 log.debug(f"{self} was stopped at remote end")
 
@@ -187,7 +196,6 @@ class ReceiveMsgStream(trio.abc.ReceiveChannel):
             return
 
         self._eoc = True
-        self._closed = True
 
         # NOTE: this is super subtle IPC messaging stuff:
         # Relay stop iteration to far end **iff** we're
@@ -204,12 +212,8 @@ class ReceiveMsgStream(trio.abc.ReceiveChannel):
 
         # In the bidirectional case, `Context.open_stream()` will create
         # the `Actor._cids2qs` entry from a call to
-        # `Actor.get_context()` and will send the stop message in
+        # `Actor.get_context()` and will call us here to send the stop
-        # ``__aexit__()`` on teardown so it **does not** need to be
+        # msg in ``__aexit__()`` on teardown.
-        # called here.
-        if not self._ctx._portal:
-            # Only for 2 way streams can we can send stop from the
-            # caller side.
         try:
             # NOTE: if this call is cancelled we expect this end to
             # handle as though the stop was never sent (though if it
@@ -226,7 +230,14 @@ class ReceiveMsgStream(trio.abc.ReceiveChannel):
             # the underlying channel may already have been pulled
             # in which case our stop message is meaningless since
             # it can't traverse the transport.
-                log.debug(f'Channel for {self} was already closed')
+            ctx = self._ctx
+            log.warning(
+                f'Stream was already destroyed?\n'
+                f'actor: {ctx.chan.uid}\n'
+                f'ctx id: {ctx.cid}'
+            )
+
+        self._closed = True
 
         # Do we close the local mem chan ``self._rx_chan`` ??!?
 
@@ -269,7 +280,8 @@ class ReceiveMsgStream(trio.abc.ReceiveChannel):
         self,
 
     ) -> AsyncIterator[BroadcastReceiver]:
-        '''Allocate and return a ``BroadcastReceiver`` which delegates
+        '''
+        Allocate and return a ``BroadcastReceiver`` which delegates
         to this message stream.
 
         This allows multiple local tasks to receive each their own copy
@@ -306,15 +318,15 @@ class ReceiveMsgStream(trio.abc.ReceiveChannel):
         async with self._broadcaster.subscribe() as bstream:
             assert bstream.key != self._broadcaster.key
             assert bstream._recv == self._broadcaster._recv
+
+            # NOTE: we patch on a `.send()` to the bcaster so that the
+            # caller can still conduct 2-way streaming using this
+            # ``bstream`` handle transparently as though it was the msg
+            # stream instance.
+            bstream.send = self.send  # type: ignore
+
             yield bstream
 
-
-class MsgStream(ReceiveMsgStream, trio.abc.Channel):
-    '''
-    Bidirectional message stream for use within an inter-actor actor
-    ``Context```.
-
-    '''
     async def send(
         self,
         data: Any
@@ -369,6 +381,8 @@ class Context:
 
     # status flags
     _cancel_called: bool = False
+    _cancel_msg: Optional[str] = None
+    _enter_debugger_on_cancel: bool = True
     _started_called: bool = False
     _started_received: bool = False
     _stream_opened: bool = False
@@ -393,7 +407,7 @@ class Context:
 
     async def _maybe_raise_from_remote_msg(
         self,
-        msg: Dict[str, Any],
+        msg: dict[str, Any],
 
     ) -> None:
         '''
@@ -450,7 +464,11 @@ class Context:
                 if not self._scope_nursery._closed:  # type: ignore
                     self._scope_nursery.start_soon(raiser)
 
-    async def cancel(self) -> None:
+    async def cancel(
+        self,
+        msg: Optional[str] = None,
+
+    ) -> None:
         '''
         Cancel this inter-actor-task context.
 
@@ -459,6 +477,8 @@ class Context:
 
         '''
         side = 'caller' if self._portal else 'callee'
+        if msg:
+            assert side == 'callee', 'Only callee side can provide cancel msg'
 
         log.cancel(f'Cancelling {side} side of context to {self.chan.uid}')
 
@@ -495,8 +515,10 @@ class Context:
                     log.cancel(
                         "Timed out on cancelling remote task "
                         f"{cid} for {self._portal.channel.uid}")
-        else:
+
         # callee side remote task
+        else:
+            self._cancel_msg = msg
 
             # TODO: should we have an explicit cancel message
             # or is relaying the local `trio.Cancelled` as an
@@ -581,27 +603,34 @@ class Context:
         async with MsgStream(
             ctx=self,
             rx_chan=ctx._recv_chan,
-        ) as rchan:
+        ) as stream:
 
             if self._portal:
-                self._portal._streams.add(rchan)
+                self._portal._streams.add(stream)
 
             try:
                 self._stream_opened = True
 
-                # ensure we aren't cancelled before delivering
+                # XXX: do we need this?
-                # the stream
+                # ensure we aren't cancelled before yielding the stream
                 # await trio.lowlevel.checkpoint()
-                yield rchan
+                yield stream
 
-                # XXX: Make the stream "one-shot use".  On exit, signal
+                # NOTE: Make the stream "one-shot use".  On exit, signal
                 # ``trio.EndOfChannel``/``StopAsyncIteration`` to the
                 # far end.
-                await self.send_stop()
+                await stream.aclose()
 
             finally:
                 if self._portal:
-                    self._portal._streams.remove(rchan)
+                    try:
+                        self._portal._streams.remove(stream)
+                    except KeyError:
+                        log.warning(
+                            f'Stream was already destroyed?\n'
+                            f'actor: {self.chan.uid}\n'
+                            f'ctx id: {self.cid}'
+                        )
 
     async def result(self) -> Any:
         '''

tractor/_supervise.py

@@ -18,20 +18,23 @@
 ``trio`` inspired apis and helpers
 
 """
+from contextlib import asynccontextmanager as acm
 from functools import partial
 import inspect
-import multiprocessing as mp
+from typing import (
-from typing import Tuple, List, Dict, Optional
+    Optional,
+    TYPE_CHECKING,
+)
 import typing
 import warnings
 
+from exceptiongroup import BaseExceptionGroup
 import trio
-from async_generator import asynccontextmanager
 
 from ._debug import maybe_wait_for_debugger
 from ._state import current_actor, is_main_process
 from .log import get_logger, get_loglevel
-from ._actor import Actor
+from ._runtime import Actor
 from ._portal import Portal
 from ._exceptions import is_multi_cancelled
 from ._root import open_root_actor
@@ -39,9 +42,12 @@ from . import _state
 from . import _spawn
 
 
+if TYPE_CHECKING:
+    import multiprocessing as mp
+
 log = get_logger(__name__)
 
-_default_bind_addr: Tuple[str, int] = ('127.0.0.1', 0)
+_default_bind_addr: tuple[str, int] = ('127.0.0.1', 0)
 
 
 class ActorNursery:
@@ -77,15 +83,19 @@ class ActorNursery:
         actor: Actor,
         ria_nursery: trio.Nursery,
         da_nursery: trio.Nursery,
-        errors: Dict[Tuple[str, str], Exception],
+        errors: dict[tuple[str, str], BaseException],
     ) -> None:
         # self.supervisor = supervisor  # TODO
         self._actor: Actor = actor
         self._ria_nursery = ria_nursery
         self._da_nursery = da_nursery
-        self._children: Dict[
+        self._children: dict[
-            Tuple[str, str],
+            tuple[str, str],
-            Tuple[Actor, mp.Process, Optional[Portal]]
+            tuple[
+                Actor,
+                trio.Process | mp.Process,
+                Optional[Portal],
+            ]
         ] = {}
         # portals spawned with ``run_in_actor()`` are
         # cancelled when their "main" result arrives
@@ -100,12 +110,12 @@ class ActorNursery:
         self,
         name: str,
         *,
-        bind_addr: Tuple[str, int] = _default_bind_addr,
+        bind_addr: tuple[str, int] = _default_bind_addr,
-        rpc_module_paths: List[str] = None,
+        rpc_module_paths: list[str] | None = None,
-        enable_modules: List[str] = None,
+        enable_modules: list[str] | None = None,
-        loglevel: str = None,  # set log level per subactor
+        loglevel: str | None = None,  # set log level per subactor
-        nursery: trio.Nursery = None,
+        nursery: trio.Nursery | None = None,
-        debug_mode: Optional[bool] = None,
+        debug_mode: Optional[bool] | None = None,
         infect_asyncio: bool = False,
     ) -> Portal:
         '''
@@ -171,10 +181,10 @@ class ActorNursery:
         *,
 
         name: Optional[str] = None,
-        bind_addr: Tuple[str, int] = _default_bind_addr,
+        bind_addr: tuple[str, int] = _default_bind_addr,
-        rpc_module_paths: Optional[List[str]] = None,
+        rpc_module_paths: list[str] | None = None,
-        enable_modules: List[str] = None,
+        enable_modules: list[str] | None = None,
-        loglevel: str = None,  # set log level per subactor
+        loglevel: str | None = None,  # set log level per subactor
         infect_asyncio: bool = False,
 
         **kwargs,  # explicit args to ``fn``
@@ -285,13 +295,17 @@ class ActorNursery:
         self._join_procs.set()
 
 
-@asynccontextmanager
+@acm
 async def _open_and_supervise_one_cancels_all_nursery(
     actor: Actor,
+
 ) -> typing.AsyncGenerator[ActorNursery, None]:
 
+    # TODO: yay or nay?
+    __tracebackhide__ = True
+
     # the collection of errors retreived from spawned sub-actors
-    errors: Dict[Tuple[str, str], Exception] = {}
+    errors: dict[tuple[str, str], BaseException] = {}
 
     # This is the outermost level "deamon actor" nursery. It is awaited
     # **after** the below inner "run in actor nursery". This allows for
@@ -324,19 +338,17 @@ async def _open_and_supervise_one_cancels_all_nursery(
                     # after we yield upwards
                     yield anursery
 
+                    # When we didn't error in the caller's scope,
+                    # signal all process-monitor-tasks to conduct
+                    # the "hard join phase".
                     log.runtime(
                         f"Waiting on subactors {anursery._children} "
                         "to complete"
                     )
-
-                    # Last bit before first nursery block ends in the case
-                    # where we didn't error in the caller's scope
-
-                    # signal all process monitor tasks to conduct
-                    # hard join phase.
                     anursery._join_procs.set()
 
-                except BaseException as err:
+                except BaseException as inner_err:
+                    errors[actor.uid] = inner_err
 
                     # If we error in the root but the debugger is
                     # engaged we don't want to prematurely kill (and
@@ -353,19 +365,18 @@ async def _open_and_supervise_one_cancels_all_nursery(
                     # worry more are coming).
                     anursery._join_procs.set()
 
-                    try:
                     # XXX: hypothetically an error could be
                     # raised and then a cancel signal shows up
                     # slightly after in which case the `else:`
                     # block here might not complete?  For now,
                     # shield both.
                     with trio.CancelScope(shield=True):
-                            etype = type(err)
+                        etype = type(inner_err)
                         if etype in (
                             trio.Cancelled,
                             KeyboardInterrupt
                         ) or (
-                                is_multi_cancelled(err)
+                            is_multi_cancelled(inner_err)
                         ):
                             log.cancel(
                                 f"Nursery for {current_actor().uid} "
@@ -373,29 +384,23 @@ async def _open_and_supervise_one_cancels_all_nursery(
                         else:
                             log.exception(
                                 f"Nursery for {current_actor().uid} "
-                                    f"errored with {err}, ")
+                                f"errored with")
 
                         # cancel all subactors
                         await anursery.cancel()
 
-                    except trio.MultiError as merr:
-                        # If we receive additional errors while waiting on
-                        # remaining subactors that were cancelled,
-                        # aggregate those errors with the original error
-                        # that triggered this teardown.
-                        if err not in merr.exceptions:
-                            raise trio.MultiError(merr.exceptions + [err])
-                    else:
-                        raise
-
             # ria_nursery scope end
 
-        # XXX: do we need a `trio.Cancelled` catch here as well?
+        # TODO: this is the handler around the ``.run_in_actor()``
-        # this is the catch around the ``.run_in_actor()`` nursery
+        # nursery. Ideally we can drop this entirely in the future as
+        # the whole ``.run_in_actor()`` API should be built "on top of"
+        # this lower level spawn-request-cancel "daemon actor" API where
+        # a local in-actor task nursery is used with one-to-one task
+        # + `await Portal.run()` calls and the results/errors are
+        # handled directly (inline) and errors by the local nursery.
         except (
-
             Exception,
-            trio.MultiError,
+            BaseExceptionGroup,
             trio.Cancelled
 
         ) as err:
@@ -427,18 +432,20 @@ async def _open_and_supervise_one_cancels_all_nursery(
                     with trio.CancelScope(shield=True):
                         await anursery.cancel()
 
-                # use `MultiError` as needed
+                # use `BaseExceptionGroup` as needed
                 if len(errors) > 1:
-                    raise trio.MultiError(tuple(errors.values()))
+                    raise BaseExceptionGroup(
+                        'tractor.ActorNursery errored with',
+                        tuple(errors.values()),
+                    )
                 else:
                     raise list(errors.values())[0]
 
-        # ria_nursery scope end - nursery checkpoint
+        # da_nursery scope end - nursery checkpoint
-
+    # final exit
-    # after nursery exit
 
 
-@asynccontextmanager
+@acm
 async def open_nursery(
     **kwargs,
 

tractor/experimental/_pubsub.py

@@ -26,7 +26,10 @@ support provided by ``tractor.Context.open_stream()`` and friends.
 from __future__ import annotations
 import inspect
 import typing
-from typing import Dict, Any, Set, Callable, List, Tuple
+from typing import (
+    Any,
+    Callable,
+)
 from functools import partial
 from async_generator import aclosing
 
@@ -44,8 +47,8 @@ log = get_logger('messaging')
 
 async def fan_out_to_ctxs(
     pub_async_gen_func: typing.Callable,  # it's an async gen ... gd mypy
-    topics2ctxs: Dict[str, list],
+    topics2ctxs: dict[str, list],
-    packetizer: typing.Callable = None,
+    packetizer: typing.Callable | None = None,
 ) -> None:
     '''
     Request and fan out quotes to each subscribed actor channel.
@@ -61,7 +64,7 @@ async def fan_out_to_ctxs(
 
         async for published in pub_gen:
 
-            ctx_payloads: List[Tuple[Context, Any]] = []
+            ctx_payloads: list[tuple[Context, Any]] = []
 
             for topic, data in published.items():
                 log.debug(f"publishing {topic, data}")
@@ -103,8 +106,8 @@ async def fan_out_to_ctxs(
 
 def modify_subs(
 
-    topics2ctxs: Dict[str, List[Context]],
+    topics2ctxs: dict[str, list[Context]],
-    topics: Set[str],
+    topics: set[str],
     ctx: Context,
 
 ) -> None:
@@ -136,20 +139,20 @@ def modify_subs(
             topics2ctxs.pop(topic)
 
 
-_pub_state: Dict[str, dict] = {}
+_pub_state: dict[str, dict] = {}
-_pubtask2lock: Dict[str, trio.StrictFIFOLock] = {}
+_pubtask2lock: dict[str, trio.StrictFIFOLock] = {}
 
 
 def pub(
-    wrapped: typing.Callable = None,
+    wrapped: typing.Callable | None = None,
     *,
-    tasks: Set[str] = set(),
+    tasks: set[str] = set(),
 ):
     """Publisher async generator decorator.
 
     A publisher can be called multiple times from different actors but
     will only spawn a finite set of internal tasks to stream values to
-    each caller. The ``tasks: Set[str]`` argument to the decorator
+    each caller. The ``tasks: set[str]`` argument to the decorator
     specifies the names of the mutex set of publisher tasks.  When the
     publisher function is called, an argument ``task_name`` must be
     passed to specify which task (of the set named in ``tasks``) should
@@ -158,9 +161,9 @@ def pub(
     necessary.
 
     Values yielded from the decorated async generator must be
-    ``Dict[str, Dict[str, Any]]`` where the fist level key is the topic
+    ``dict[str, dict[str, Any]]`` where the fist level key is the topic
     string and determines which subscription the packet will be
-    delivered to and the value is a packet ``Dict[str, Any]`` by default
+    delivered to and the value is a packet ``dict[str, Any]`` by default
     of the form:
 
     .. ::python
@@ -186,7 +189,7 @@ def pub(
 
 
     The publisher must be called passing in the following arguments:
-    - ``topics: Set[str]`` the topic sequence or "subscriptions"
+    - ``topics: set[str]`` the topic sequence or "subscriptions"
     - ``task_name: str`` the task to use (if ``tasks`` was passed)
     - ``ctx: Context`` the tractor context (only needed if calling the
       pub func without a nursery, otherwise this is provided implicitly)
@@ -231,7 +234,7 @@ def pub(
     if wrapped is None:
         return partial(pub, tasks=tasks)
 
-    task2lock: Dict[str, trio.StrictFIFOLock] = {}
+    task2lock: dict[str, trio.StrictFIFOLock] = {}
 
     for name in tasks:
         task2lock[name] = trio.StrictFIFOLock()
@@ -243,11 +246,11 @@ def pub(
         # `wrapt` docs
         async def _execute(
             ctx: Context,
-            topics: Set[str],
+            topics: set[str],
             *args,
             # *,
-            task_name: str = None,  # default: only one task allocated
+            task_name: str | None = None,  # default: only one task allocated
-            packetizer: Callable = None,
+            packetizer: Callable | None = None,
             **kwargs,
         ):
             if task_name is None:

tractor/log.py

@@ -18,12 +18,14 @@
 Log like a forester!
 
 """
+from collections.abc import Mapping
 import sys
 import logging
 import colorlog  # type: ignore
-from typing import Optional
 
-from ._state import ActorContextInfo
+import trio
+
+from ._state import current_actor
 
 
 _proj_name: str = 'tractor'
@@ -36,7 +38,8 @@ LOG_FORMAT = (
     # "{bold_white}{log_color}{asctime}{reset}"
     "{log_color}{asctime}{reset}"
     " {bold_white}{thin_white}({reset}"
-    "{thin_white}{actor}, {process}, {task}){reset}{bold_white}{thin_white})"
+    "{thin_white}{actor_name}[{actor_uid}], "
+    "{process}, {task}){reset}{bold_white}{thin_white})"
     " {reset}{log_color}[{reset}{bold_log_color}{levelname}{reset}{log_color}]"
     " {log_color}{name}"
     " {thin_white}{filename}{log_color}:{reset}{thin_white}{lineno}{log_color}"
@@ -136,9 +139,40 @@ class StackLevelAdapter(logging.LoggerAdapter):
         )
 
 
+_conc_name_getters = {
+    'task': lambda: trio.lowlevel.current_task().name,
+    'actor': lambda: current_actor(),
+    'actor_name': lambda: current_actor().name,
+    'actor_uid': lambda: current_actor().uid[1][:6],
+}
+
+
+class ActorContextInfo(Mapping):
+    "Dyanmic lookup for local actor and task names"
+    _context_keys = (
+        'task',
+        'actor',
+        'actor_name',
+        'actor_uid',
+    )
+
+    def __len__(self):
+        return len(self._context_keys)
+
+    def __iter__(self):
+        return iter(self._context_keys)
+
+    def __getitem__(self, key: str) -> str:
+        try:
+            return _conc_name_getters[key]()
+        except RuntimeError:
+            # no local actor/task context initialized yet
+            return f'no {key} context'
+
+
 def get_logger(
 
-    name: str = None,
+    name: str | None = None,
     _root_name: str = _proj_name,
 
 ) -> StackLevelAdapter:
@@ -173,7 +207,7 @@ def get_logger(
 
 
 def get_console_log(
-    level: str = None,
+    level: str | None = None,
     **kwargs,
 ) -> logging.LoggerAdapter:
     '''Get the package logger and enable a handler which writes to stderr.

tractor/msg.py

@@ -24,7 +24,7 @@ Built-in messaging patterns, types, APIs and helpers.
 # ``pkgutil.resolve_name()`` internally uses
 # ``importlib.import_module()`` which can be filtered by inserting
 # a ``MetaPathFinder`` into ``sys.meta_path`` (which we could do before
-# entering the ``Actor._process_messages()`` loop).
+# entering the ``_runtime.process_messages()`` loop).
 # - https://github.com/python/cpython/blob/main/Lib/pkgutil.py#L645
 # - https://stackoverflow.com/questions/1350466/preventing-python-code-from-importing-certain-modules
 #   - https://stackoverflow.com/a/63320902

tractor/testing/__init__.py

@@ -1,17 +0,0 @@
-# tractor: structured concurrent "actors".
-# Copyright 2018-eternity Tyler Goodlet.
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-from ._tractor_test import tractor_test

tractor/testing/_tractor_test.py

@@ -1,104 +0,0 @@
-# tractor: structured concurrent "actors".
-# Copyright 2018-eternity Tyler Goodlet.
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-import inspect
-import platform
-from functools import partial, wraps
-
-import trio
-import tractor
-
-
-__all__ = ['tractor_test']
-
-
-def tractor_test(fn):
-    """
-    Use:
-
-    @tractor_test
-    async def test_whatever():
-        await ...
-
-    If fixtures:
-
-        - ``arb_addr`` (a socket addr tuple where arbiter is listening)
-        - ``loglevel`` (logging level passed to tractor internals)
-        - ``start_method`` (subprocess spawning backend)
-
-    are defined in the `pytest` fixture space they will be automatically
-    injected to tests declaring these funcargs.
-    """
-    @wraps(fn)
-    def wrapper(
-        *args,
-        loglevel=None,
-        arb_addr=None,
-        start_method=None,
-        **kwargs
-    ):
-        # __tracebackhide__ = True
-
-        if 'arb_addr' in inspect.signature(fn).parameters:
-            # injects test suite fixture value to test as well
-            # as `run()`
-            kwargs['arb_addr'] = arb_addr
-
-        if 'loglevel' in inspect.signature(fn).parameters:
-            # allows test suites to define a 'loglevel' fixture
-            # that activates the internal logging
-            kwargs['loglevel'] = loglevel
-
-        if start_method is None:
-            if platform.system() == "Windows":
-                start_method = 'spawn'
-            else:
-                start_method = 'trio'
-
-        if 'start_method' in inspect.signature(fn).parameters:
-            # set of subprocess spawning backends
-            kwargs['start_method'] = start_method
-
-        if kwargs:
-
-            # use explicit root actor start
-
-            async def _main():
-                async with tractor.open_root_actor(
-                    # **kwargs,
-                    arbiter_addr=arb_addr,
-                    loglevel=loglevel,
-                    start_method=start_method,
-
-                    # TODO: only enable when pytest is passed --pdb
-                    # debug_mode=True,
-
-                ) as actor:
-                    await fn(*args, **kwargs)
-
-            main = _main
-
-        else:
-            # use implicit root actor start
-            main = partial(fn, *args, **kwargs)
-
-        return trio.run(main)
-            # arbiter_addr=arb_addr,
-            # loglevel=loglevel,
-            # start_method=start_method,
-        # )
-
-    return wrapper

tractor/to_asyncio.py

@@ -32,10 +32,15 @@ from typing import (
 )
 
 import trio
+from outcome import Error
 
 from .log import get_logger
 from ._state import current_actor
 from ._exceptions import AsyncioCancelled
+from .trionics._broadcast import (
+    broadcast_receiver,
+    BroadcastReceiver,
+)
 
 log = get_logger(__name__)
 
@@ -57,19 +62,35 @@ class LinkedTaskChannel(trio.abc.Channel):
 
     _trio_cs: trio.CancelScope
     _aio_task_complete: trio.Event
+    _trio_exited: bool = False
 
     # set after ``asyncio.create_task()``
     _aio_task: Optional[asyncio.Task] = None
     _aio_err: Optional[BaseException] = None
+    _broadcaster: Optional[BroadcastReceiver] = None
 
     async def aclose(self) -> None:
         await self._from_aio.aclose()
 
     async def receive(self) -> Any:
-        async with translate_aio_errors(self):
+        async with translate_aio_errors(
+            self,
+
+            # XXX: obviously this will deadlock if an on-going stream is
+            # being procesed.
+            # wait_on_aio_task=False,
+        ):
+
+            # TODO: do we need this to guarantee asyncio code get's
+            # cancelled in the case where the trio side somehow creates
+            # a state where the asyncio cycle-task isn't getting the
+            # cancel request sent by (in theory) the last checkpoint
+            # cycle on the trio side?
+            # await trio.lowlevel.checkpoint()
+
             return await self._from_aio.receive()
 
-    async def wait_ayncio_complete(self) -> None:
+    async def wait_asyncio_complete(self) -> None:
         await self._aio_task_complete.wait()
 
     # def cancel_asyncio_task(self) -> None:
@@ -84,6 +105,43 @@ class LinkedTaskChannel(trio.abc.Channel):
         '''
         self._to_aio.put_nowait(item)
 
+    def closed(self) -> bool:
+        return self._from_aio._closed  # type: ignore
+
+    # TODO: shoud we consider some kind of "decorator" system
+    # that checks for structural-typing compatibliity and then
+    # automatically adds this ctx-mngr-as-method machinery?
+    @acm
+    async def subscribe(
+        self,
+
+    ) -> AsyncIterator[BroadcastReceiver]:
+        '''
+        Allocate and return a ``BroadcastReceiver`` which delegates
+        to this inter-task channel.
+
+        This allows multiple local tasks to receive each their own copy
+        of this message stream.
+
+        See ``tractor._streaming.MsgStream.subscribe()`` for further
+        similar details.
+        '''
+        if self._broadcaster is None:
+
+            bcast = self._broadcaster = broadcast_receiver(
+                self,
+                # use memory channel size by default
+                self._from_aio._state.max_buffer_size,  # type: ignore
+                receive_afunc=self.receive,
+            )
+
+            self.receive = bcast.receive  # type: ignore
+
+        async with self._broadcaster.subscribe() as bstream:
+            assert bstream.key != self._broadcaster.key
+            assert bstream._recv == self._broadcaster._recv
+            yield bstream
+
 
 def _run_asyncio_task(
 
@@ -99,6 +157,7 @@ def _run_asyncio_task(
     or stream the result back to ``trio``.
 
     '''
+    __tracebackhide__ = True
     if not current_actor().is_infected_aio():
         raise RuntimeError("`infect_asyncio` mode is not enabled!?")
 
@@ -158,6 +217,7 @@ def _run_asyncio_task(
         try:
             result = await coro
         except BaseException as aio_err:
+            log.exception('asyncio task errored')
             chan._aio_err = aio_err
             raise
 
@@ -181,6 +241,7 @@ def _run_asyncio_task(
                 to_trio.close()
 
             aio_task_complete.set()
+            log.runtime(f'`asyncio` task: {task.get_name()} is complete')
 
     # start the asyncio task we submitted from trio
     if not inspect.isawaitable(coro):
@@ -202,20 +263,25 @@ def _run_asyncio_task(
         '''
         nonlocal chan
         aio_err = chan._aio_err
+        task_err: Optional[BaseException] = None
 
         # only to avoid ``asyncio`` complaining about uncaptured
         # task exceptions
         try:
             task.exception()
         except BaseException as terr:
+            task_err = terr
+
+            if isinstance(terr, CancelledError):
+                log.cancel(f'`asyncio` task cancelled: {task.get_name()}')
+            else:
+                log.exception(f'`asyncio` task: {task.get_name()} errored')
+
             assert type(terr) is type(aio_err), 'Asyncio task error mismatch?'
 
         if aio_err is not None:
-            if type(aio_err) is CancelledError:
+            # XXX: uhh is this true?
-                log.cancel("infected task was cancelled")
+            # assert task_err, f'Asyncio task {task.get_name()} discrepancy!?'
-            else:
-                aio_err.with_traceback(aio_err.__traceback__)
-                log.exception("infected task errorred:")
 
             # NOTE: currently mem chan closure may act as a form
             # of error relay (at least in the ``asyncio.CancelledError``
@@ -224,8 +290,28 @@ def _run_asyncio_task(
             # We might want to change this in the future though.
             from_aio.close()
 
-    task.add_done_callback(cancel_trio)
+            if type(aio_err) is CancelledError:
+                log.cancel("infected task was cancelled")
 
+                # TODO: show that the cancellation originated
+                # from the ``trio`` side? right?
+                # if cancel_scope.cancelled:
+                #     raise aio_err from err
+
+            elif task_err is None:
+                assert aio_err
+                aio_err.with_traceback(aio_err.__traceback__)
+                log.error('infected task errorred')
+
+            # XXX: alway cancel the scope on error
+            # in case the trio task is blocking
+            # on a checkpoint.
+            cancel_scope.cancel()
+
+            # raise any ``asyncio`` side error.
+            raise aio_err
+
+    task.add_done_callback(cancel_trio)
     return chan
 
 
@@ -233,6 +319,7 @@ def _run_asyncio_task(
 async def translate_aio_errors(
 
     chan: LinkedTaskChannel,
+    wait_on_aio_task: bool = False,
 
 ) -> AsyncIterator[None]:
     '''
@@ -240,8 +327,11 @@ async def translate_aio_errors(
     appropriately translates errors and cancels into ``trio`` land.
 
     '''
+    trio_task = trio.lowlevel.current_task()
+
     aio_err: Optional[BaseException] = None
 
+    # TODO: make thisi a channel method?
     def maybe_raise_aio_err(
         err: Optional[Exception] = None
     ) -> None:
@@ -260,10 +350,22 @@ async def translate_aio_errors(
     assert task
     try:
         yield
+
+    except (
+        trio.Cancelled,
+    ):
+        # relay cancel through to called ``asyncio`` task
+        assert chan._aio_task
+        chan._aio_task.cancel(
+            msg=f'the `trio` caller task was cancelled: {trio_task.name}'
+        )
+        raise
+
     except (
         # NOTE: see the note in the ``cancel_trio()`` asyncio task
         # termination callback
         trio.ClosedResourceError,
+        # trio.BrokenResourceError,
     ):
         aio_err = chan._aio_err
         if (
@@ -277,14 +379,32 @@ async def translate_aio_errors(
 
         else:
             raise
+
     finally:
-        # always cancel the ``asyncio`` task if we've made it this far
+        if (
-        # and it's not done.
+            # NOTE: always cancel the ``asyncio`` task if we've made it
-        if not task.done() and aio_err:
+            # this far and it's not done.
+            not task.done() and aio_err
+
+            # or the trio side has exited it's surrounding cancel scope
+            # indicating the lifetime of the ``asyncio``-side task
+            # should also be terminated.
+            or chan._trio_exited
+        ):
+            log.runtime(
+                f'Cancelling `asyncio`-task: {task.get_name()}'
+            )
             # assert not aio_err, 'WTF how did asyncio do this?!'
             task.cancel()
 
-        # if any ``asyncio`` error was caught, raise it here inline
+        # Required to sync with the far end ``asyncio``-task to ensure
+        # any error is captured (via monkeypatching the
+        # ``channel._aio_err``) before calling ``maybe_raise_aio_err()``
+        # below!
+        if wait_on_aio_task:
+            await chan._aio_task_complete.wait()
+
+        # NOTE: if any ``asyncio`` error was caught, raise it here inline
         # here in the ``trio`` task
         maybe_raise_aio_err()
 
@@ -309,8 +429,10 @@ async def run_task(
         **kwargs,
     )
     with chan._from_aio:
-        # try:
+        async with translate_aio_errors(
-        async with translate_aio_errors(chan):
+            chan,
+            wait_on_aio_task=True,
+        ):
             # return single value that is the output from the
             # ``asyncio`` function-as-task. Expect the mem chan api to
             # do the job of handling cross-framework cancellations
@@ -338,13 +460,21 @@ async def open_channel_from(
         **kwargs,
     )
     async with chan._from_aio:
-        async with translate_aio_errors(chan):
+        async with translate_aio_errors(
+            chan,
+            wait_on_aio_task=True,
+        ):
             # sync to a "started()"-like first delivered value from the
             # ``asyncio`` task.
+            try:
+                with chan._trio_cs:
                     first = await chan.receive()
 
-            # stream values upward
+                    # deliver stream handle upward
                     yield first, chan
+            finally:
+                chan._trio_exited = True
+                chan._to_trio.close()
 
 
 def run_as_asyncio_guest(
@@ -361,16 +491,18 @@ def run_as_asyncio_guest(
     SC semantics.
 
     '''
-    # Uh, oh. :o
+    # Uh, oh.
+    #
+    # :o
 
     # It looks like your event loop has caught a case of the ``trio``s.
 
     # :()
 
-    # Don't worry, we've heard you'll barely notice. You might hallucinate
+    # Don't worry, we've heard you'll barely notice. You might
-    # a few more propagating errors and feel like your digestion has
+    # hallucinate a few more propagating errors and feel like your
-    # slowed but if anything get's too bad your parents will know about
+    # digestion has slowed but if anything get's too bad your parents
-    # it.
+    # will know about it.
 
     # :)
 
@@ -381,8 +513,20 @@ def run_as_asyncio_guest(
 
         def trio_done_callback(main_outcome):
 
-            print(f"trio_main finished: {main_outcome!r}")
+            if isinstance(main_outcome, Error):
+                error = main_outcome.error
+                trio_done_fut.set_exception(error)
+
+                # TODO: explicit asyncio tb?
+                # traceback.print_exception(error)
+
+                # XXX: do we need this?
+                # actor.cancel_soon()
+
+                main_outcome.unwrap()
+            else:
                 trio_done_fut.set_result(main_outcome)
+                log.runtime(f"trio_main finished: {main_outcome!r}")
 
         # start the infection: run trio on the asyncio loop in "guest mode"
         log.info(f"Infecting asyncio process with {trio_main}")
@@ -392,6 +536,7 @@ def run_as_asyncio_guest(
             run_sync_soon_threadsafe=loop.call_soon_threadsafe,
             done_callback=trio_done_callback,
         )
+        # ``.unwrap()`` will raise here on error
         return (await trio_done_fut).unwrap()
 
     # might as well if it's installed.

tractor/trionics/__init__.py

@@ -21,6 +21,7 @@ Sugary patterns for trio + tractor designs.
 from ._mngrs import (
     gather_contexts,
     maybe_open_context,
+    maybe_open_nursery,
 )
 from ._broadcast import (
     broadcast_receiver,
@@ -35,4 +36,5 @@ __all__ = [
     'BroadcastReceiver',
     'Lagged',
     'maybe_open_context',
+    'maybe_open_nursery',
 ]

tractor/trionics/_broadcast.py

@@ -23,7 +23,6 @@ from __future__ import annotations
 from abc import abstractmethod
 from collections import deque
 from contextlib import asynccontextmanager
-from dataclasses import dataclass
 from functools import partial
 from operator import ne
 from typing import Optional, Callable, Awaitable, Any, AsyncIterator, Protocol
@@ -33,7 +32,10 @@ import trio
 from trio._core._run import Task
 from trio.abc import ReceiveChannel
 from trio.lowlevel import current_task
+from msgspec import Struct
+from tractor.log import get_logger
 
+log = get_logger(__name__)
 
 # A regular invariant generic type
 T = TypeVar("T")
@@ -86,8 +88,7 @@ class Lagged(trio.TooSlowError):
     '''
 
 
-@dataclass
+class BroadcastState(Struct):
-class BroadcastState:
     '''
     Common state to all receivers of a broadcast.
 
@@ -110,7 +111,35 @@ class BroadcastState:
     eoc: bool = False
 
     # If the broadcaster was cancelled, we might as well track it
-    cancelled: bool = False
+    cancelled: dict[int, Task] = {}
+
+    def statistics(self) -> dict[str, Any]:
+        '''
+        Return broadcast receiver group "statistics" like many of
+        ``trio``'s internal task-sync primitives.
+
+        '''
+        key: int | None
+        ev: trio.Event | None
+
+        subs = self.subs
+        if self.recv_ready is not None:
+            key, ev = self.recv_ready
+        else:
+            key = ev = None
+
+        qlens: dict[int, int] = {}
+        for tid, sz in subs.items():
+            qlens[tid] = sz if sz != -1 else 0
+
+        return {
+            'open_consumers': len(subs),
+            'queued_len_by_task': qlens,
+            'max_buffer_size': self.maxlen,
+            'tasks_waiting': ev.statistics().tasks_waiting if ev else 0,
+            'tasks_cancelled': self.cancelled,
+            'next_value_receiver_id': key,
+        }
 
 
 class BroadcastReceiver(ReceiveChannel):
@@ -128,23 +157,40 @@ class BroadcastReceiver(ReceiveChannel):
         rx_chan: AsyncReceiver,
         state: BroadcastState,
         receive_afunc: Optional[Callable[[], Awaitable[Any]]] = None,
+        raise_on_lag: bool = True,
 
     ) -> None:
 
         # register the original underlying (clone)
         self.key = id(self)
         self._state = state
+
+        # each consumer has an int count which indicates
+        # which index contains the next value that the task has not yet
+        # consumed and thus should read. In the "up-to-date" case the
+        # consumer task must wait for a new value from the underlying
+        # receiver and we use ``-1`` as the sentinel for this state.
         state.subs[self.key] = -1
 
         # underlying for this receiver
         self._rx = rx_chan
         self._recv = receive_afunc or rx_chan.receive
         self._closed: bool = False
+        self._raise_on_lag = raise_on_lag
 
-    async def receive(self) -> ReceiveType:
+    def receive_nowait(
+        self,
+        _key: int | None = None,
+        _state: BroadcastState | None = None,
 
-        key = self.key
+    ) -> Any:
-        state = self._state
+        '''
+        Sync version of `.receive()` which does all the low level work
+        of receiving from the underlying/wrapped receive channel.
+
+        '''
+        key = _key or self.key
+        state = _state or self._state
 
         # TODO: ideally we can make some way to "lock out" the
         # underlying receive channel in some way such that if some task
@@ -177,32 +223,47 @@ class BroadcastReceiver(ReceiveChannel):
                 # return this value."
                 # https://docs.rs/tokio/1.11.0/tokio/sync/broadcast/index.html#lagging
 
+                mxln = state.maxlen
+                lost = seq - mxln
+
                 # decrement to the last value and expect
                 # consumer to either handle the ``Lagged`` and come back
                 # or bail out on its own (thus un-subscribing)
-                state.subs[key] = state.maxlen - 1
+                state.subs[key] = mxln - 1
 
                 # this task was overrun by the producer side
                 task: Task = current_task()
-                raise Lagged(f'Task {task.name} was overrun')
+                msg = f'Task `{task.name}` overrun and dropped `{lost}` values'
+
+                if self._raise_on_lag:
+                    raise Lagged(msg)
+                else:
+                    log.warning(msg)
+                    return self.receive_nowait(_key, _state)
 
             state.subs[key] -= 1
             return value
 
-        # current task already has the latest value **and** is the
+        raise trio.WouldBlock
-        # first task to begin waiting for a new one
+
-        if state.recv_ready is None:
+    async def _receive_from_underlying(
+        self,
+        key: int,
+        state: BroadcastState,
+
+    ) -> ReceiveType:
 
         if self._closed:
             raise trio.ClosedResourceError
 
         event = trio.Event()
+        assert state.recv_ready is None
         state.recv_ready = key, event
 
+        try:
             # if we're cancelled here it should be
             # fine to bail without affecting any other consumers
             # right?
-            try:
             value = await self._recv()
 
             # items with lower indices are "newer"
@@ -220,7 +281,6 @@ class BroadcastReceiver(ReceiveChannel):
             # already retreived the last value
 
             # XXX: which of these impls is fastest?
-
             # subs = state.subs.copy()
             # subs.pop(key)
 
@@ -251,54 +311,85 @@ class BroadcastReceiver(ReceiveChannel):
             # consumers will be awoken with a sequence of -1
             # and will potentially try to rewait the underlying
             # receiver instead of just cancelling immediately.
-                self._state.cancelled = True
+            self._state.cancelled[key] = current_task()
             if event.statistics().tasks_waiting:
                 event.set()
             raise
 
         finally:
-
             # Reset receiver waiter task event for next blocking condition.
             # this MUST be reset even if the above ``.recv()`` call
             # was cancelled to avoid the next consumer from blocking on
             # an event that won't be set!
             state.recv_ready = None
 
+    async def receive(self) -> ReceiveType:
+        key = self.key
+        state = self._state
+
+        try:
+            return self.receive_nowait(
+                _key=key,
+                _state=state,
+            )
+        except trio.WouldBlock:
+            pass
+
+        # current task already has the latest value **and** is the
+        # first task to begin waiting for a new one so we begin blocking
+        # until rescheduled with the a new value from the underlying.
+        if state.recv_ready is None:
+            return await self._receive_from_underlying(key, state)
+
         # This task is all caught up and ready to receive the latest
-        # value, so queue sched it on the internal event.
+        # value, so queue/schedule it to be woken on the next internal
+        # event.
         else:
-            seq = state.subs[key]
+            while state.recv_ready is not None:
-            assert seq == -1  # sanity
+                # seq = state.subs[key]
+                # assert seq == -1  # sanity
                 _, ev = state.recv_ready
                 await ev.wait()
+                try:
+                    return self.receive_nowait(
+                        _key=key,
+                        _state=state,
+                    )
+                except trio.WouldBlock:
+                    if self._closed:
+                        raise trio.ClosedResourceError
 
-            # NOTE: if we ever would like the behaviour where if the
+                    subs = state.subs
-            # first task to recv on the underlying is cancelled but it
+                    if (
-            # still DOES trigger the ``.recv_ready``, event we'll likely need
+                        len(subs) == 1
-            # this logic:
+                        and key in subs
+                        # or cancelled
+                    ):
+                        # XXX: we are the last and only user of this BR so
+                        # likely it makes sense to unwind back to the
+                        # underlying?
+                        # import tractor
+                        # await tractor.breakpoint()
+                        log.warning(
+                            f'Only one sub left for {self}?\n'
+                            'We can probably unwind from breceiver?'
+                        )
 
-            if seq > -1:
-                # stuff from above..
-                seq = state.subs[key]
-
-                value = state.queue[seq]
-                state.subs[key] -= 1
-                return value
-
-            elif seq == -1:
                     # XXX: In the case where the first task to allocate the
-                # ``.recv_ready`` event is cancelled we will be woken with
+                    # ``.recv_ready`` event is cancelled we will be woken
-                # a non-incremented sequence number and thus will read the
+                    # with a non-incremented sequence number (the ``-1``
-                # oldest value if we use that. Instead we need to detect if
+                    # sentinel) and thus will read the oldest value if we
-                # we have not been incremented and then receive again.
+                    # use that. Instead we need to detect if we have not
-                return await self.receive()
+                    # been incremented and then receive again.
+                    # return await self.receive()
 
-            else:
+            return await self._receive_from_underlying(key, state)
-                raise ValueError(f'Invalid sequence {seq}!?')
 
     @asynccontextmanager
     async def subscribe(
         self,
+        raise_on_lag: bool = True,
+
     ) -> AsyncIterator[BroadcastReceiver]:
         '''
         Subscribe for values from this broadcast receiver.
@@ -316,6 +407,7 @@ class BroadcastReceiver(ReceiveChannel):
             rx_chan=self._rx,
             state=state,
             receive_afunc=self._recv,
+            raise_on_lag=raise_on_lag,
         )
         # assert clone in state.subs
         assert br.key in state.subs
@@ -352,7 +444,8 @@ def broadcast_receiver(
 
     recv_chan: AsyncReceiver,
     max_buffer_size: int,
-    **kwargs,
+    receive_afunc: Optional[Callable[[], Awaitable[Any]]] = None,
+    raise_on_lag: bool = True,
 
 ) -> BroadcastReceiver:
 
@@ -363,5 +456,6 @@ def broadcast_receiver(
             maxlen=max_buffer_size,
             subs={},
         ),
-        **kwargs,
+        receive_afunc=receive_afunc,
+        raise_on_lag=raise_on_lag,
     )

tractor/trionics/_mngrs.py

@@ -19,6 +19,7 @@ Async context manager primitives with hard ``trio``-aware semantics
 
 '''
 from contextlib import asynccontextmanager as acm
+import inspect
 from typing import (
     Any,
     AsyncContextManager,
@@ -34,8 +35,8 @@ from typing import (
 import trio
 from trio_typing import TaskStatus
 
-from ..log import get_logger
 from .._state import current_actor
+from ..log import get_logger
 
 
 log = get_logger(__name__)
@@ -44,6 +45,25 @@ log = get_logger(__name__)
 T = TypeVar("T")
 
 
+@acm
+async def maybe_open_nursery(
+    nursery: trio.Nursery | None = None,
+    shield: bool = False,
+) -> AsyncGenerator[trio.Nursery, Any]:
+    '''
+    Create a new nursery if None provided.
+
+    Blocks on exit as expected if no input nursery is provided.
+
+    '''
+    if nursery is not None:
+        yield nursery
+    else:
+        async with trio.open_nursery() as nursery:
+            nursery.cancel_scope.shield = shield
+            yield nursery
+
+
 async def _enter_and_wait(
 
     mngr: AsyncContextManager[T],
@@ -81,7 +101,7 @@ async def gather_contexts(
     This function is somewhat similar to common usage of
     ``contextlib.AsyncExitStack.enter_async_context()`` (in a loop) in
     combo with ``asyncio.gather()`` except the managers are concurrently
-    entered and exited cancellation just works.
+    entered and exited, and cancellation just works.
 
     '''
     unwrapped: dict[int, Optional[T]] = {}.fromkeys(id(mngr) for mngr in mngrs)
@@ -89,6 +109,17 @@ async def gather_contexts(
     all_entered = trio.Event()
     parent_exit = trio.Event()
 
+    # XXX: ensure greedy sequence of manager instances
+    # since a lazy inline generator doesn't seem to work
+    # with `async with` syntax.
+    mngrs = list(mngrs)
+
+    if not mngrs:
+        raise ValueError(
+            'input mngrs is empty?\n'
+            'Did try to use inline generator syntax?'
+        )
+
     async with trio.open_nursery() as n:
         for mngr in mngrs:
             n.start_soon(
@@ -102,10 +133,12 @@ async def gather_contexts(
         # deliver control once all managers have started up
         await all_entered.wait()
 
+        try:
             yield tuple(unwrapped.values())
-
+        finally:
-        # we don't need a try/finally since cancellation will be triggered
+            # NOTE: this is ABSOLUTELY REQUIRED to avoid
-        # by the surrounding nursery on error.
+            # the following wacky bug:
+            # <tractorbugurlhere>
             parent_exit.set()
 
 
@@ -119,13 +152,15 @@ class _Cache:
     a kept-alive-while-in-use async resource.
 
     '''
-    lock = trio.Lock()
+    service_n: Optional[trio.Nursery] = None
+    locks: dict[Hashable, trio.Lock] = {}
     users: int = 0
     values: dict[Any,  Any] = {}
     resources: dict[
         Hashable,
         tuple[trio.Nursery, trio.Event]
     ] = {}
+    # nurseries: dict[int, trio.Nursery] = {}
     no_more_users: Optional[trio.Event] = None
 
     @classmethod
@@ -156,7 +191,7 @@ async def maybe_open_context(
     # XXX: used as cache key after conversion to tuple
     # and all embedded values must also be hashable
     kwargs: dict = {},
-    key: Hashable = None,
+    key: Hashable | Callable[..., Hashable] = None,
 
 ) -> AsyncIterator[tuple[bool, T]]:
     '''
@@ -165,51 +200,69 @@ async def maybe_open_context(
     _Cached instance on a _Cache hit.
 
     '''
-    # lock resource acquisition around task racing  / ``trio``'s
+    fid = id(acm_func)
-    # scheduler protocol
-    await _Cache.lock.acquire()
 
-    ctx_key = (id(acm_func), key or tuple(kwargs.items()))
+    if inspect.isfunction(key):
-    value = None
+        ctx_key = (fid, key(**kwargs))
+    else:
+        ctx_key = (fid, key or tuple(kwargs.items()))
+
+    # yielded output
+    yielded: Any = None
+
+    # Lock resource acquisition around task racing  / ``trio``'s
+    # scheduler protocol.
+    # NOTE: the lock is target context manager func specific in order
+    # to allow re-entrant use cases where one `maybe_open_context()`
+    # wrapped factor may want to call into another.
+    lock = _Cache.locks.setdefault(fid, trio.Lock())
+    await lock.acquire()
+
+    # XXX: one singleton nursery per actor and we want to
+    # have it not be closed until all consumers have exited (which is
+    # currently difficult to implement any other way besides using our
+    # pre-allocated runtime instance..)
+    service_n: trio.Nursery = current_actor()._service_n
+
+    # TODO: is there any way to allocate
+    # a 'stays-open-till-last-task-finshed nursery?
+    # service_n: trio.Nursery
+    # async with maybe_open_nursery(_Cache.service_n) as service_n:
+    #     _Cache.service_n = service_n
 
     try:
         # **critical section** that should prevent other tasks from
         # checking the _Cache until complete otherwise the scheduler
         # may switch and by accident we create more then one resource.
-        value = _Cache.values[ctx_key]
+        yielded = _Cache.values[ctx_key]
 
     except KeyError:
         log.info(f'Allocating new {acm_func} for {ctx_key}')
-
         mngr = acm_func(**kwargs)
-        # TODO: avoid pulling from ``tractor`` internals and
-        # instead offer a "root nursery" in piker actors?
-        service_n = current_actor()._service_n
-
-        # TODO: does this need to be a tractor "root nursery"?
         resources = _Cache.resources
         assert not resources.get(ctx_key), f'Resource exists? {ctx_key}'
-        ln, _ = resources[ctx_key] = (service_n, trio.Event())
+        resources[ctx_key] = (service_n, trio.Event())
 
-        value = await ln.start(
+        # sync up to the mngr's yielded value
+        yielded = await service_n.start(
             _Cache.run_ctx,
             mngr,
             ctx_key,
         )
         _Cache.users += 1
-        _Cache.lock.release()
+        lock.release()
-        yield False, value
+        yield False, yielded
 
     else:
         log.info(f'Reusing _Cached resource for {ctx_key}')
         _Cache.users += 1
-        _Cache.lock.release()
+        lock.release()
-        yield True, value
+        yield True, yielded
 
     finally:
         _Cache.users -= 1
 
-        if value is not None:
+        if yielded is not None:
             # if no more consumers, teardown the client
             if _Cache.users <= 0:
                 log.info(f'De-allocating resource for {ctx_key}')
@@ -221,3 +274,5 @@ async def maybe_open_context(
                 if entry:
                     _, no_more_users = entry
                     no_more_users.set()
+
+                _Cache.locks.pop(fid)