Merge pull request #358 from goodboy/switch_to_pdbp

Switch to `pdbp` 🏄🏼

.github/workflows/ci.yml

@@ -0,0 +1,131 @@
+name: CI
+
+on:
+  # any time someone pushes a new branch to origin
+  push:
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+
+  mypy:
+    name: 'MyPy'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install -U . --upgrade-strategy eager -r requirements-test.txt
+
+      - name: Run MyPy check
+        run: mypy tractor/ --ignore-missing-imports --show-traceback
+
+  # test that we can generate a software distribution and install it
+  # thus avoid missing file issues after packaging.
+  sdist-linux:
+    name: 'sdist'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      - name: Build sdist
+        run: python setup.py sdist --formats=zip
+
+      - name: Install sdist from .zips
+        run: python -m pip install dist/*.zip
+
+
+  testing-linux:
+    name: '${{ matrix.os }} Python ${{ matrix.python }} - ${{ matrix.spawn_backend }}'
+    timeout-minutes: 10
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python: ['3.10']
+        spawn_backend: [
+          'trio',
+          'mp_spawn',
+          'mp_forkserver',
+        ]
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '${{ matrix.python }}'
+
+      - name: Install dependencies
+        run: pip install -U . -r requirements-test.txt -r requirements-docs.txt --upgrade-strategy eager
+
+      - name: List dependencies
+        run: pip list
+
+      - name: Run tests
+        run: pytest tests/ --spawn-backend=${{ matrix.spawn_backend }} -rsx
+
+  # We skip 3.10 on windows for now due to not having any collabs to
+  # debug the CI failures. Anyone wanting to hack and solve them is very
+  # welcome, but our primary user base is not using that OS.
+
+  # TODO: use job filtering to accomplish instead of repeated
+  # boilerplate as is above XD:
+  # - https://docs.github.com/en/actions/learn-github-actions/managing-complex-workflows
+  # - https://docs.github.com/en/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
+  # - https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idif
+  # testing-windows:
+  #   name: '${{ matrix.os }} Python ${{ matrix.python }} - ${{ matrix.spawn_backend }}'
+  #   timeout-minutes: 12
+  #   runs-on: ${{ matrix.os }}
+
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       os: [windows-latest]
+  #       python: ['3.10']
+  #       spawn_backend: ['trio', 'mp']
+
+  #   steps:
+
+  #     - name: Checkout
+  #       uses: actions/checkout@v2
+
+  #     - name: Setup python
+  #       uses: actions/setup-python@v2
+  #       with:
+  #         python-version: '${{ matrix.python }}'
+
+  #     - name: Install dependencies
+  #       run: pip install -U . -r requirements-test.txt -r requirements-docs.txt --upgrade-strategy eager
+
+  #     # TODO: pretty sure this solves debugger deps-issues on windows, but it needs to
+  #     # be verified by someone with a native setup.
+  #     # - name: Force pyreadline3
+  #     #   run: pip uninstall pyreadline; pip install -U pyreadline3
+
+  #     - name: List dependencies
+  #       run: pip list
+
+  #     - name: Run tests
+  #       run: pytest tests/ --spawn-backend=${{ matrix.spawn_backend }} -rsx

.travis.yml

@@ -1,16 +0,0 @@
-language: python
-
-matrix:
-    include:
-        - python: 3.6
-        - python: 3.7
-          dist: xenial
-          sudo: required
-
-install:
-    - cd $TRAVIS_BUILD_DIR
-    - pip install -U . -r requirements-test.txt
-
-script:
-    - mypy tractor/  --ignore-missing-imports
-    - pytest tests/ --no-print-logs

LICENSE

@@ -1,23 +1,21 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
                             Preamble
 
-  The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
 
   The licenses for most software and other practical works are designed
 to take away your freedom to share and change the works.  By contrast,
-the GNU General Public License is intended to guarantee your freedom to
+our General Public Licenses are intended to guarantee your freedom to
 share and change all versions of a program--to make sure it remains free
-software for all its users.  We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors.  You can apply it to
-your programs, too.
+software for all its users.
 
   When we speak of free software, we are referring to freedom, not
 price.  Our General Public Licenses are designed to make sure that you
@@ -26,44 +24,34 @@ them if you wish), that you receive source code or can get it if you
 want it, that you can change the software or use pieces of it in new
 free programs, and that you know you can do these things.
 
-  To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights.  Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
 
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received.  You must make sure that they, too, receive
-or can get the source code.  And you must show them these terms so they
-know their rights.
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
 
-  Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
 
-  For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software.  For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-  Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so.  This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software.  The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable.  Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products.  If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
-  Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary.  To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
 
   The precise terms and conditions for copying, distribution and
 modification follow.
@@ -72,7 +60,7 @@ modification follow.
 
   0. Definitions.
 
-  "This License" refers to version 3 of the GNU General Public License.
+  "This License" refers to version 3 of the GNU Affero General Public License.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
@@ -549,35 +537,45 @@ to collect a royalty for further conveying from those to whom you convey
 the Program, the only way you could satisfy both those terms and this
 License would be to refrain entirely from conveying the Program.
 
-  13. Use with the GNU Affero General Public License.
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
 
   Notwithstanding any other provision of this License, you have
 permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
+under version 3 of the GNU General Public License into a single
 combined work, and to convey the resulting work.  The terms of this
 License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
 
   14. Revised Versions of this License.
 
   The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
 address new problems or concerns.
 
   Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
+Program specifies that a certain numbered version of the GNU Affero General
 Public License "or any later version" applies to it, you have the
 option of following the terms and conditions either of that numbered
 version or of any later version published by the Free Software
 Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
+GNU Affero General Public License, you may choose any version ever published
 by the Free Software Foundation.
 
   If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
+versions of the GNU Affero General Public License can be used, that proxy's
 public statement of acceptance of a version permanently authorizes you
 to choose that version for the Program.
 
@@ -635,40 +633,29 @@ the "copyright" line and a pointer to where the full notice is found.
     Copyright (C) <year>  <name of author>
 
     This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
+    it under the terms of the GNU Affero General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
     (at your option) any later version.
 
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
+    GNU Affero General Public License for more details.
 
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
 
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

MANIFEST.in

@@ -0,0 +1,2 @@
+# https://packaging.python.org/en/latest/guides/using-manifest-in/#using-manifest-in
+include docs/README.rst

NEWS.rst

@@ -0,0 +1,528 @@
+=========
+Changelog
+=========
+
+.. towncrier release notes start
+
+tractor 0.1.0a5 (2022-08-03)
+============================
+
+This is our final release supporting Python 3.9 since we will be moving
+internals to the new `match:` syntax from 3.10 going forward and
+further, we have officially dropped usage of the `msgpack` library and
+happily adopted `msgspec`.
+
+Features
+--------
+
+- `#165 <https://github.com/goodboy/tractor/issues/165>`_: Add SIGINT
+  protection to our `pdbpp` based debugger subystem such that for
+  (single-depth) actor trees in debug mode we ignore interrupts in any
+  actor currently holding the TTY lock thus avoiding clobbering IPC
+  connections and/or task and process state when working in the REPL.
+
+  As a big note currently so called "nested" actor trees (trees with
+  actors having more then one parent/ancestor) are not fully supported
+  since we don't yet have a mechanism to relay the debug mode knowledge
+  "up" the actor tree (for eg. when handling a crash in a leaf actor).
+  As such currently there is a set of tests and known scenarios which will
+  result in process cloberring by the zombie repaing machinery and these
+  have been documented in https://github.com/goodboy/tractor/issues/320.
+
+  The implementation details include:
+
+  - utilizing a custom SIGINT handler which we apply whenever an actor's
+    runtime enters the debug machinery, which we also make sure the
+    stdlib's `pdb` configuration doesn't override (which it does by
+    default without special instance config).
+  - litter the runtime with `maybe_wait_for_debugger()` mostly in spots
+    where the root actor should block before doing embedded nursery
+    teardown ops which both cancel potential-children-in-deubg as well
+    as eventually trigger zombie reaping machinery.
+  - hardening of the TTY locking semantics/API both in terms of IPC
+    terminations and cancellation and lock release determinism from
+    sync debugger instance methods.
+  - factoring of locking infrastructure into a new `._debug.Lock` global
+    which encapsulates all details of the ``trio`` sync primitives and
+    task/actor uid management and tracking.
+
+  We also add `ctrl-c` cases throughout the test suite though these are
+  disabled for py3.9 (`pdbpp` UX differences that don't seem worth
+  compensating for, especially since this will be our last 3.9 supported
+  release) and there are a slew of marked cases that aren't expected to
+  work in CI more generally (as mentioned in the "nested" tree note
+  above) despite seemingly working  when run manually on linux.
+
+- `#304 <https://github.com/goodboy/tractor/issues/304>`_: Add a new
+  ``to_asyncio.LinkedTaskChannel.subscribe()`` which gives task-oriented
+  broadcast functionality semantically equivalent to
+  ``tractor.MsgStream.subscribe()`` this makes it possible for multiple
+  ``trio``-side tasks to consume ``asyncio``-side task msgs in tandem.
+
+  Further Improvements to the test suite were added in this patch set
+  including a new scenario test for a sub-actor managed "service nursery"
+  (implementing the basics of a "service manager") including use of
+  *infected asyncio* mode. Further we added a lower level
+  ``test_trioisms.py`` to start to track issues we need to work around in
+  ``trio`` itself which in this case included a bug we were trying to
+  solve related to https://github.com/python-trio/trio/issues/2258.
+
+
+Bug Fixes
+---------
+
+- `#318 <https://github.com/goodboy/tractor/issues/318>`_: Fix
+  a previously undetected ``trio``-``asyncio`` task lifetime linking
+  issue with the ``to_asyncio.open_channel_from()`` api where both sides
+  where not properly waiting/signalling termination and it was possible
+  for ``asyncio``-side errors to not propagate due to a race condition.
+
+  The implementation fix summary is:
+  - add state to signal the end of the ``trio`` side task to be
+    read by the ``asyncio`` side and always cancel any ongoing
+    task in such cases.
+  - always wait on the ``asyncio`` task termination from the ``trio``
+    side on error before maybe raising said error.
+  - always close the ``trio`` mem chan on exit to ensure the other
+    side can detect it and follow.
+
+
+Trivial/Internal Changes
+------------------------
+
+- `#248 <https://github.com/goodboy/tractor/issues/248>`_: Adjust the
+  `tractor._spawn.soft_wait()` strategy to avoid sending an actor cancel
+  request (via `Portal.cancel_actor()`) if either the child process is
+  detected as having terminated or the IPC channel is detected to be
+  closed.
+
+  This ensures (even) more deterministic inter-actor cancellation by
+  avoiding the timeout condition where possible when a whild never
+  sucessfully spawned, crashed, or became un-contactable over IPC.
+
+- `#295 <https://github.com/goodboy/tractor/issues/295>`_: Add an
+  experimental ``tractor.msg.NamespacePath`` type for passing Python
+  objects by "reference" through a ``str``-subtype message and using the
+  new ``pkgutil.resolve_name()`` for reference loading.
+
+- `#298 <https://github.com/goodboy/tractor/issues/298>`_: Add a new
+  `tractor.experimental` subpackage for staging new high level APIs and
+  subystems that we might eventually make built-ins.
+
+- `#300 <https://github.com/goodboy/tractor/issues/300>`_: Update to and
+  pin latest ``msgpack`` (1.0.3) and ``msgspec`` (0.4.0) both of which
+  required adjustments for backwards imcompatible API tweaks.
+
+- `#303 <https://github.com/goodboy/tractor/issues/303>`_: Fence off
+  ``multiprocessing`` imports until absolutely necessary in an effort to
+  avoid "resource tracker" spawning side effects that seem to have
+  varying degrees of unreliability per Python release. Port to new
+  ``msgspec.DecodeError``.
+
+- `#305 <https://github.com/goodboy/tractor/issues/305>`_: Add
+  ``tractor.query_actor()`` an addr looker-upper which doesn't deliver
+  a ``Portal`` instance and instead just a socket address ``tuple``.
+
+  Sometimes it's handy to just have a simple way to figure out if
+  a "service" actor is up, so add this discovery helper for that. We'll
+  prolly just leave it undocumented for now until we figure out
+  a longer-term/better discovery system.
+
+- `#316 <https://github.com/goodboy/tractor/issues/316>`_: Run windows
+  CI jobs on python 3.10 after some hacks for ``pdbpp`` dependency
+  issues.
+
+  Issue was to do with the now deprecated `pyreadline` project which
+  should be changed over to `pyreadline3`.
+
+- `#317 <https://github.com/goodboy/tractor/issues/317>`_: Drop use of
+  the ``msgpack`` package and instead move fully to the ``msgspec``
+  codec library.
+
+  We've now used ``msgspec`` extensively in production and there's no
+  reason to not use it as default. Further this change preps us for the up
+  and coming typed messaging semantics (#196), dialog-unprotocol system
+  (#297), and caps-based messaging-protocols (#299) planned before our
+  first beta.
+
+
+tractor 0.1.0a4 (2021-12-18)
+============================
+
+Features
+--------
+- `#275 <https://github.com/goodboy/tractor/issues/275>`_: Re-license
+  code base under AGPLv3. Also see `#274
+  <https://github.com/goodboy/tractor/pull/274>`_ for majority
+  contributor consensus on this decision.
+
+- `#121 <https://github.com/goodboy/tractor/issues/121>`_: Add
+  "infected ``asyncio`` mode; a sub-system to spawn and control
+  ``asyncio`` actors using ``trio``'s guest-mode.
+
+  This gets us the following very interesting functionality:
+
+  - ability to spawn an actor that has a process entry point of
+    ``asyncio.run()`` by passing ``infect_asyncio=True`` to
+    ``Portal.start_actor()`` (and friends).
+  - the ``asyncio`` actor embeds ``trio`` using guest-mode and starts
+    a main ``trio`` task which runs the ``tractor.Actor._async_main()``
+    entry point engages all the normal ``tractor`` runtime IPC/messaging
+    machinery; for all purposes the actor is now running normally on
+    a ``trio.run()``.
+  - the actor can now make one-to-one task spawning requests to the
+    underlying ``asyncio`` event loop using either of:
+
+    * ``to_asyncio.run_task()`` to spawn and run an ``asyncio`` task to
+      completion and block until a return value is delivered.
+    * ``async with to_asyncio.open_channel_from():`` which spawns a task
+      and hands it a pair of "memory channels" to allow for bi-directional
+      streaming between the now SC-linked ``trio`` and ``asyncio`` tasks.
+
+  The output from any call(s) to ``asyncio`` can be handled as normal in
+  ``trio``/``tractor`` task operation with the caveat of the overhead due
+  to guest-mode use.
+
+  For more details see the `original PR
+  <https://github.com/goodboy/tractor/pull/121>`_ and `issue
+  <https://github.com/goodboy/tractor/issues/120>`_.
+
+- `#257 <https://github.com/goodboy/tractor/issues/257>`_: Add
+  ``trionics.maybe_open_context()`` an actor-scoped async multi-task
+  context manager resource caching API.
+
+  Adds an SC-safe cacheing async context manager api that only enters on
+  the *first* task entry and only exits on the *last* task exit while in
+  between delivering the same cached value per input key. Keys can be
+  either an explicit ``key`` named arg provided by the user or a
+  hashable ``kwargs`` dict (will be converted to a ``list[tuple]``) which
+  is passed to the underlying manager function as input.
+
+- `#261 <https://github.com/goodboy/tractor/issues/261>`_: Add
+  cross-actor-task ``Context`` oriented error relay, a new stream
+  overrun error-signal ``StreamOverrun``, and support disabling
+  ``MsgStream`` backpressure as the default before a stream is opened or
+  by choice of the user.
+
+  We added stricter semantics around ``tractor.Context.open_stream():``
+  particularly to do with streams which are only opened at one end.
+  Previously, if only one end opened a stream there was no way for that
+  sender to know if msgs are being received until first, the feeder mem
+  chan on the receiver side hit a backpressure state and then that
+  condition delayed its msg loop processing task to eventually create
+  backpressure on the associated IPC transport. This is non-ideal in the
+  case where the receiver side never opened a stream by mistake since it
+  results in silent block of the sender and no adherence to the underlying
+  mem chan buffer size settings (which is still unsolved btw).
+
+  To solve this we add non-backpressure style message pushing inside
+  ``Actor._push_result()`` by default and only use the backpressure
+  ``trio.MemorySendChannel.send()`` call **iff** the local end of the
+  context has entered ``Context.open_stream():``. This way if the stream
+  was never opened but the mem chan is overrun, we relay back to the
+  sender a (new exception) ``SteamOverrun`` error which is raised in the
+  sender's scope with a special error message about the stream never
+  having been opened. Further, this behaviour (non-backpressure style
+  where senders can expect an error on overruns) can now be enabled with
+  ``.open_stream(backpressure=False)`` and the underlying mem chan size
+  can be specified with a kwarg ``msg_buffer_size: int``.
+
+  Further bug fixes and enhancements in this changeset include:
+
+  - fix a race we were ignoring where if the callee task opened a context
+    it could enter ``Context.open_stream()`` before calling
+    ``.started()``.
+  - Disallow calling ``Context.started()`` more then once.
+  - Enable ``Context`` linked tasks error relaying via the new
+    ``Context._maybe_raise_from_remote_msg()`` which (for now) uses
+    a simple ``trio.Nursery.start_soon()`` to raise the error via closure
+    in the local scope.
+
+- `#267 <https://github.com/goodboy/tractor/issues/267>`_: This
+  (finally) adds fully acknowledged remote cancellation messaging
+  support for both explicit ``Portal.cancel_actor()`` calls as well as
+  when there is a "runtime-wide" cancellations (eg. during KBI or
+  general actor nursery exception handling which causes a full actor
+  "crash"/termination).
+
+  You can think of this as the most ideal case in 2-generals where the
+  actor requesting the cancel of its child is able to always receive back
+  the ACK to that request. This leads to a more deterministic shutdown of
+  the child where the parent is able to wait for the child to fully
+  respond to the request. On a localhost setup, where the parent can
+  monitor the state of the child through process or other OS APIs instead
+  of solely through IPC messaging, the parent can know whether or not the
+  child decided to cancel with more certainty. In the case of separate
+  hosts, we still rely on a simple timeout approach until such a time
+  where we prefer to get "fancier".
+
+- `#271 <https://github.com/goodboy/tractor/issues/271>`_: Add a per
+  actor ``debug_mode: bool`` control to our nursery.
+
+  This allows spawning actors via ``ActorNursery.start_actor()`` (and
+  other dependent methods) with a ``debug_mode=True`` flag much like
+  ``tractor.open_nursery():`` such that per process crash handling
+  can be toggled for cases where a user does not need/want all child actors
+  to drop into the debugger on error. This is often useful when you have
+  actor-tasks which are expected to error often (and be re-run) but want
+  to specifically interact with some (problematic) child.
+
+
+Bugfixes
+--------
+
+- `#239 <https://github.com/goodboy/tractor/issues/239>`_: Fix
+  keyboard interrupt handling in ``Portal.open_context()`` blocks.
+
+  Previously this was not triggering cancellation of the remote task
+  context and could result in hangs if a stream was also opened. This
+  fix is to accept `BaseException` since it is likely any other top
+  level exception other then KBI (even though not expected) should also
+  get this result.
+
+- `#264 <https://github.com/goodboy/tractor/issues/264>`_: Fix
+  ``Portal.run_in_actor()`` returns ``None`` result.
+
+  ``None`` was being used as the cached result flag and obviously breaks
+  on a ``None`` returned from the remote target task. This would cause an
+  infinite hang if user code ever called ``Portal.result()`` *before* the
+  nursery exit. The simple fix is to use the *return message* as the
+  initial "no-result-received-yet" flag value and, once received, the
+  return value is read from the message to avoid the cache logic error.
+
+- `#266 <https://github.com/goodboy/tractor/issues/266>`_: Fix
+  graceful cancellation of daemon actors
+
+  Previously, his was a bug where if the soft wait on a sub-process (the
+  ``await .proc.wait()``) in the reaper task teardown was cancelled we
+  would fail over to the hard reaping sequence (meant for culling off any
+  potential zombies via system kill signals). The hard reap has a timeout
+  of 3s (currently though in theory we could make it shorter?) before
+  system signalling kicks in. This means that any daemon actor still
+  running during nursery exit would get hard reaped (3s later) instead of
+  cancelled via IPC message. Now we catch the ``trio.Cancelled``, call
+  ``Portal.cancel_actor()`` on the daemon and expect the child to
+  self-terminate after the runtime cancels and shuts down the process.
+
+- `#278 <https://github.com/goodboy/tractor/issues/278>`_: Repair
+  inter-actor stream closure semantics to work correctly with
+  ``tractor.trionics.BroadcastReceiver`` task fan out usage.
+
+  A set of previously unknown bugs discovered in `#257
+  <https://github.com/goodboy/tractor/pull/257>`_ let graceful stream
+  closure result in hanging consumer tasks that use the broadcast APIs.
+  This adds better internal closure state tracking to the broadcast
+  receiver and message stream APIs and in particular ensures that when an
+  underlying stream/receive-channel (a broadcast receiver is receiving
+  from) is closed, all consumer tasks waiting on that underlying channel
+  are woken so they can receive the ``trio.EndOfChannel`` signal and
+  promptly terminate.
+
+
+tractor 0.1.0a3 (2021-11-02)
+============================
+
+Features
+--------
+
+- Switch to using the ``trio`` process spawner by default on windows. (#166)
+
+  This gets windows users debugger support (manually tested) and in
+  general a more resilient (nested) actor tree implementation.
+
+- Add optional `msgspec <https://jcristharif.com/msgspec/>`_ support
+  as an alernative, faster MessagePack codec. (#214)
+
+  Provides us with a path toward supporting typed IPC message contracts. Further,
+  ``msgspec`` structs may be a valid tool to start for formalizing our
+  "SC dialog un-protocol" messages as described in `#36
+  <https://github.com/goodboy/tractor/issues/36>`_.
+
+- Introduce a new ``tractor.trionics`` `sub-package`_ that exposes
+  a selection of our relevant high(er) level trio primitives and
+  goodies. (#241)
+
+  At outset we offer a ``gather_contexts()`` context manager for
+  concurrently entering a sequence of async context managers (much like
+  a version of ``asyncio.gather()`` but for context managers) and use it
+  in a new ``tractor.open_actor_cluster()`` manager-helper that can be
+  entered to concurrently spawn a flat actor pool. We also now publicly
+  expose our "broadcast channel" APIs (``open_broadcast_receiver()``)
+  from here.
+
+.. _sub-package: ../tractor/trionics
+
+- Change the core message loop to handle task and actor-runtime cancel
+  requests immediately instead of scheduling them as is done for rpc-task
+  requests. (#245)
+
+  In order to obtain more reliable teardown mechanics for (complex) actor
+  trees it's important that we specially treat cancel requests as having
+  higher priority. Previously, it was possible that task cancel requests
+  could actually also themselves be cancelled if a "actor-runtime" cancel
+  request was received (can happen during messy multi actor crashes that
+  propagate). Instead cancels now block the msg loop until serviced and
+  a response is relayed back to the requester. This also allows for
+  improved debugger support since we have determinism guarantees about
+  which processes must wait before hard killing their children.
+
+- (`#248 <https://github.com/goodboy/tractor/pull/248>`_) Drop Python
+  3.8 support in favour of rolling with two latest releases for the time
+  being.
+
+
+Misc
+----
+
+- (`#243 <https://github.com/goodboy/tractor/pull/243>`_) add a distinct
+  ``'CANCEL'`` log level to allow the runtime to emit details about
+  cancellation machinery statuses.
+
+
+tractor 0.1.0a2 (2021-09-07)
+============================
+
+Features
+--------
+
+- Add `tokio-style broadcast channels
+  <https://docs.rs/tokio/1.11.0/tokio/sync/broadcast/index.html>`_ as
+  a solution for `#204 <https://github.com/goodboy/tractor/pull/204>`_ and
+  discussed thoroughly in `trio/#987
+  <https://github.com/python-trio/trio/issues/987>`_.
+
+  This gives us local task broadcast functionality using a new
+  ``BroadcastReceiver`` type which can wrap ``trio.ReceiveChannel``  and
+  provide fan-out copies of a stream of data to every subscribed consumer.
+  We use this new machinery to provide a ``ReceiveMsgStream.subscribe()``
+  async context manager which can be used by actor-local concumers tasks
+  to easily pull from a shared and dynamic IPC stream. (`#229
+  <https://github.com/goodboy/tractor/pull/229>`_)
+
+
+Bugfixes
+--------
+
+- Handle broken channel/stream faults where the root's tty lock is left
+  acquired by some child actor who went MIA and the root ends up hanging
+  indefinitely. (`#234 <https://github.com/goodboy/tractor/pull/234>`_)
+
+  There's two parts here: we no longer shield wait on the lock and,
+  now always do our best to release the lock on the expected worst
+  case connection faults.
+
+
+Deprecations and Removals
+-------------------------
+
+- Drop stream "shielding" support which was originally added to sidestep
+  a cancelled call to ``.receive()``
+
+  In the original api design a stream instance was returned directly from
+  a call to ``Portal.run()`` and thus there was no "exit phase" to handle
+  cancellations and errors which would trigger implicit closure. Now that
+  we have said enter/exit semantics with ``Portal.open_stream_from()`` and
+  ``Context.open_stream()`` we can drop this implicit (and arguably
+  confusing) behavior. (`#230 <https://github.com/goodboy/tractor/pull/230>`_)
+
+- Drop Python 3.7 support in preparation for supporting 3.9+ syntax.
+  (`#232 <https://github.com/goodboy/tractor/pull/232>`_)
+
+
+tractor 0.1.0a1 (2021-08-01)
+============================
+
+Features
+--------
+- Updated our uni-directional streaming API (`#206
+  <https://github.com/goodboy/tractor/pull/206>`_) to require a context
+  manager style ``async with Portal.open_stream_from(target) as stream:``
+  which explicitly determines when to stop a stream in the calling (aka
+  portal opening) actor much like ``async_generator.aclosing()``
+  enforcement.
+
+- Improved the ``multiprocessing`` backend sub-actor reaping (`#208
+  <https://github.com/goodboy/tractor/pull/208>`_) during actor nursery
+  exit, particularly during cancellation scenarios that previously might
+  result in hard to debug hangs.
+
+- Added initial bi-directional streaming support in `#219
+  <https://github.com/goodboy/tractor/pull/219>`_ with follow up debugger
+  improvements via `#220 <https://github.com/goodboy/tractor/pull/220>`_
+  using the new ``tractor.Context`` cross-actor task syncing system.
+  The debugger upgrades add an edge triggered last-in-tty-lock semaphore
+  which allows the root process for a tree to avoid clobbering children
+  who have queued to acquire the ``pdb`` repl by waiting to cancel
+  sub-actors until the lock is known to be released **and** has no
+  pending waiters.
+
+
+Experiments and WIPs
+--------------------
+- Initial optional ``msgspec`` serialization support in `#214
+  <https://github.com/goodboy/tractor/pull/214>`_ which should hopefully
+  land by next release.
+
+- Improved "infect ``asyncio``" cross-loop task cancellation and error
+  propagation by vastly simplifying the cross-loop-task streaming approach. 
+  We may end up just going with a use of ``anyio`` in the medium term to
+  avoid re-doing work done by their cross-event-loop portals.  See the
+  ``infect_asyncio`` for details.
+
+
+Improved Documentation
+----------------------
+- `Updated our readme <https://github.com/goodboy/tractor/pull/211>`_ to
+  include more (and better) `examples
+  <https://github.com/goodboy/tractor#run-a-func-in-a-process>`_ (with
+  matching multi-terminal process monitoring shell commands) as well as
+  added many more examples to the `repo set
+  <https://github.com/goodboy/tractor/tree/master/examples>`_.
+
+- Added a readme `"actors under the hood" section
+  <https://github.com/goodboy/tractor#under-the-hood>`_ in an effort to
+  guard against suggestions for changing the API away from ``trio``'s
+  *tasks-as-functions* style.
+
+- Moved to using the `sphinx book theme
+  <https://sphinx-book-theme.readthedocs.io/en/latest/index.html>`_
+  though it needs some heavy tweaking and doesn't seem to show our logo
+  on rtd :(
+
+
+Trivial/Internal Changes
+------------------------
+- Added a new ``TransportClosed`` internal exception/signal (`#215
+  <https://github.com/goodboy/tractor/pull/215>`_ for catching TCP
+  channel gentle closes instead of silently falling through the message
+  handler loop via an async generator ``return``.
+
+
+Deprecations and Removals
+-------------------------
+- Dropped support for invoking sync functions (`#205
+  <https://github.com/goodboy/tractor/pull/205>`_) in other
+  actors/processes since you can always wrap a sync function from an
+  async one.  Users can instead consider using ``trio-parallel`` which
+  is a project specifically geared for purely synchronous calls in
+  sub-processes.
+
+- Deprecated our ``tractor.run()`` entrypoint `#197
+  <https://github.com/goodboy/tractor/pull/197>`_; the runtime is now
+  either started implicitly in first actor nursery use or via an
+  explicit call to ``tractor.open_root_actor()``. Full removal of
+  ``tractor.run()`` will come by beta release.
+
+
+tractor 0.1.0a0 (2021-02-28)
+============================
+
+..
+    TODO: fill out more of the details of the initial feature set in some TLDR form
+
+Summary
+-------
+- ``trio`` based process spawner (using ``subprocess``)
+- initial multi-process debugging with ``pdb++``
+- windows support using both ``trio`` and ``multiprocessing`` spawners
+- "portal" api for cross-process, structured concurrent, (streaming) IPC

README.rst

@@ -1,631 +0,0 @@
-tractor
-=======
-An async-native `actor model`_ built on trio_ and multiprocessing_.
-
-
-|travis|
-
-.. |travis| image:: https://img.shields.io/travis/tgoodlet/tractor/master.svg
-    :target: https://travis-ci.org/tgoodlet/tractor
-
-.. _actor model: https://en.wikipedia.org/wiki/Actor_model
-.. _trio: https://github.com/python-trio/trio
-.. _multiprocessing: https://docs.python.org/3/library/multiprocessing.html
-.. _trionic: https://trio.readthedocs.io/en/latest/design.html#high-level-design-principles
-.. _async sandwich: https://trio.readthedocs.io/en/latest/tutorial.html#async-sandwich
-.. _always propagate: https://trio.readthedocs.io/en/latest/design.html#exceptions-always-propagate
-.. _causality: https://vorpus.org/blog/some-thoughts-on-asynchronous-api-design-in-a-post-asyncawait-world/#c-c-c-c-causality-breaker
-.. _shared nothing architecture: https://en.wikipedia.org/wiki/Shared-nothing_architecture
-.. _cancellation: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-and-timeouts
-.. _channels: https://en.wikipedia.org/wiki/Channel_(programming)
-.. _chaos engineering: http://principlesofchaos.org/
-
-
-What's this? Spawning event loops in processes?
------------------------------------------------
-Heh, yeah pretty much.
-
-``tractor`` is an attempt to take trionic_ concurrency concepts and apply
-them to distributed multi-core Python.
-
-``tractor`` lets you run and spawn *actors*: separate processes which run a ``trio``
-scheduler and task tree (also known as an `async sandwich`_).
-*Actors* communicate by sending messages_ over channels_ and avoid sharing any state.
-This `actor model`_ allows for highly distributed software architecture which works just as
-well on multiple cores as it does over many hosts.
-``tractor`` takes much inspiration from pulsar_ and execnet_ but attempts to be much more
-focussed on sophistication of the lower level distributed architecture as well as have first
-class support for `modern async Python`_.
-
-The first step to grok ``tractor`` is to get the basics of ``trio``
-down. A great place to start is the `trio docs`_ and this `blog post`_.
-
-.. _messages: https://en.wikipedia.org/wiki/Message_passing
-.. _trio docs: https://trio.readthedocs.io/en/latest/
-.. _blog post: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/
-.. _modern async Python: https://www.python.org/dev/peps/pep-0525/
-
-
-Philosophy
-----------
-``tractor``'s tenets non-comprehensively include:
-
-- no spawning of processes *willy-nilly*; causality_ is paramount!
-- `shared nothing architecture`_
-- remote errors `always propagate`_ back to the caller
-- verbatim support for ``trio``'s cancellation_ system
-- no use of *proxy* objects to wrap RPC calls
-- an immersive debugging experience
-- anti-fragility through `chaos engineering`_
-
-.. warning:: ``tractor`` is in alpha-alpha and is expected to change rapidly!
-    Expect nothing to be set in stone. Your ideas about where it should go
-    are greatly appreciated!
-
-.. _pulsar: http://quantmind.github.io/pulsar/design.html
-.. _execnet: https://codespeak.net/execnet/
-
-
-Install
--------
-No PyPi release yet!
-
-::
-
-    pip install git+git://github.com/tgoodlet/tractor.git
-
-
-A trynamic first scene
-----------------------
-Let's direct a couple *actors* and have them run their lines for
-the hip new film we're shooting:
-
-.. code:: python
-
-    import tractor
-    from functools import partial
-
-    _this_module = __name__
-    the_line = 'Hi my name is {}'
-
-
-    async def hi():
-        return the_line.format(tractor.current_actor().name)
-
-
-    async def say_hello(other_actor):
-        async with tractor.wait_for_actor(other_actor) as portal:
-            return await portal.run(_this_module, 'hi')
-
-
-    async def main():
-        """Main tractor entry point, the "master" process (for now
-        acts as the "director").
-        """
-        async with tractor.open_nursery() as n:
-            print("Alright... Action!")
-
-            donny = await n.run_in_actor(
-                'donny',
-                say_hello,
-                other_actor='gretchen',
-            )
-            gretchen = await n.run_in_actor(
-                'gretchen',
-                say_hello,
-                other_actor='donny',
-            )
-            print(await gretchen.result())
-            print(await donny.result())
-            print("CUTTTT CUUTT CUT!!! Donny!! You're supposed to say...")
-
-
-    tractor.run(main)
-
-
-We spawn two *actors*, *donny* and *gretchen*.
-Each actor starts up and executes their *main task* defined by an
-async function, ``say_hello()``.  The function instructs each actor
-to find their partner and say hello by calling their partner's
-``hi()`` function using something called a *portal*. Each actor
-receives a response and relays that back to the parent actor (in
-this case our "director" executing ``main()``).
-
-
-Actor spawning and causality
-----------------------------
-``tractor`` tries to take ``trio``'s concept of causal task lifetimes
-to multi-process land. Accordingly, ``tractor``'s *actor nursery* behaves
-similar to ``trio``'s nursery_. That is, ``tractor.open_nursery()``
-opens an ``ActorNursery`` which waits on spawned *actors* to complete
-(or error) in the same causal_ way ``trio`` waits on spawned subtasks.
-This includes errors from any one actor causing all other actors
-spawned by the same nursery to be cancelled_.
-
-To spawn an actor and run a function in it, open a *nursery block*
-and use the ``run_in_actor()`` method:
-
-.. code:: python
-
-    import tractor
-
-
-        def cellar_door():
-            return "Dang that's beautiful"
-
-
-        async def main():
-            """The main ``tractor`` routine.
-            """
-            async with tractor.open_nursery() as n:
-
-                portal = await n.run_in_actor('frank', movie_theatre_question)
-
-            # The ``async with`` will unblock here since the 'frank'
-            # actor has completed its main task ``movie_theatre_question()``.
-
-            print(await portal.result())
-
-
-    tractor.run(main)
-
-
-What's going on?
-
-- an initial *actor* is started with ``tractor.run()`` and told to execute
-  its main task_: ``main()``
-
-- inside ``main()`` an actor is *spawned* using an ``ActorNusery`` and is told
-  to run a single function: ``cellar_door()``
-
-- a ``portal`` instance (we'll get to what it is shortly)
-  returned from ``nursery.run_in_actor()`` is used to communicate with
-  the newly spawned *sub-actor*
-
-- the second actor, *frank*, in a new *process* running a new ``trio`` task_
-  then executes ``cellar_door()`` and returns its result over a *channel* back
-  to the parent actor
-
-- the parent actor retrieves the subactor's (*frank*) *final result* using ``portal.result()``
-  much like you'd expect from a future_.
-
-This ``run_in_actor()`` API should look very familiar to users of
-``asyncio``'s run_in_executor_ which uses a ``concurrent.futures`` Executor_.
-
-Since you might also want to spawn long running *worker* or *daemon*
-actors, each actor's *lifetime* can be determined based on the spawn
-method:
-
-- if the actor is spawned using ``run_in_actor()`` it terminates when
-  its *main* task completes (i.e. when the (async) function submitted
-  to it *returns*). The ``with tractor.open_nursery()`` exits only once
-  all actors' main function/task complete (just like the nursery_ in ``trio``)
-
-- actors can be spawned to *live forever* using the ``start_actor()``
-  method and act like an RPC daemon that runs indefinitely (the
-  ``with tractor.open_nursery()`` wont' exit) until cancelled_
-
-Had we wanted the latter form in our example it would have looked like:
-
-.. code:: python
-
-    def movie_theatre_question():
-        """A question asked in a dark theatre, in a tangent
-        (errr, I mean different) process.
-        """
-        return 'have you ever seen a portal?'
-
-
-    async def main():
-        """The main ``tractor`` routine.
-        """
-        async with tractor.open_nursery() as n:
-
-            portal = await n.start_actor(
-                'frank',
-                # enable the actor to run funcs from this current module
-                rpc_module_paths=[__name__],
-            )
-
-            print(await portal.run(__name__, 'movie_theatre_question'))
-            # call the subactor a 2nd time
-            print(await portal.run(__name__, 'movie_theatre_question'))
-
-            # the async with will block here indefinitely waiting
-            # for our actor "frank" to complete, but since it's an
-            # "outlive_main" actor it will never end until cancelled
-            await portal.cancel_actor()
-
-
-The ``rpc_module_paths`` `kwarg` above is a list of module path
-strings that will be loaded and made accessible for execution in the
-remote actor through a call to ``Portal.run()``. For now this is
-a simple mechanism to restrict the functionality of the remote
-(and possibly daemonized) actor and uses Python's module system to
-limit the allowed remote function namespace(s).
-
-``tractor`` is opinionated about the underlying threading model used for
-each *actor*. Since Python has a GIL and an actor model by definition
-shares no state between actors, it fits naturally to use a multiprocessing_
-``Process``. This allows ``tractor`` programs to leverage not only multi-core
-hardware but also distribute over many hardware hosts (each *actor* can talk
-to all others with ease over standard network protocols).
-
-.. _task: https://trio.readthedocs.io/en/latest/reference-core.html#tasks-let-you-do-multiple-things-at-once
-.. _nursery: https://trio.readthedocs.io/en/latest/reference-core.html#nurseries-and-spawning
-.. _causal: https://vorpus.org/blog/some-thoughts-on-asynchronous-api-design-in-a-post-asyncawait-world/#causality
-.. _cancelled: https://trio.readthedocs.io/en/latest/reference-core.html#child-tasks-and-cancellation
-.. _run_in_executor: https://docs.python.org/3/library/asyncio-eventloop.html#executor
-.. _Executor: https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Executor
-
-
-Transparent remote function calling using *portals*
----------------------------------------------------
-``tractor`` introduces the concept of a *portal* which is an API
-borrowed_ from ``trio``. A portal may seem similar to the idea of
-a RPC future_ except a *portal* allows invoking remote *async* functions and
-generators and intermittently blocking to receive responses. This allows
-for fully async-native IPC between actors.
-
-When you invoke another actor's routines using a *portal* it looks as though
-it was called locally in the current actor. So when you see a call to
-``await portal.run()`` what you get back is what you'd expect
-to if you'd called the function directly in-process. This approach avoids
-the need to add any special RPC *proxy* objects to the library by instead just
-relying on the built-in (async) function calling semantics and protocols of Python.
-
-Depending on the function type ``Portal.run()`` tries to
-correctly interface exactly like a local version of the remote
-built-in Python *function type*. Currently async functions, generators,
-and regular functions are supported. Inspiration for this API comes
-from the way execnet_ does `remote function execution`_ but without
-the client code (necessarily) having to worry about the underlying
-channels_ system or shipping code over the network.
-
-This *portal* approach turns out to be paricularly exciting with the
-introduction of `asynchronous generators`_ in Python 3.6! It means that
-actors can compose nicely in a data processing pipeline.
-
-As an example here's an actor that streams for 1 second from a remote async
-generator function running in a separate actor:
-
-.. code:: python
-
-    from itertools import repeat
-    import trio
-    import tractor
-
-
-    async def stream_forever():
-        for i in repeat("I can see these little future bubble things"):
-            # each yielded value is sent over the ``Channel`` to the
-            # parent actor
-            yield i
-            await trio.sleep(0.01)
-
-
-    async def main():
-        # stream for at most 1 seconds
-        with trio.move_on_after(1) as cancel_scope:
-            async with tractor.open_nursery() as n:
-                portal = await n.start_actor(
-                    f'donny',
-                    rpc_module_paths=[__name__],
-                )
-
-                # this async for loop streams values from the above
-                # async generator running in a separate process
-                async for letter in await portal.run(__name__, 'stream_forever'):
-                    print(letter)
-
-        # we support trio's cancellation system
-        assert cancel_scope.cancelled_caught
-        assert n.cancelled
-
-
-    tractor.run(main)
-
-
-Alright, let's get fancy.
-
-Say you wanted to spawn two actors which each pulling data feeds from
-two different sources (and wanted this work spread across 2 cpus).
-You also want to aggregate these feeds, do some processing on them and then
-deliver the final result stream to a client (or in this case parent) actor
-and print the results to your screen:
-
-.. code:: python
-
-    import time
-    import trio
-    import tractor
-
-
-    # this is the first 2 actors, streamer_1 and streamer_2
-    async def stream_data(seed):
-        for i in range(seed):
-            yield i
-            await trio.sleep(0)  # trigger scheduler
-
-
-    # this is the third actor; the aggregator
-    async def aggregate(seed):
-        """Ensure that the two streams we receive match but only stream
-        a single set of values to the parent.
-        """
-        async with tractor.open_nursery() as nursery:
-            portals = []
-            for i in range(1, 3):
-                # fork point
-                portal = await nursery.start_actor(
-                    name=f'streamer_{i}',
-                    rpc_module_paths=[__name__],
-                )
-
-                portals.append(portal)
-
-            q = trio.Queue(500)
-
-            async def push_to_q(portal):
-                async for value in await portal.run(
-                    __name__, 'stream_data', seed=seed
-                ):
-                    # leverage trio's built-in backpressure
-                    await q.put(value)
-
-                await q.put(None)
-                print(f"FINISHED ITERATING {portal.channel.uid}")
-
-            # spawn 2 trio tasks to collect streams and push to a local queue
-            async with trio.open_nursery() as n:
-                for portal in portals:
-                    n.start_soon(push_to_q, portal)
-
-                unique_vals = set()
-                async for value in q:
-                    if value not in unique_vals:
-                        unique_vals.add(value)
-                        # yield upwards to the spawning parent actor
-                        yield value
-
-                        if value is None:
-                            break
-
-                    assert value in unique_vals
-
-                print("FINISHED ITERATING in aggregator")
-
-            await nursery.cancel()
-            print("WAITING on `ActorNursery` to finish")
-        print("AGGREGATOR COMPLETE!")
-
-
-    # this is the main actor and *arbiter*
-    async def main():
-        # a nursery which spawns "actors"
-        async with tractor.open_nursery() as nursery:
-
-            seed = int(1e3)
-            import time
-            pre_start = time.time()
-
-            portal = await nursery.run_in_actor(
-                'aggregator',
-                aggregate,
-                seed=seed,
-            )
-
-            start = time.time()
-            # the portal call returns exactly what you'd expect
-            # as if the remote "aggregate" function was called locally
-            result_stream = []
-            async for value in await portal.result():
-                result_stream.append(value)
-
-            print(f"STREAM TIME = {time.time() - start}")
-            print(f"STREAM + SPAWN TIME = {time.time() - pre_start}")
-            assert result_stream == list(range(seed)) + [None]
-            return result_stream
-
-
-    final_stream = tractor.run(main, arbiter_addr=('127.0.0.1', 1616))
-
-
-Here there's four actors running in separate processes (using all the
-cores on you machine). Two are streaming by *yielding* values from the
-``stream_data()`` async generator, one is aggregating values from
-those two in ``aggregate()`` (also an async generator) and shipping the
-single stream of unique values up the parent actor (the ``'MainProcess'``
-as ``multiprocessing`` calls it) which is running ``main()``. 
-
-.. _future: https://en.wikipedia.org/wiki/Futures_and_promises
-.. _borrowed:
-    https://trio.readthedocs.io/en/latest/reference-core.html#getting-back-into-the-trio-thread-from-another-thread
-.. _asynchronous generators: https://www.python.org/dev/peps/pep-0525/
-.. _remote function execution: https://codespeak.net/execnet/example/test_info.html#remote-exec-a-function-avoiding-inlined-source-part-i
-.. _asyncitertools: https://github.com/vodik/asyncitertools
-
-
-Cancellation
-------------
-``tractor`` supports ``trio``'s cancellation_ system verbatim.
-Cancelling a nursery block cancels all actors spawned by it.
-Eventually ``tractor`` plans to support different `supervision strategies`_ like ``erlang``.
-
-.. _supervision strategies: http://erlang.org/doc/man/supervisor.html#sup_flags
-
-
-Remote error propagation
-------------------------
-Any task invoked in a remote actor should ship any error(s) back to the calling
-actor where it is raised and expected to be dealt with. This way remote actors
-are never cancelled unless explicitly asked or there's a bug in ``tractor`` itself.
-
-.. code:: python
-
-    async def assert_err():
-        assert 0
-
-
-    async def main():
-        async with tractor.open_nursery() as n:
-            real_actors = []
-            for i in range(3):
-                real_actors.append(await n.start_actor(
-                    f'actor_{i}',
-                    rpc_module_paths=[__name__],
-                ))
-
-            # start one actor that will fail immediately
-            await n.run_in_actor('extra', assert_err)
-
-        # should error here with a ``RemoteActorError`` containing
-        # an ``AssertionError`` and all the other actors have been cancelled
-
-    try:
-        # also raises
-        tractor.run(main)
-    except tractor.RemoteActorError:
-        print("Look Maa that actor failed hard, hehhh!")
-
-
-You'll notice the nursery cancellation conducts a *one-cancels-all*
-supervisory strategy `exactly like trio`_. The plan is to add more
-`erlang strategies`_ in the near future by allowing nurseries to accept
-a ``Supervisor`` type.
-
-.. _exactly like trio: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-semantics
-.. _erlang strategies: http://learnyousomeerlang.com/supervisors
-
-
-Shared task state
------------------
-Although ``tractor`` uses a *shared-nothing* architecture between processes
-you can of course share state within an actor.  ``trio`` tasks spawned via
-multiple RPC calls to an actor can access global data using the per actor
-``statespace`` dictionary:
-
-.. code:: python
-
-
-        statespace = {'doggy': 10}
-
-
-        def check_statespace():
-            # Remember this runs in a new process so no changes
-            # will propagate back to the parent actor
-            assert tractor.current_actor().statespace == statespace
-
-
-        async def main():
-            async with tractor.open_nursery() as n:
-                await n.run_in_actor(
-                    'checker',
-                    check_statespace,
-                    statespace=statespace
-                )
-
-
-Of course you don't have to use the ``statespace`` variable (it's mostly
-a convenience for passing simple data to newly spawned actors); building
-out a state sharing system per-actor is totally up to you.
-
-
-How do actors find each other (a poor man's *service discovery*)?
------------------------------------------------------------------
-Though it will be built out much more in the near future, ``tractor``
-currently keeps track of actors by ``(name: str, id: str)`` using a
-special actor called the *arbiter*. Currently the *arbiter* must exist
-on a host (or it will be created if one can't be found) and keeps a
-simple ``dict`` of actor names to sockets for discovery by other actors.
-Obviously this can be made more sophisticated (help me with it!) but for
-now it does the trick.
-
-To find the arbiter from the current actor use the ``get_arbiter()`` function and to
-find an actor's socket address by name use the ``find_actor()`` function:
-
-.. code:: python
-
-    import tractor
-
-
-    async def main(service_name):
-
-        async with tractor.get_arbiter() as portal:
-            print(f"Arbiter is listening on {portal.channel}")
-
-        async with tractor.find_actor(service_name) as sockaddr:
-            print(f"my_service is found at {my_service}")
-
-
-    tractor.run(main, service_name)
-
-
-The ``name`` value you should pass to ``find_actor()`` is the one you passed as the
-*first* argument to either ``tractor.run()`` or ``ActorNursery.start_actor()``.
-
-
-Using ``Channel`` directly (undocumented)
------------------------------------------
-You can use the ``Channel`` api if necessary by simply defining a
-``chan`` and ``cid`` *kwarg* in your async function definition.
-``tractor`` will treat such async functions like async generators on
-the calling side (for now anyway) such that you can push stream values
-a little more granularly if you find *yielding* values to be restrictive.
-I am purposely not documenting this feature with code because I'm not yet
-sure yet how it should be used correctly. If you'd like more details
-please feel free to ask me on the `trio gitter channel`_.
-
-
-Running actors standalone (without spawning)
---------------------------------------------
-You don't have to spawn any actors using ``open_nursery()`` if you just
-want to run a single actor that connects to an existing cluster.
-All the comms and arbiter registration stuff still works. This can
-somtimes turn out being handy when debugging mult-process apps when you
-need to hop into a debugger. You just need to pass the existing
-*arbiter*'s socket address you'd like to connect to:
-
-.. code:: python
-
-    tractor.run(main, arbiter_addr=('192.168.0.10', 1616))
-
-
-Enabling logging
-----------------
-Considering how complicated distributed software can become it helps to know
-what exactly it's doing (even at the lowest levels). Luckily ``tractor`` has
-tons of logging throughout the core. ``tractor`` isn't opinionated on
-how you use this information and users are expected to consume log messages in
-whichever way is appropriate for the system at hand. That being said, when hacking
-on ``tractor`` there is a prettified console formatter which you can enable to
-see what the heck is going on. Just put the following somewhere in your code:
-
-.. code:: python
-
-    from tractor.log import get_console_log
-    log = get_console_log('trace')
-
-
-What the future holds
----------------------
-Stuff I'd like to see ``tractor`` do one day:
-
-- erlang-like supervisors_
-- native support for zeromq_ as a channel transport
-- native `gossip protocol`_ support for service discovery and arbiter election
-- a distributed log ledger for tracking cluster behaviour
-- a slick multi-process aware debugger much like in celery_
-  but with better `pdb++`_ support
-- an extensive `chaos engineering`_ test suite
-- support for reactive programming primitives and native support for asyncitertools_ like libs
-
-If you're interested in tackling any of these please do shout about it on the
-`trio gitter channel`_!
-
-.. _supervisors: http://learnyousomeerlang.com/supervisors
-.. _zeromq: https://en.wikipedia.org/wiki/ZeroMQ
-.. _gossip protocol: https://en.wikipedia.org/wiki/Gossip_protocol
-.. _trio gitter channel: https://gitter.im/python-trio/general
-.. _celery: http://docs.celeryproject.org/en/latest/userguide/debugging.html
-.. _pdb++: https://github.com/antocuni/pdb

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/README.rst

@@ -0,0 +1,632 @@
+|logo| ``tractor``: next-gen Python parallelism
+
+|gh_actions|
+|docs|
+
+``tractor`` is a `structured concurrent`_, multi-processing_ runtime
+built on trio_.
+
+Fundamentally, ``tractor`` gives you parallelism via
+``trio``-"*actors*": independent Python processes (aka
+non-shared-memory threads) which maintain structured
+concurrency (SC) *end-to-end* inside a *supervision tree*.
+
+Cross-process (and thus cross-host) SC is accomplished through the
+combined use of our "actor nurseries_" and an "SC-transitive IPC
+protocol" constructed on top of multiple Pythons each running a ``trio``
+scheduled runtime - a call to ``trio.run()``.
+
+We believe the system adheres to the `3 axioms`_ of an "`actor model`_"
+but likely *does not* look like what *you* probably think an "actor
+model" looks like, and that's *intentional*.
+
+The first step to grok ``tractor`` is to get the basics of ``trio`` down.
+A great place to start is the `trio docs`_ and this `blog post`_.
+
+
+Features
+--------
+- **It's just** a ``trio`` API
+- *Infinitely nesteable* process trees
+- Builtin IPC streaming APIs with task fan-out broadcasting
+- A "native" multi-core debugger REPL using `pdbp`_ (a fork & fix of
+  `pdb++`_ thanks to @mdmintz!)
+- Support for a swappable, OS specific, process spawning layer
+- A modular transport stack, allowing for custom serialization (eg. with
+  `msgspec`_), communications protocols, and environment specific IPC
+  primitives
+- Support for spawning process-level-SC, inter-loop one-to-one-task oriented
+  ``asyncio`` actors via "infected ``asyncio``" mode
+- `structured chadcurrency`_ from the ground up
+
+
+Run a func in a process
+-----------------------
+Use ``trio``'s style of focussing on *tasks as functions*:
+
+.. code:: python
+
+    """
+    Run with a process monitor from a terminal using::
+
+        $TERM -e watch -n 0.1  "pstree -a $$" \
+            & python examples/parallelism/single_func.py \
+            && kill $!
+
+    """
+    import os
+
+    import tractor
+    import trio
+
+
+    async def burn_cpu():
+
+        pid = os.getpid()
+
+        # burn a core @ ~ 50kHz
+        for _ in range(50000):
+            await trio.sleep(1/50000/50)
+
+        return os.getpid()
+
+
+    async def main():
+
+        async with tractor.open_nursery() as n:
+
+            portal = await n.run_in_actor(burn_cpu)
+
+            #  burn rubber in the parent too
+            await burn_cpu()
+
+            # wait on result from target function
+            pid = await portal.result()
+
+        # end of nursery block
+        print(f"Collected subproc {pid}")
+
+
+    if __name__ == '__main__':
+        trio.run(main)
+
+
+This runs ``burn_cpu()`` in a new process and reaps it on completion
+of the nursery block.
+
+If you only need to run a sync function and retreive a single result, you
+might want to check out `trio-parallel`_.
+
+
+Zombie safe: self-destruct a process tree
+-----------------------------------------
+``tractor`` tries to protect you from zombies, no matter what.
+
+.. code:: python
+
+    """
+    Run with a process monitor from a terminal using::
+
+        $TERM -e watch -n 0.1  "pstree -a $$" \
+            & python examples/parallelism/we_are_processes.py \
+            && kill $!
+
+    """
+    from multiprocessing import cpu_count
+    import os
+
+    import tractor
+    import trio
+
+
+    async def target():
+        print(
+            f"Yo, i'm '{tractor.current_actor().name}' "
+            f"running in pid {os.getpid()}"
+        )
+
+        await trio.sleep_forever()
+
+
+    async def main():
+
+        async with tractor.open_nursery() as n:
+
+            for i in range(cpu_count()):
+                await n.run_in_actor(target, name=f'worker_{i}')
+
+            print('This process tree will self-destruct in 1 sec...')
+            await trio.sleep(1)
+
+            # raise an error in root actor/process and trigger
+            # reaping of all minions
+            raise Exception('Self Destructed')
+
+
+    if __name__ == '__main__':
+        try:
+            trio.run(main)
+        except Exception:
+            print('Zombies Contained')
+
+
+If you can create zombie child processes (without using a system signal)
+it **is a bug**.
+
+
+"Native" multi-process debugging
+--------------------------------
+Using the magic of `pdbp`_ and our internal IPC, we've
+been able to create a native feeling debugging experience for
+any (sub-)process in your ``tractor`` tree.
+
+.. code:: python
+
+    from os import getpid
+
+    import tractor
+    import trio
+
+
+    async def breakpoint_forever():
+        "Indefinitely re-enter debugger in child actor."
+        while True:
+            yield 'yo'
+            await tractor.breakpoint()
+
+
+    async def name_error():
+        "Raise a ``NameError``"
+        getattr(doggypants)
+
+
+    async def main():
+        """Test breakpoint in a streaming actor.
+        """
+        async with tractor.open_nursery(
+            debug_mode=True,
+            loglevel='error',
+        ) as n:
+
+            p0 = await n.start_actor('bp_forever', enable_modules=[__name__])
+            p1 = await n.start_actor('name_error', enable_modules=[__name__])
+
+            # retreive results
+            stream = await p0.run(breakpoint_forever)
+            await p1.run(name_error)
+
+
+    if __name__ == '__main__':
+        trio.run(main)
+
+
+You can run this with::
+
+    >>> python examples/debugging/multi_daemon_subactors.py
+
+And, yes, there's a built-in crash handling mode B)
+
+We're hoping to add a respawn-from-repl system soon!
+
+
+SC compatible bi-directional streaming
+--------------------------------------
+Yes, you saw it here first; we provide 2-way streams
+with reliable, transitive setup/teardown semantics.
+
+Our nascent api is remniscent of ``trio.Nursery.start()``
+style invocation:
+
+.. code:: python
+
+    import trio
+    import tractor
+
+
+    @tractor.context
+    async def simple_rpc(
+
+        ctx: tractor.Context,
+        data: int,
+
+    ) -> None:
+        '''Test a small ping-pong 2-way streaming server.
+
+        '''
+        # signal to parent that we're up much like
+        # ``trio_typing.TaskStatus.started()``
+        await ctx.started(data + 1)
+
+        async with ctx.open_stream() as stream:
+
+            count = 0
+            async for msg in stream:
+
+                assert msg == 'ping'
+                await stream.send('pong')
+                count += 1
+
+            else:
+                assert count == 10
+
+
+    async def main() -> None:
+
+        async with tractor.open_nursery() as n:
+
+            portal = await n.start_actor(
+                'rpc_server',
+                enable_modules=[__name__],
+            )
+
+            # XXX: this syntax requires py3.9
+            async with (
+
+                portal.open_context(
+                    simple_rpc,
+                    data=10,
+                ) as (ctx, sent),
+
+                ctx.open_stream() as stream,
+            ):
+
+                assert sent == 11
+
+                count = 0
+                # receive msgs using async for style
+                await stream.send('ping')
+
+                async for msg in stream:
+                    assert msg == 'pong'
+                    await stream.send('ping')
+                    count += 1
+
+                    if count >= 9:
+                        break
+
+
+            # explicitly teardown the daemon-actor
+            await portal.cancel_actor()
+
+
+    if __name__ == '__main__':
+        trio.run(main)
+
+
+See original proposal and discussion in `#53`_ as well
+as follow up improvements in `#223`_ that we'd love to
+hear your thoughts on!
+
+.. _#53: https://github.com/goodboy/tractor/issues/53
+.. _#223: https://github.com/goodboy/tractor/issues/223
+
+
+Worker poolz are easy peasy
+---------------------------
+The initial ask from most new users is *"how do I make a worker
+pool thing?"*.
+
+``tractor`` is built to handle any SC (structured concurrent) process
+tree you can imagine; a "worker pool" pattern is a trivial special
+case.
+
+We have a `full worker pool re-implementation`_ of the std-lib's
+``concurrent.futures.ProcessPoolExecutor`` example for reference.
+
+You can run it like so (from this dir) to see the process tree in
+real time::
+
+    $TERM -e watch -n 0.1  "pstree -a $$" \
+        & python examples/parallelism/concurrent_actors_primes.py \
+        && kill $!
+
+This uses no extra threads, fancy semaphores or futures; all we need
+is ``tractor``'s IPC!
+
+"Infected ``asyncio``" mode
+---------------------------
+Have a bunch of ``asyncio`` code you want to force to be SC at the process level?
+
+Check out our experimental system for `guest-mode`_ controlled
+``asyncio`` actors:
+
+.. code:: python
+
+    import asyncio
+    from statistics import mean
+    import time
+
+    import trio
+    import tractor
+
+
+    async def aio_echo_server(
+        to_trio: trio.MemorySendChannel,
+        from_trio: asyncio.Queue,
+    ) -> None:
+
+        # a first message must be sent **from** this ``asyncio``
+        # task or the ``trio`` side will never unblock from
+        # ``tractor.to_asyncio.open_channel_from():``
+        to_trio.send_nowait('start')
+
+        # XXX: this uses an ``from_trio: asyncio.Queue`` currently but we
+        # should probably offer something better.
+        while True:
+            # echo the msg back
+            to_trio.send_nowait(await from_trio.get())
+            await asyncio.sleep(0)
+
+
+    @tractor.context
+    async def trio_to_aio_echo_server(
+        ctx: tractor.Context,
+    ):
+        # this will block until the ``asyncio`` task sends a "first"
+        # message.
+        async with tractor.to_asyncio.open_channel_from(
+            aio_echo_server,
+        ) as (first, chan):
+
+            assert first == 'start'
+            await ctx.started(first)
+
+            async with ctx.open_stream() as stream:
+
+                async for msg in stream:
+                    await chan.send(msg)
+
+                    out = await chan.receive()
+                    # echo back to parent actor-task
+                    await stream.send(out)
+
+
+    async def main():
+
+        async with tractor.open_nursery() as n:
+            p = await n.start_actor(
+                'aio_server',
+                enable_modules=[__name__],
+                infect_asyncio=True,
+            )
+            async with p.open_context(
+                trio_to_aio_echo_server,
+            ) as (ctx, first):
+
+                assert first == 'start'
+
+                count = 0
+                async with ctx.open_stream() as stream:
+
+                    delays = []
+                    send = time.time()
+
+                    await stream.send(count)
+                    async for msg in stream:
+                        recv = time.time()
+                        delays.append(recv - send)
+                        assert msg == count
+                        count += 1
+                        send = time.time()
+                        await stream.send(count)
+
+                        if count >= 1e3:
+                            break
+
+            print(f'mean round trip rate (Hz): {1/mean(delays)}')
+            await p.cancel_actor()
+
+
+    if __name__ == '__main__':
+        trio.run(main)
+
+
+Yes, we spawn a python process, run ``asyncio``, start ``trio`` on the
+``asyncio`` loop, then send commands to the ``trio`` scheduled tasks to
+tell ``asyncio`` tasks what to do XD
+
+We need help refining the `asyncio`-side channel API to be more
+`trio`-like. Feel free to sling your opinion in `#273`_!
+
+
+.. _#273: https://github.com/goodboy/tractor/issues/273
+
+
+Higher level "cluster" APIs
+---------------------------
+To be extra terse the ``tractor`` devs have started hacking some "higher
+level" APIs for managing actor trees/clusters. These interfaces should
+generally be condsidered provisional for now but we encourage you to try
+them and provide feedback. Here's a new API that let's you quickly
+spawn a flat cluster:
+
+.. code:: python
+
+    import trio
+    import tractor
+
+
+    async def sleepy_jane():
+        uid = tractor.current_actor().uid
+        print(f'Yo i am actor {uid}')
+        await trio.sleep_forever()
+
+
+    async def main():
+        '''
+        Spawn a flat actor cluster, with one process per
+        detected core.
+
+        '''
+        portal_map: dict[str, tractor.Portal]
+        results: dict[str, str]
+
+        # look at this hip new syntax!
+        async with (
+
+            tractor.open_actor_cluster(
+                modules=[__name__]
+            ) as portal_map,
+
+            trio.open_nursery() as n,
+        ):
+
+            for (name, portal) in portal_map.items():
+                n.start_soon(portal.run, sleepy_jane)
+
+            await trio.sleep(0.5)
+
+            # kill the cluster with a cancel
+            raise KeyboardInterrupt
+
+
+    if __name__ == '__main__':
+        try:
+            trio.run(main)
+        except KeyboardInterrupt:
+            pass
+
+
+.. _full worker pool re-implementation: https://github.com/goodboy/tractor/blob/master/examples/parallelism/concurrent_actors_primes.py
+
+
+Install
+-------
+From PyPi::
+
+    pip install tractor
+
+
+From git::
+
+    pip install git+git://github.com/goodboy/tractor.git
+
+
+Under the hood
+--------------
+``tractor`` is an attempt to pair trionic_ `structured concurrency`_ with
+distributed Python. You can think of it as a ``trio``
+*-across-processes* or simply as an opinionated replacement for the
+stdlib's ``multiprocessing`` but built on async programming primitives
+from the ground up.
+
+Don't be scared off by this description. ``tractor`` **is just** ``trio``
+but with nurseries for process management and cancel-able streaming IPC.
+If you understand how to work with ``trio``, ``tractor`` will give you
+the parallelism you may have been needing.
+
+
+Wait, huh?! I thought "actors" have messages, and mailboxes and stuff?!
+***********************************************************************
+Let's stop and ask how many canon actor model papers have you actually read ;)
+
+From our experience many "actor systems" aren't really "actor models"
+since they **don't adhere** to the `3 axioms`_ and pay even less
+attention to the problem of *unbounded non-determinism* (which was the
+whole point for creation of the model in the first place).
+
+From the author's mouth, **the only thing required** is `adherance to`_
+the `3 axioms`_, *and that's it*.
+
+``tractor`` adheres to said base requirements of an "actor model"::
+
+    In response to a message, an actor may:
+
+    - send a finite number of new messages
+    - create a finite number of new actors
+    - designate a new behavior to process subsequent messages
+
+
+**and** requires *no further api changes* to accomplish this.
+
+If you want do debate this further please feel free to chime in on our
+chat or discuss on one of the following issues *after you've read
+everything in them*:
+
+- https://github.com/goodboy/tractor/issues/210
+- https://github.com/goodboy/tractor/issues/18
+
+
+Let's clarify our parlance
+**************************
+Whether or not ``tractor`` has "actors" underneath should be mostly
+irrelevant to users other then for referring to the interactions of our
+primary runtime primitives: each Python process + ``trio.run()``
++ surrounding IPC machinery. These are our high level, base
+*runtime-units-of-abstraction* which both *are* (as much as they can
+be in Python) and will be referred to as our *"actors"*.
+
+The main goal of ``tractor`` is is to allow for highly distributed
+software that, through the adherence to *structured concurrency*,
+results in systems which fail in predictable, recoverable and maybe even
+understandable ways; being an "actor model" is just one way to describe
+properties of the system.
+
+
+What's on the TODO:
+-------------------
+Help us push toward the future of distributed `Python`.
+
+- Erlang-style supervisors via composed context managers (see `#22
+  <https://github.com/goodboy/tractor/issues/22>`_)
+- Typed messaging protocols (ex. via ``msgspec.Struct``, see `#36
+  <https://github.com/goodboy/tractor/issues/36>`_)
+- Typed capability-based (dialog) protocols ( see `#196
+  <https://github.com/goodboy/tractor/issues/196>`_ with draft work
+  started in `#311 <https://github.com/goodboy/tractor/pull/311>`_)
+- We **recently disabled CI-testing on windows** and need help getting
+  it running again! (see `#327
+  <https://github.com/goodboy/tractor/pull/327>`_). **We do have windows
+  support** (and have for quite a while) but since no active hacker
+  exists in the user-base to help test on that OS, for now we're not
+  actively maintaining testing due to the added hassle and general
+  latency..
+
+
+Feel like saying hi?
+--------------------
+This project is very much coupled to the ongoing development of
+``trio`` (i.e. ``tractor`` gets most of its ideas from that brilliant
+community). If you want to help, have suggestions or just want to
+say hi, please feel free to reach us in our `matrix channel`_.  If
+matrix seems too hip, we're also mostly all in the the `trio gitter
+channel`_!
+
+.. _structured concurrent: https://trio.discourse.group/t/concise-definition-of-structured-concurrency/228
+.. _multi-processing: https://en.wikipedia.org/wiki/Multiprocessing
+.. _trio: https://github.com/python-trio/trio
+.. _nurseries: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/#nurseries-a-structured-replacement-for-go-statements
+.. _actor model: https://en.wikipedia.org/wiki/Actor_model
+.. _trionic: https://trio.readthedocs.io/en/latest/design.html#high-level-design-principles
+.. _async sandwich: https://trio.readthedocs.io/en/latest/tutorial.html#async-sandwich
+.. _3 axioms: https://www.youtube.com/watch?v=7erJ1DV_Tlo&t=162s
+.. .. _3 axioms: https://en.wikipedia.org/wiki/Actor_model#Fundamental_concepts
+.. _adherance to: https://www.youtube.com/watch?v=7erJ1DV_Tlo&t=1821s
+.. _trio gitter channel: https://gitter.im/python-trio/general
+.. _matrix channel: https://matrix.to/#/!tractor:matrix.org
+.. _pdbp: https://github.com/mdmintz/pdbp
+.. _pdb++: https://github.com/pdbpp/pdbpp
+.. _guest mode: https://trio.readthedocs.io/en/stable/reference-lowlevel.html?highlight=guest%20mode#using-guest-mode-to-run-trio-on-top-of-other-event-loops
+.. _messages: https://en.wikipedia.org/wiki/Message_passing
+.. _trio docs: https://trio.readthedocs.io/en/latest/
+.. _blog post: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/
+.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency
+.. _structured chadcurrency: https://en.wikipedia.org/wiki/Structured_concurrency
+.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency
+.. _unrequirements: https://en.wikipedia.org/wiki/Actor_model#Direct_communication_and_asynchrony
+.. _async generators: https://www.python.org/dev/peps/pep-0525/
+.. _trio-parallel: https://github.com/richardsheridan/trio-parallel
+.. _msgspec: https://jcristharif.com/msgspec/
+.. _guest-mode: https://trio.readthedocs.io/en/stable/reference-lowlevel.html?highlight=guest%20mode#using-guest-mode-to-run-trio-on-top-of-other-event-loops
+
+
+.. |gh_actions| image:: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fgoodboy%2Ftractor%2Fbadge&style=popout-square
+    :target: https://actions-badge.atrox.dev/goodboy/tractor/goto
+
+.. |docs| image:: https://readthedocs.org/projects/tractor/badge/?version=latest
+    :target: https://tractor.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status
+
+.. |logo| image:: _static/tractor_logo_side.svg
+    :width: 250
+    :align: middle

docs/_static/tractor_logo_side.svg

@@ -0,0 +1,458 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 24.2.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 2670 1980" style="enable-background:new 0 0 2670 1980;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#0A0A0A;}
+	.st1{fill:#FCFCFB;}
+</style>
+<g>
+	<path class="st0" d="M2275.3,1000.1c-0.1,2.3,0,4.7,0,7c0,189.3,0,378.7,0,568c0,6,0.1,12.1-0.8,17.9c-2.4,15.2-12.9,17.4-23.6,10
+		c-45.4-31-91.2-61.4-136.9-91.9l-30.1-20.3c-9.8-6.3-19.2-13.2-29.2-19.3c-6.7-4.1-9.8-9.7-9.9-17.2c-0.1-9.3-0.6-18.7-0.3-28
+		c0.2-7.3-1.7-13-6.5-19c-18.6-23.3-36.2-47.3-56.9-69c-18-18.8-36.7-36.8-56.4-53.8c-31.8-27.5-65.4-52.7-101-74.9
+		c-32.7-20.4-66.8-38.4-102.2-54c-50.7-22.4-103.2-39.2-157.1-51.5c-36.7-8.4-73.8-14.4-111.2-18.5c-34.5-3.8-69-6.5-103.7-5.8
+		c-23.1,0.5-46.2-1.3-69.3,0.4c-22.9,1.8-45.9,3-68.7,5.6c-72.7,8.4-143.7,24.6-212.1,50.6c-65.5,24.8-127.4,57-184.9,97.4
+		c-46.8,32.8-90.1,69.8-129.5,111.1c-18.1,19-35.1,39.2-50.5,60.6c-3.9,5.5-5.5,11-5.3,17.6c0.4,9.3,0,18.7,0,28
+		c0,9.3-3.1,16.4-11.8,21.4c-10.4,5.9-19.9,13.4-30.1,19.8l-25.4,17.2c-11.6,7.8-23.3,15.5-34.9,23.3c-34.3,23-68.6,46-102.9,69
+		c-14.6,9.7-23.6,5.5-25.2-12.1c-0.5-5.3-0.5-10.7-0.5-16c0-189.7,0-379.3,0-569l-0.1-26.1c0.2-190.3,0.1-380.7,0.2-571
+		c0-5.7,0.4-11.3,0.5-17c0.1-3.7,1.2-7.3,3-10.5c3.4-5.9,9.3-7.7,15.4-4.8c2.7,1.3,5.1,3.1,7.6,4.8c44.3,29.6,88.5,59.3,132.8,88.9
+		l29.7,20.1c10.8,6.6,20.9,14.3,31.7,20.9c6.7,4.1,9.7,9.8,9.8,17.3c0.2,11,0.4,22,0.2,33c-0.1,5.2,1.2,9.6,4.2,13.9
+		c15.4,22.2,33,42.8,51.6,62.2c53.9,56.4,114.1,105.1,181.4,144.6c38.4,22.6,78.3,42.4,119.9,58.6c40,15.6,80.9,28.3,122.8,38
+		c52.7,12.1,106,19.3,160,22.6c18.3,1.1,36.6,2.1,54.9,1.5c17.7-0.6,35.5,1.1,53.2-0.1c17.6-1.1,35.2-2.3,52.8-3.6
+		c35.9-2.5,71.4-8.5,106.7-15.1c53.4-9.9,105.5-25,156-45.1c53.8-21.5,105.1-48.2,153.2-80.9c43.4-29.5,83.6-62.7,121.1-99.3
+		c27.3-26.6,50.3-56.8,73.9-86.6c2.9-3.6,3.6-7.6,3.6-12.1c-0.1-10.7,0.2-21.3,0.4-32c0.1-7.5,3.2-13.1,9.9-17.2
+		c10.2-6.2,19.9-13.4,30-19.8l23.2-16.3c4.2-1.3,7.4-4.3,11-6.7c42.9-28.7,85.8-57.4,128.7-86.1c1.1-0.7,2.3-1.4,3.3-2.3
+		c10.9-9.1,21.4-4.7,23.8,10.7c0.8,5.6,0.6,11.3,0.6,17c0,189.3,0,378.7,0,568L2275.3,1000.1z M740,1203.1c-2-0.2-4-0.3-6-0.5
+		c-0.5,2.5-4.8,2-4.2,5.8c2.9-1,6.4-1.2,4.2-5.8C735.8,1206,738.1,1202.3,740,1203.1c-0.6-0.7-0.5-1.3,0.2-1.7
+		c-1.1-0.3-1.1-0.1-0.4,0.8C740,1202.3,739.9,1202.8,740,1203.1z M1935.5,780.5c-0.2,0.2-0.4,0.4-0.6,0.4c-0.3,0.1-0.6,0-0.8,0
+		c0.4-0.2,0.8-0.5,1.1-0.8c1.4-0.9,2-1.9,0.5-3.3c3.2,1.1,6.1,1.2,7.9-2.8c-3.3-0.7-4.6,3.8-7.7,2.9
+		C1935.7,778.2,1935.6,779.4,1935.5,780.5z M1934.8,1202.5c-0.1-0.2-0.1-0.4-0.2-0.6c0,0,0.1-0.1,0.1-0.1c0,0.2,0.1,0.3,0.1,0.5
+		c3.1,1.6,5,5.7,11.3,5.1C1941.3,1205,1938.9,1202,1934.8,1202.5z M2072.4,516.6c-1,2.3-3.4,3-5.3,4.3c-7.4,5.1-8.2,7.2-5.3,17.8
+		c4.2-6.5,8-12.6,11.9-18.6c1.1-1.7,0.9-3.1-1.3-3.6c0,1,0.3,1.8,1.4,2c0.2,0,0.4-0.4,0.6-0.7C2073.8,517.4,2073.1,517,2072.4,516.6
+		z M599.6,517.5c-0.4-0.8-1.1-1.7-1.8-0.8c-0.8,0.8,0.1,1.5,1,1.6c2.4,8.9,7.5,16.2,13.4,24.1c2.3-16.5,1.9-17.3-10.7-24.4
+		C600.9,517.6,600.2,517.6,599.6,517.5z M534.9,1353.5c-20,21.8-37.3,44.1-53.4,67.4c-16.1,23.4-26.8,49.5-41.8,73.5
+		c9-21,17.4-42.2,28.9-61.9c16.9-28.7,35.4-56.4,57.5-81.5c0.9-1,1.6-2.1,2.6-3.1c2.9-2.9,3.3-6,2.3-10
+		c-9.1-33.8-14.6-68.3-19.2-103c-5-38-8.3-76.1-10.6-114.3c-2.5-39.9-3.4-79.9-3.9-119.8c-0.1-5.1-1.4-6.8-6.7-6.9
+		c-21.6-0.2-43.3-0.8-64.9-1.3c-2,0-4-0.4-6,0.8c0,195.2,0,390.5,0,585.6c2.1,0.6,3.1-0.4,4.1-1.1c52.7-33.8,105.3-67.7,158-101.4
+		c3.6-2.3,5.7-5.4,7.3-9.1c6-13,12.5-25.8,20.8-37.4c4.2-5.9,3.9-11.9,2.5-19.1c-2.4,1.8-4,2.9-5.5,4.2
+		c-13.9,12.6-32.1,8.7-42.7-2.9c-7.3-8-12.4-17.2-17.1-26.8C542.3,1375.7,539.4,1365.1,534.9,1353.5z M534.6,627.2
+		c3.6-5,4.3-10.7,6.3-15.9c6.2-15.8,12.6-31.5,25.2-43.7c10.8-10.5,26.8-12.6,38.9-2.8c2.3,1.9,4.3,5,7.9,5
+		c1.6-8.3-0.2-15.3-5.1-21.9c-7.4-9.9-12.7-21-17.8-32.3c-2.4-5.3-5.8-9.2-10.8-12.4c-50.5-32.2-100.9-64.7-151.3-97.1
+		c-2.4-1.5-4.5-3.9-8.7-3.9c0,3.7,0,7.2,0,10.7c0,187.9,0,375.8,0,563.7c0,2,0.2,4,0,6c-0.6,4.9,1.4,6.3,6.2,6.1
+		c13.6-0.6,27.3-0.7,41-1c8.3-0.2,16.7-0.6,25-0.4c4.9,0.1,6.7-1.8,6.1-6.4c-0.1-1,0-2,0-3c1.2-75,4.3-149.8,13.7-224.3
+		c4.6-36.3,10-72.5,19.6-107.9c1.6-5.7,0.4-9.9-3.4-14.4c-10.7-12.8-21.7-25.4-31-39.1c-19-28.1-36.8-57-49.6-88.7
+		c-2.2-5.4-5.5-10.4-6.2-16.6C465,538.3,494.6,586,534.6,627.2z M2060.9,1411.1c-2.3,8.5-0.1,15.1,4,21.4
+		c5.9,9.3,13.4,18.1,17.1,28.2c3.9,10.7,10.9,16.6,19.9,22.4c47.8,30.3,95.3,61.1,143,91.7c2.7,1.7,5,4.3,9.5,4.5c0-3.4,0-6.6,0-9.8
+		c0-188.3,0-376.6,0-565c0-2-0.1-4,0-6c0.2-3.4-1.1-5-4.6-4.8c-2.3,0.1-4.7,0-7,0.1c-20,0.6-40,1.2-60,1.9c-6.7,0.2-6.7,0.4-7,7.4
+		c0,1.3,0,2.7-0.1,4c-0.4,60-3,119.9-8.9,179.7c-5,50.1-11.3,99.9-24,148.7c-1.4,5.2,0.1,8.3,3.5,12.3c15,17.8,29.6,36.1,42.4,55.5
+		c20,30.4,36.1,62.9,49.3,96.8c3.2,8.2,5.7,16.6,6.8,25.4c-33.9-90.8-82.6-145-107.2-169.7c-4.1,11-7.5,22.1-12.7,32.5
+		c-4.9,9.9-10.3,19.5-18.8,26.8c-7.1,6.1-15.1,8.9-24.8,7.5C2073.2,1421.4,2068.3,1415.5,2060.9,1411.1z M2244.8,454.3
+		c0.6,4.5-0.2,6.7-0.8,8.9c-6.5,24.2-17,47-28.2,69.2c-18.9,37.2-43,71.2-70.4,102.8c-2.7,3.2-3.6,5.8-2.5,9.8
+		c7.9,29.6,13.1,59.8,17.3,90.2c6.4,45.6,10.2,91.3,12.8,137.3c2,35.3,2.6,70.6,3.1,105.9c0.1,7.6,0.2,7.6,7.6,7.8
+		c17,0.5,34,1,51,1.4c5,0.1,10,0.2,15,0.5c3.2,0.1,5.1-0.9,4.9-4.5c-0.1-1.7,0-3.3,0-5c0-189.6,0-379.3-0.1-568.9
+		c0-2.4,1.1-5.2-1.2-7.3c-0.8,0.1-1.6,0-2.1,0.4c-52.7,33.8-105.3,67.7-158,101.5c-2,1.3-3.7,2.6-4.7,4.9c-6.9,16-16.9,30.1-25.8,45
+		c-3.1,5.2-3.4,10.2-2,15.7c2.6,0.1,3.5-1.6,4.6-2.6c15.7-13.9,33-10.6,45.6,3.8c6,6.8,10.4,14.6,14.4,22.7
+		c5.1,10.3,8.3,21.3,12.8,32.9C2185.2,576.1,2221.7,520.5,2244.8,454.3z M553.7,662.8c-0.3,1.3-0.7,2.8-1,4.3
+		c-7.1,31.2-11.7,62.8-15.4,94.5c-7.1,59.9-10.3,120.1-11.8,180.4c-0.3,13-0.2,26-0.5,39c-0.1,4.4,1.5,5.7,5.7,5.6
+		c25-0.5,50-0.7,74.9-1.1c7.4-0.1,7.6-0.2,7.6-7.8c0-85.6,0-171.2,0.1-256.8c0-5.3-1.7-8.5-5.8-11.7c-16.6-12.8-32.4-26.6-46.9-41.9
+		C559,665.6,557.6,663,553.7,662.8z M553.6,1319.7c2.1-1,2.8-1.2,3.2-1.6c16.4-17.1,34-32.9,52.7-47.4c3-2.4,3.8-5,3.8-8.6
+		c-0.1-13.3-0.1-26.7-0.1-40c0-72.3,0-144.7,0-217c0-10.3,0.4-9-9.2-9.2c-23.3-0.4-46.7-0.8-70-1.1c-2.8,0-5.9-1-9.3,1.5
+		C526.2,1104.4,531.1,1212.3,553.6,1319.7z M2119.8,1319.2c4.8-20.9,8.3-40.2,11.1-59.6c5.8-39.5,9.8-79.2,12.5-119.1
+		c3.1-45.5,4.7-91.1,5.1-136.7c0.1-7.6-0.1-7.7-7.6-7.7c-24.3,0.1-48.6,0.3-72.9,0.5c-7.9,0.1-8,0.1-8,8.5c0,84.6,0,169.1-0.1,253.7
+		c0,5.6,1.7,9.3,5.9,12.7c12.1,9.9,24.2,19.9,36,30.1C2107.7,1306.9,2113.1,1312.7,2119.8,1319.2z M2120.4,664.4
+		c-2.1-0.6-2.9-0.1-3.6,0.7c-15.2,16.5-33.1,29.9-50.2,44.2c-4.8,4-6.8,8-6.8,14.3c0.3,32,0.2,64,0.2,96c0,52.3,0,104.6,0,157
+		c0,8.1,0.1,8.2,8.2,8.2c18.3,0.2,36.7,0.2,55,0.4c6.7,0.1,13.3,0.1,20,0.3c3.7,0.1,5.7-1.1,5.4-5.1c-0.2-2.3,0-4.7-0.1-7
+		c-0.5-45-2.1-89.9-5.2-134.8C2139.2,780.1,2132.7,721.9,2120.4,664.4z M878.7,1138.5c-36,15-69.8,32.9-102.3,53.1
+		c-3.4,2.1-4.9,4.7-5.7,8.6c-2.8,14.4-5.7,28.8-12.2,42c-3.6,7.3-7.7,14.4-17.3,14.4c-9.6,0-13.4-7.4-17.4-14.4
+		c-1.9-3.2-2-7.6-5.4-9.8c-22.3,17.4-41.6,37.2-58.8,58.9c-5.8,7.3-10.2,15.3-12,24.8c-3.8,20.3-9.1,40.2-16.7,59.5
+		c-2.6,6.7-1.3,13.3-1.5,20.3c1.5-0.4,2-0.3,2.1-0.5c3.2-3.8,6.4-7.6,9.4-11.6c31.2-40.5,66-77.4,105.4-110.2
+		c36.9-30.8,75.6-59.1,117.2-83.1c54.3-31.2,110.9-57.4,170.8-76.4c38.4-12.2,77.4-21.8,117-29.2c9.5-1.8,19.1-3,28.6-4.6
+		c1.5-0.3,3.8,0.3,4.4-1.5c1.4-3.9,0.6-8-0.4-11.7c-1-3.6-4.5-1.7-6.9-1.5c-40.6,3.2-80.4,11-120.4,18c-7,1.2-10.4,4.1-10.9,11.2
+		c-0.3,3.8-0.5,9.1-5.8,9c-4.6-0.1-4.7-5-5.3-8.5c-0.7-4.5-2.4-6-7.3-4.9c-39.3,9.1-78,19.9-115.5,34.7c-4.6,1.8-7.1,4.2-7.8,8.9
+		c-0.9,5.9-2.2,11.8-5.2,17.1c-4.1,7.1-10.6,7.2-14.6,0C881.9,1147.7,880.7,1143.4,878.7,1138.5z M1032.8,891.9
+		c0.9-3.8,1.6-6.7,2.4-9.5c0.7-2.3,1.7-4.6,4.4-4.9c3.4-0.3,4.4,2.3,5.3,4.9c0.4,1.3,0.8,2.6,0.8,3.9c0.1,7.7,4.5,10.4,11.7,11.6
+		c32.5,5.5,64.8,11.9,97.5,15.7c7.9,0.9,15.9,1.7,23.8,2.4c1.9,0.2,4,0.7,5.3-1.6c3.5-6.5,0.3-13-7-14.2c-1-0.2-2-0.2-3-0.4
+		c-42.6-5.8-84.2-15.9-125.4-28.2c-41.8-12.4-82.5-28.2-121.6-47.1c-51-24.6-99.8-53.3-144.7-87.9c-20.6-15.8-40.9-32.1-59.9-49.7
+		c-29.1-26.9-56.4-55.7-80.1-87.7c-3.7-4.9-7.7-9.6-11.6-14.3c-0.8,7.6-2.4,14.7,0.5,21.8c7.4,18.2,12.4,37.2,16,56.5
+		c1.7,9.1,5,17.3,10.6,24.4c9.2,11.4,18.5,22.8,28.7,33.3c9.9,10.1,21.1,19,32.2,28.9c3.8-6.5,5.6-13,9.8-18.2
+		c7.7-9.4,17.7-9.5,25.3-0.1c3.6,4.5,5.7,9.7,7.8,15c4.3,10.9,6.8,22.3,8.9,33.7c0.9,5.2,3.3,8.1,7.5,10.7
+		c21.9,13.4,44.1,26,67.3,37.1c10.9,5.2,22.1,10,33.5,15.2c1.8-4.8,3.1-8.6,4.7-12.2c1.6-3.5,4.2-6.3,8.3-6.1
+		c3.6,0.2,5.8,2.9,7.3,6.1c2.4,5.2,3.8,10.6,4.8,16.2c0.8,4.6,2.9,7.2,7.7,9C950.6,871.5,990.8,883,1032.8,891.9z M1953.8,1233.7
+		c-2.9,5.8-4.9,11.5-8.8,16.2c-7.6,9.3-17.8,9.3-25.3,0c-3.4-4.2-5.5-9-7.5-14c-4.4-11.5-7.2-23.5-9.4-35.7
+		c-0.7-3.9-2.4-6.4-5.8-8.6c-30.4-19.4-62.4-35.6-95-50.8c-6.9-3.3-7-3-9.9,4.7c-0.7,1.9-1.5,3.7-2.3,5.5c-1.6,3.1-3.7,5.6-7.6,5.6
+		c-3.8,0-5.9-2.5-7.6-5.6c-2.9-5.4-4.3-11.3-5.1-17.2c-0.7-5-3-7.3-7.5-9.1c-37.8-15.2-77.3-24.6-116.7-34.3c-3.8-0.9-5.4,0.2-6.2,4
+		c-0.7,3.9-0.2,9.9-6.3,9.5c-5-0.3-5.1-6.1-5.3-9.6c-0.4-6.6-3.3-8.9-9.6-10c-32.8-5.6-65.4-12.2-98.6-15.7
+		c-8.3-0.9-16.5-2.1-24.8-2.7c-5.4-0.4-7.1,1.2-7.5,6.6c-0.3,3.9-1.3,8,5.2,8.4c6.3,0.4,12.4,2.1,18.7,3.1
+		c47.7,7.5,94.4,19.1,140.3,34.3c42.4,14,83.5,31.2,123,52c46.3,24.4,90.2,52.6,131,85.6c48.5,39.1,93.7,81.2,129.4,132.6
+		c1.9,2.8,4.8,7.5,7.3,6.7c4.5-1.5,1.4-6.8,2.1-10.4c0.6-3.4-0.5-6.6-1.8-9.7c-7.8-19.5-12.6-40-17.1-60.4c-1.1-5.1-3-9.4-6-13.5
+		c-17-22.8-36.7-43.1-57.6-62.2C1959.6,1237.1,1957.6,1234.7,1953.8,1233.7z M2043.6,585.1c-4.2,2.7-6.2,6.2-8.5,9.2
+		c-29.9,40.3-64.6,76.1-102.5,109c-33.8,29.3-69.5,56.2-107.6,79.4c-51.8,31.6-106.3,57.7-163.9,77.3
+		c-40.7,13.8-82.1,24.9-124.2,33.1c-13.7,2.7-27.6,4.7-41.3,7c-2.2,0.4-5.3-0.1-6.2,2.4c-1.4,3.8-1.1,8.2,0.3,11.7
+		c1.4,3.5,5.3,1.6,8.1,1.3c40.1-4.2,79.8-10.7,119.5-18.1c6.2-1.2,10.4-2.7,10.5-10.2c0-3.5,0.3-9.3,5.4-9.5c6-0.3,5.5,5.7,6.2,9.6
+		c0.7,3.9,2.3,4.9,6.1,3.9c10-2.7,20-5.1,30-7.5c28.5-6.9,56.4-15.6,83.9-25.7c6.8-2.5,10.2-6.3,11-13.3c0.5-5,2-9.8,4.5-14.2
+		c4.2-7.6,10.8-7.6,15.1,0.1c1.6,2.9,2.5,6.2,3.4,9.3c0.9,2.9,2.5,3.5,5.2,2.2c2.4-1.2,5-1.9,7.3-3.1c26.6-13.1,53.2-26,78.7-41.2
+		c11.9-7.1,19.6-14.7,21.3-29.3c1.4-12.4,5.5-24.8,12.8-35.4c8.1-11.7,19.5-11.7,27.5,0.1c3.1,4.6,5.4,9.7,8.2,14.9
+		c1.9-1.4,3.6-2.5,5.1-3.8c22.2-19.8,42.7-41.1,60.6-64.9c2.6-3.5,4.3-7.3,5.2-11.7c4.5-21.2,9.6-42.2,17.6-62.4
+		C2045.3,599.2,2044.5,592.6,2043.6,585.1z M777.2,1163.8c1.2,0.2,1.6,0.4,1.9,0.3c30.4-14.3,60.9-28.3,92.6-39.6
+		c4.2-1.5,4.4-3.7,3.9-7.5c-4.5-37.4-6.3-75-6.5-112.6c0-7.8-0.2-7.9-8.5-7.9c-22,0-44,0-65.9,0c-8.6,0-8.8,0.1-8.8,8.7
+		c-0.4,45.3-2.2,90.5-6.6,135.6C778.4,1148.4,776.3,1155.9,777.2,1163.8z M776.8,817.8c0.3,3.8,0.5,7.1,0.9,10.4
+		c2.5,21.2,4.1,42.4,5.5,63.7c1.8,28.6,2.3,57.2,2.7,85.9c0.1,7.1,0.3,7.3,7.2,7.3c22.6,0.1,45.3,0.1,67.9,0.1
+		c8.8,0,8.1-0.2,8.1-8.3c0.1-37,1.9-73.9,6.4-110.6c0.7-5.8-0.5-8.2-6-10.1c-29.5-10.3-58-23.3-86.2-36.8
+		C781.5,818.4,779.9,817.2,776.8,817.8z M1896.4,1163.9c1-2.7,0.2-5.3-0.1-7.9c-2.4-19.8-3.9-39.8-5.3-59.7
+		c-2.1-30.3-2.5-60.6-3.3-90.9c-0.2-8.7-0.2-8.8-8.6-8.8c-16.7-0.1-33.3,0-50-0.1c-6.3,0-12.7,0.2-19-0.1c-3.9-0.1-5.5,1.4-5.2,5.3
+		c0.1,1.7,0.1,3.3,0,5c-1.1,36-2,72-6.4,107.7c-0.9,7.7-1.1,7.9,6.3,10.6c28.5,10.4,56.1,22.9,83.3,36.1
+		C1890.6,1162.4,1893.1,1164.4,1896.4,1163.9z M1896.8,817.7c-2.5-0.3-4.3,0.6-6.1,1.5c-6,2.9-12,5.6-18,8.6
+		c-22.1,10.9-44.9,20.1-67.9,28.7c-7.5,2.8-7.3,3-6.5,10.6c1,8.9,1.9,17.9,2.7,26.8c2.4,28.2,3.2,56.5,3.6,84.7
+		c0.1,5.4,1.8,6.7,6.8,6.6c22.6-0.2,45.3-0.1,67.9-0.2c1,0,2-0.1,3,0c3.8,0.3,5.6-1.2,5.2-5.1c-0.2-1.6,0-3.3,0.1-5
+		c0.7-38.6,1.7-77.2,5.2-115.6C1894.1,845.4,1895.5,831.6,1896.8,817.7z M908.3,1109.6c1.4,0,2.4,0.2,3.2,0
+		c37.8-11.7,75.5-24,114.2-32.5c4.5-1,5.8-2.9,5.5-7.4c-1.3-22-2.5-43.9-2.3-65.9c0.1-7.3-0.2-7.4-8-7.4c-33.3,0-66.7,0.1-100,0.2
+		c-7.7,0-7.9,0.1-7.9,7.5c0,31-1.6,61.9-4,92.9C908.8,1101,907.3,1105.2,908.3,1109.6z M1765.3,1110.4c0-2.5,0.1-3.5,0-4.4
+		c-3.1-33.9-4.7-67.8-4.8-101.8c0-7.4-0.4-7.6-8.2-7.7c-33.3-0.1-66.7-0.2-100-0.2c-7.3,0-7.2,0.1-7.5,7.9c-0.7,21-1.1,42-2.3,62.9
+		c-0.5,9-1.3,8.3,7.6,10.4c29.8,7.1,59.4,14.9,88.5,24.5C1747.2,1104.8,1755.7,1107.4,1765.3,1110.4z M908.8,872.2
+		c-0.2,0.3-0.6,0.6-0.6,0.8c2.8,35.5,5.2,71,4.8,106.7c-0.1,4.6,1.9,5.6,6,5.6c34.7-0.1,69.3-0.1,104,0.1c4.9,0,6.2-1.8,6.1-6.4
+		c-0.5-22,0.8-44,2.1-65.9c0.3-5.2-0.9-7.7-6.4-8.8c-36.9-7.6-72.4-20.1-108.5-30.5C913.8,873,911.6,871.1,908.8,872.2z
+		 M1765.6,872.4c-3.2-0.6-5.1,0.3-7,0.9c-22.2,6.7-44.2,14.5-66.7,20.3c-14.5,3.7-29,7.3-43.5,10.8c-4.4,1.1-6.5,2.6-6.2,8
+		c1.3,21.9,1.8,43.9,2.6,65.8c0.2,7,0.2,7.1,7.1,7.1c33.6,0,67.2,0,100.8-0.1c7.5,0,7.7-0.2,7.7-7.8c0.2-24.6,0.9-49.2,2.6-73.8
+		C1763.8,893.4,1764.7,883.1,1765.6,872.4z M725.8,1192.5c8.7-4.8,15.8-9.8,23.8-12.8c9.6-3.7,12.7-10.5,13-19.7
+		c0.1-1.3,0.2-2.7,0.4-4c3.4-27.1,5.6-54.3,6.7-81.5c1-23.6,1.5-47.2,2-70.8c0.2-7-0.1-7-7.3-7.1c-6.7-0.1-13.3,0-20,0
+		c-9,0-18,0.2-27,0c-4.2-0.1-6.2,1.3-5.9,5.7c0.2,2.3,0,4.7,0.1,7c0.8,27.6,1.2,55.2,3,82.8C717,1125.1,719.9,1158.2,725.8,1192.5z
+		 M1946.2,1191.7c2.9-2.2,2.4-4.7,2.8-6.9c4.5-26.6,6.9-53.5,8.9-80.4c2.5-33.9,3.2-67.9,3.7-101.8c0.1-5-1.8-6.2-6.4-6.1
+		c-15.3,0.2-30.6,0-46,0.1c-7.6,0-7.8,0.2-7.7,7.8c0.7,44,2,87.9,6.7,131.6c1.1,10.3,2.2,20.5,3.5,30.7c0.4,2.8,0.2,6,3.5,7.8
+		C1925.8,1180.2,1936.2,1186.1,1946.2,1191.7z M1947.7,789.1c-9.1,5-16.9,10.1-25.3,13.8c-8.2,3.6-10.5,9.5-11.5,17.8
+		c-3,27.1-5.6,54.3-6.9,81.6c-1.2,25.3-1.5,50.6-2.5,75.9c-0.2,5.1,1.5,7,6.7,7c15-0.3,30-0.1,45-0.2c8.6,0,8.4,0.6,8.3-8.9
+		c-0.3-30.6-1-61.3-3.1-91.9C1956.3,852.9,1953.6,821.8,1947.7,789.1z M726,789.2c-3.3,16.4-5.1,31.3-6.7,46.2
+		c-4.6,42.4-6.3,85-7.3,127.7c-0.1,5.7,0,11.3-0.2,17c-0.1,3.4,1.4,4.8,4.7,4.8c17,0,34,0,51,0.1c3.6,0,4.6-1.9,4.4-5
+		c-0.1-2.3-0.1-4.7-0.1-7c-0.5-37.3-1.7-74.6-5-111.8c-1.4-15.9-2.8-31.9-5.5-47.6c-0.4-2.6-0.3-5.2-3.2-6.8
+		C747.6,801.1,737.2,795.4,726,789.2z M1625.1,910.5c-9.8,1.8-18,3.4-26.2,4.6c-23.3,3.5-46.2,9.9-70,10
+		c-10.9,0.1-21.8,2.4-32.7,3.7c-3.5,0.4-6.2,1.1-6,6c0.5,15,0.7,30,0.7,44.9c0,4.4,1.7,5.8,5.9,5.7c8.3-0.2,16.6,0,25-0.1
+		c30.6,0,61.3,0,91.9-0.1c9,0,9-0.1,9.2-9.4c0.2-13,0.2-26,0.6-38.9C1623.7,928.4,1624.5,919.8,1625.1,910.5z M1625.2,1071.1
+		c-2.2-23.8-2.3-46.1-2.4-68.4c0-4.5-1-6.6-6.1-6.6c-39.9,0.1-79.9,0.1-119.8-0.1c-5.1,0-6.1,2-6.1,6.5c0,14.6-0.1,29.3-0.6,43.9
+		c-0.1,4.5,1.6,5.6,5.6,6.1c12.6,1.4,25,3.8,37.7,4c8,0.1,15.9,1.1,23.8,2.5C1579.5,1063,1601.7,1066.9,1625.2,1071.1z
+		 M1049.3,1070.4c2.6,1,4.9-0.1,7-0.6c23-5.5,46.3-8.9,69.7-12.3c16.7-2.5,33.7-3.8,50.6-5.2c5.1-0.4,6.8-1.9,6.6-7
+		c-0.4-14-0.3-27.9-0.5-41.9c-0.1-7-0.2-7.2-7.2-7.2c-39.3,0-78.5,0-117.8,0.1c-6.7,0-6.8,0.2-6.8,7.5c0,16.3-0.5,32.6-1.4,48.9
+		C1049.3,1058.6,1048.4,1064.5,1049.3,1070.4z M1049.8,910.2c-0.4,1.2-0.8,1.8-0.8,2.4c0.6,22.6,2.5,45.2,1.9,67.8
+		c-0.1,4.2,2.1,4.8,5.6,4.8c39.9,0,79.9-0.1,119.8,0.1c5.5,0,6.7-2.3,6.6-7.2c-0.2-11,0.1-22,0.2-32.9c0.1-15.9,0.1-15.5-16.3-17.4
+		c-16.5-1.9-33.1-2.4-49.6-5.2C1094.7,918.9,1072,915.9,1049.8,910.2z M1260.5,985.4c21.6,0,43.2,0,64.9,0c9.1,0,9.1-0.1,9.1-9.4
+		c0-10.3,0-20.6-0.1-30.9c-0.1-7.7-0.1-7.7-7.6-7.9c-2.3-0.1-4.7,0-7,0c-41.9,0-83.8-2.2-125.5-6.2c-4.6-0.4-5.9,0.9-6,5.1
+		c-0.2,13.6-0.6,27.3-0.8,40.9c-0.1,8.3,0.1,8.4,8.1,8.4C1217.3,985.4,1238.9,985.4,1260.5,985.4z M1485,931.5
+		c-3.1-1.4-5.4-0.6-7.7-0.4c-26.2,1.9-52.4,4.3-78.7,5.1c-17.3,0.5-34.6,1.3-51.9,1.1c-7.4-0.1-7.5,0.1-7.6,7.8
+		c-0.1,11.7,0,23.3-0.1,35c0,3.7,1,5.8,5,5.5c2-0.1,4,0,6,0c41.6,0,83.2,0,124.9,0c2,0,4-0.1,6,0c3.3,0.1,5-1.2,4.9-4.6
+		C1485.5,964.2,1485.3,947.6,1485,931.5z M1485,1051.1c0.3-17,0.5-33.6,0.9-50.1c0.1-4.2-2.2-4.8-5.7-4.8c-45,0.1-89.9,0.1-134.9,0
+		c-4.7,0-6.5,1.4-6.3,6.2c0.3,12,0.4,24,0,36c-0.2,5.2,2.1,6,6.5,6c11.6-0.1,23.3,0.1,34.9,0.4
+		C1415,1045.8,1449.5,1048.1,1485,1051.1z M1188.4,1050.6c47.5-3.4,93.2-6.8,139.2-6.4c0.7,0,1.3,0,2,0c3.2,0.2,4.9-0.9,4.9-4.5
+		c-0.1-13,0-25.9,0-38.9c0-3.4-1.6-4.8-4.9-4.6c-1.7,0.1-3.3,0-5,0c-42.6,0-85.1,0-127.7,0c-1.7,0-3.3,0.1-5,0
+		c-2.8-0.1-4.4,1.1-4.4,4.1C1187.9,1016.8,1188.1,1033.4,1188.4,1050.6z M2010.7,1230.1c0.9-7.3-0.9-13.8-1.5-20.3
+		c-6.8-66-10.3-132.2-10.7-198.6c-0.1-16.1-0.3-16.3-16.8-14.7c-2.9,0.3-4.2,1.1-4.1,4.1c0.1,3,0,6,0,9
+		c-0.3,41.3-2.1,82.6-6.1,123.8c-2,20.2-4.3,40.4-8.2,60.4c-0.9,4.8-0.2,7.8,4.5,10.4c10.2,5.7,19.9,12.2,29.8,18.3
+		C2001.8,1225,2005.6,1228.2,2010.7,1230.1z M2011.2,752.5c-2.7-0.8-3.6,0.3-4.7,1c-13.3,8.3-26.4,16.7-39.8,24.7
+		c-3.6,2.1-4.2,4.5-3.5,8.2c1.4,7.8,2.8,15.7,3.9,23.6c7.4,53.8,10.1,107.9,10.6,162.2c0.1,14.1-2.6,12.8,13.7,12.8
+		c7,0,7-0.2,7.1-7.3c0.4-27.3,0.6-54.6,1.8-81.8c1.4-30.9,3.1-61.8,6.1-92.6C2008,786.3,2009.6,769.5,2011.2,752.5z M662.1,1231.5
+		c3.7-2.4,5.6-3.7,7.5-4.9c11.6-7.7,22.9-16.2,35.5-22.2c5.3-2.5,6.2-5.5,5.1-11c-4.6-22.5-7-45.4-9-68.3
+		c-3.5-39.9-5-79.8-5.4-119.8c-0.1-8.6-0.2-8.6-8.8-8.7c-1.3,0-2.7,0-4,0c-7.9,0.1-8,0.1-7.9,8.4c0,24.7-0.6,49.3-1.6,74
+		c-1.3,32-3,63.9-6,95.8C665.8,1192.9,664.1,1211.1,662.1,1231.5z M662.9,752.1c-1,1.4-0.4,3-0.3,4.6c2,20.5,4.6,41.1,6,61.7
+		c3.8,52.5,6.5,105.1,6.4,157.8c0,8.8,0.1,8.9,8.5,8.9c14.5-0.1,12.1,1.3,12.3-12.1c1.1-62.7,3.5-125.2,14.8-187
+		c0.8-4.1-0.9-6-4.2-7.7c-13.4-6.8-25.6-15.5-38-23.9C666.8,753.1,665.5,751.1,662.9,752.1z M2044.3,728.6c-5.4,1.1-7.1,3.9-7.7,8.3
+		c-1.7,14.9-4,29.7-5.6,44.6c-4.6,42.1-7.2,84.2-8.9,126.5c-1,25.3-1.3,50.5-1.3,76.8c6.5,0,12.4-0.1,18.3,0
+		c3.8,0.1,5.5-1.4,5.2-5.3c-0.2-1.7,0-3.3,0-5c0-78.9,0-157.9,0-236.8C2044.3,734.9,2044.3,732,2044.3,728.6z M2044.3,1253
+		c0-3.7,0-6.4,0-9c0-78.6,0-157.2,0-235.8c0-13,1.3-11.5-11.9-11.5c-13,0-11.5-1.8-11.5,11.8c0,59.3,3.1,118.5,8.7,177.6
+		c1.9,19.5,4,39.1,7,58.5C2037.3,1248.9,2038.8,1251.8,2044.3,1253z M629.2,728.7c0,4.3,0,8,0,11.6c0,76.5,0,153.1,0,229.6
+		c0,15.8,0,15.8,16,15.3c7.6-0.2,7.6-0.2,7.7-7.7c0.1-3.3,0-6.7,0-10c-0.8-55.6-3.1-111.1-8.5-166.4c-2-21.2-5-42.3-7.3-63.4
+		C636.6,733.3,635,730.3,629.2,728.7z M629.2,1253.2c5.3-2.1,7.4-4.5,7.9-9.2c1.6-14.9,4-29.7,5.5-44.6
+		c6.5-62.4,9.5-124.9,10.3-187.6c0.2-15.6,0.1-15.6-15-15.6c-9,0-8.8-0.6-8.7,8.6c0,2,0,4,0,6c0,75.7,0,151.3,0,227
+		C629.2,1242.5,629.2,1247.2,629.2,1253.2z M2060,697.5c19-15.9,36.7-29.3,52.3-45.3c3.5-3.5,3.7-6.7,2.4-10.7
+		c-2.8-8.9-5.4-17.8-8.7-26.5c-3.5-9.3-7.4-18.5-13.7-26.4c-5.3-6.6-8-6.6-13.4-0.4c-1.3,1.5-2.7,3-3.6,4.7
+		c-8.9,15.7-17.3,31.5-15.3,50.5c0.4,3.6,0.1,7.3,0.1,11C2060,668,2060,681.5,2060,697.5z M613.2,1283.1
+		c-20.8,13.6-36.9,29.9-53.8,45.4c-2.8,2.6-2.3,5.1-1.5,8.1c4.1,14.7,8.6,29.3,15.3,43.2c2.6,5.4,5.4,10.7,9.8,15
+		c4.3,4.2,6.2,4.6,10,0.4c10.7-11.9,18.4-26.3,19.7-41.7C614.7,1330.6,613.2,1307.6,613.2,1283.1z M2060,1284.4
+		c0,19.5,0.7,36.8-0.2,54c-1.1,19.1,6.3,35.2,15.8,50.5c7.4,12.1,12.1,11.7,19.8-0.6c9.7-15.4,14.6-32.7,19.7-50
+		c1.1-3.7,0-6.2-2.6-8.8C2096.8,1313.6,2079.3,1299.9,2060,1284.4z M611.9,698.2c2.1-3.3,1.3-5.9,1.3-8.4c0.1-16-0.8-32,0.3-47.9
+		c1.2-18.4-6.2-33.8-15.1-48.6c-7.6-12.7-12.6-12.4-20.4,0.3c-10.1,16.3-15.1,34.6-20.4,52.8c-1,3.6,0.6,5.5,3,7.7
+		c8.8,8.1,17.3,16.7,26.3,24.5C594.8,685.4,603.3,691.5,611.9,698.2z M1788.2,864c-12.5,3.2-12.6,3.2-13.9,13.8
+		c-0.9,7.6-1.6,15.2-2.1,22.8c-1.8,26.2-2.9,52.5-3.2,78.8c0,4,0.8,6.3,5.4,6c4.6-0.3,9.3-0.2,14,0c5.1,0.1,6.9-2.1,6.7-7.3
+		c-0.5-11.3-0.2-22.6-0.7-33.9C1793.2,917.5,1792.3,890.9,1788.2,864z M885.3,1118.4c12.8-3,12.5-3,13.8-13.9
+		c3.5-29.5,4.6-59.1,5-88.8c0.3-23.5,5.2-18.7-19.8-19.4c-4.4-0.1-5.7,1.5-5.7,5.8C878.8,1040.8,880.3,1079.4,885.3,1118.4z
+		 M885.7,863.5c-0.2,0.9-0.6,1.9-0.7,2.9c-4.3,33.7-6,67.5-6.1,101.4c0,5.5-2.5,12.6,1.3,16.2c3.8,3.6,10.8,1.1,16.3,1.2
+		c7.9,0.1,8.1-0.1,7.9-8.4c-0.6-33.2-1.5-66.4-5.3-99.4C897.9,865.6,898,865.6,885.7,863.5z M1788.4,1117.5
+		c4.4-34.4,5.9-68.6,6.5-102.9c0.1-5.6,2.4-12.7-0.9-16.5c-3.9-4.3-11.3-1.4-17.2-1.6c-7.6-0.2-7.7,0.1-7.7,7.6
+		c0,24.7,1.3,49.3,2.8,73.9c0.6,10.3,2.2,20.5,2.9,30.8c0.3,3.6,1.7,5.5,4.9,6.5C1782.2,1116,1784.4,1117.7,1788.4,1117.5z
+		 M1481.1,1078.1c0-2.7,0.1-4.4,0-6c-0.3-7.8-0.3-8.1-8.2-8.7c-22.9-1.7-45.8-3.3-68.7-4.7c-19.9-1.2-39.9-0.9-59.9-1
+		c-4.6,0-5.5,2-5.4,6c0.1,3.7,0.3,6.7,5.2,6.2c1.3-0.1,2.7,0,4,0c34.3-0.3,68.4,1.9,102.5,5.6
+		C1460.6,1076.6,1470.4,1078.5,1481.1,1078.1z M1480,903c-15.3,1.6-30.8,3.6-46.3,4.8c-29.5,2.4-59.1,4.7-88.8,3.9
+		c-5.1-0.1-6.1,2-6,6.5c0.1,4.1,1.1,6.3,5.6,5.8c1.3-0.2,2.7,0,4,0c42.3,0.3,84.5-2.6,126.7-5.9c4.2-0.3,6.2-1.6,5.8-5.9
+		C1480.7,909.3,1482,906.3,1480,903z M1192.2,1078.2c3.1-0.2,5.4-0.3,7.6-0.5c37-4.6,74-7.6,111.3-7.8c6,0,12-0.1,18,0
+		c3.7,0.1,5.3-1.1,5.4-5.1c0.1-6.5,0-6.9-7.3-7c-40.6-0.8-81.1,1.7-121.6,4.6C1193.1,1063.3,1193.2,1063.6,1192.2,1078.2z
+		 M1192.6,903.2c0.2,14.4,0.2,14.4,12.8,16c3,0.4,6,0.6,8.9,0.8c19.3,1.1,38.5,2,57.8,3.2c18.6,1.2,37.2,0.8,55.9,0.8
+		c6.4,0,6.3-0.5,6.6-6.8c0.2-5.8-3.3-5.5-7.1-5.4c-28.3,0.8-56.5-1.5-84.7-3.6C1226.3,907,1209.8,905.4,1192.6,903.2z M653.6,694.6
+		c2,12.3,3.8,22.4,5.1,32.6c0.5,3.7,1.7,6.1,4.8,8c14.2,8.8,27.5,18.8,42.2,26.6c2.7,1.4,6.2,6.3,8.9,2.4c2.7-3.7-2.6-6.2-5-8.5
+		c-19-17.3-36.7-35.8-51.8-56.7C657,698.1,656,697.3,653.6,694.6z M654,1284.6c0.8,0,1.3,0.1,1.5,0c1.4-1.4,2.9-2.7,4-4.3
+		c14.6-20.8,32.8-38.3,51.2-55.5c1.2-1.1,2.6-2.1,3.6-3.4c1.4-1.7,1.5-3.6-0.2-5.7c-2.9,0.3-5.3,2.1-7.8,3.6
+		c-10.2,6.3-19.8,13.5-30.5,18.8c-12.9,6.4-19.3,15.7-19.2,30.2C656.6,1273.7,654.9,1279.1,654,1284.6z M2019.1,698.8
+		c-0.5-0.3-1-0.6-1.5-0.9c-1,1.2-2,2.4-3,3.6c-6.8,9-13,18.4-21.4,26.1c-11,10.1-20.5,21.7-32.4,30.8c-1.9,1.5-3.6,3.2-2.1,5.9
+		c2,3.5,4.1,0.5,5.7-0.4c15.4-9.7,30.8-19.6,46.1-29.5c2.3-1.5,4-3.1,4.4-6.2C2016.2,718.5,2017.7,708.6,2019.1,698.8z
+		 M2017.1,1283.9c0.7-0.2,1.4-0.5,2.1-0.7c-1.5-10.1-3-20.2-4.4-30.4c-0.4-2.6-1.9-4.1-3.9-5.4c-15.6-10-31.3-20.1-46.9-30
+		c-1.5-1-3.3-2.8-5-0.5c-1.7,2.4-0.3,4.2,1.7,5.9c4,3.4,8.1,6.9,11.8,10.6C1988.4,1249.3,2005.3,1264.3,2017.1,1283.9z M1044.6,1034
+		c0.2,0,0.3,0,0.5,0c0-11-0.1-21.9,0.1-32.9c0.1-4.2-2-4.8-5.5-4.9c-4.5-0.1-4.6,2.6-4.6,5.9c0.1,22.3,1.2,44.5,2.4,66.8
+		c0.1,1,0.2,2,0.4,3c0.2,1.2,1.2,1.8,2.1,1.7c0.8-0.1,1.9-1,2.2-1.7c0.5-1.5,0.7-3.2,0.8-4.9C1043.5,1055.9,1044,1045,1044.6,1034z
+		 M1636.4,1033.8c0.7,0,1.4,0,2.1,0c0-10.6,0-21.2,0-31.8c0-2.7,0.2-5.3-3.6-5.8c-4.7-0.6-6,0.4-6,6.1c-0.1,21.9,0.2,43.7,2.1,65.6
+		c0.1,1.3,0.1,2.7,0.6,3.9c0.3,0.7,1.5,1.6,2.2,1.5c0.7,0,1.7-1.1,1.9-1.8c0.5-1.9,0.7-3.9,0.7-5.9
+		C1636.4,1055,1636.4,1044.4,1636.4,1033.8z M1638.5,948.1c-0.2,0-0.5,0-0.7,0c-0.5-11.6-1-23.3-1.6-34.9c-0.1-1.9,0.2-4.8-2.3-4.9
+		c-3.3-0.1-2.8,3-3,5.2c-0.3,3-0.4,6-0.7,9c-1.5,19-1.4,37.9-1.4,56.9c0,5.3,2,6.8,6.4,5.8c3.8-0.8,3.3-3.6,3.3-6.2
+		C1638.5,968.8,1638.5,958.4,1638.5,948.1z M1045.1,948c-0.1,0-0.3,0-0.4,0c-0.6-11.9-1.2-23.9-1.9-35.8c-0.1-1.9-0.4-4.3-3.2-3.9
+		c-2.2,0.3-1.9,2.4-2,4c-1.3,22.5-2.3,45.1-2.5,67.7c0,3.7,1.2,5.4,5.1,5.4c4.1,0.1,5-1.9,5-5.5C1045,969.3,1045.1,958.7,1045.1,948
+		z M741.9,740.1c-4.9,5.5-6.3,11-8.3,16.2c-0.8,2-0.8,4,1.2,5.3c6.2,4.1,11.6,9.4,19.7,13C751.6,762.1,748.8,750.9,741.9,740.1z
+		 M1931.7,1241.3c11.7-16.6,11.5-18.2-3-28.3c-2.7-1.9-4.9-4.5-9.1-4.9C1921.9,1219.6,1924.7,1230.7,1931.7,1241.3z M1931.6,740
+		c-6.6,10.9-9.8,22-12.1,33.9c4-0.9,6.1-3.1,8.5-4.8C1943.5,758.3,1943.6,757.2,1931.6,740z M742.1,1241.3
+		c6.6-11.1,9.8-21.9,12.2-34.3c-8,3.6-13.2,8.8-19.2,12.6c-2,1.3-2.3,3.1-1.6,5.1C735.6,1230.2,736.7,1236.2,742.1,1241.3z
+		 M597.6,1464.4c16.9-6.5,17.3-8.9,14-24.4C605.8,1447.2,601.7,1455,597.6,1464.4z M2061.5,1442.5c-5.1,14.3,5.6,17.3,12.5,22.8
+		C2072.1,1457,2066.2,1451,2061.5,1442.5z M1781.6,836.9c-1,5.2-3.6,8.5-1.8,12.6C1785.7,846.2,1785.7,846.2,1781.6,836.9z
+		 M895.4,849.3c-1.5-4.5-1.6-7.9-3.8-11.1C887.9,846.3,887.9,846.3,895.4,849.3z M630.1,1284.6c-0.4-5.3,4.1-10.2,1.8-17
+		C627.4,1273.6,629.9,1279.2,630.1,1284.6z M629.9,697.1c0.2,5.4-3,11.4,2.4,16.6C633.8,707.4,630.4,702.5,629.9,697.1z
+		 M2044.5,1284.5c-1-5.2,1.5-10.3-2.8-15C2040.3,1275.4,2042.4,1279.6,2044.5,1284.5z M2041.9,712.1c4.1-4.8,1.6-10,2.2-14.6
+		C2043,702.1,2039.8,706.2,2041.9,712.1z M892.2,1143.9c0.4-4.2,3.5-7.3,1.6-11.5C887.6,1136,887.6,1136,892.2,1143.9z
+		 M1779.6,1132.2c-1,2.9-0.4,4.8,0.4,6.5c0.7,1.5,0.4,3.6,2.3,4.6C1785.7,1135.3,1785.7,1135.3,1779.6,1132.2z M729.5,774
+		c1.5,1.6,2.6,4,5.8,3.2C733.9,774.7,732.2,773.5,729.5,774z M737.9,778.4c-0.2,0.2-0.5,0.4-0.7,0.6c0.2,0.2,0.4,0.6,0.6,0.6
+		c0.2,0,0.5-0.3,0.7-0.5C738.4,778.8,738.1,778.6,737.9,778.4z"/>
+	<path class="st1" d="M534.9,1353.5c4.5,11.6,7.4,22.2,12.2,32c4.7,9.6,9.8,18.8,17.1,26.8c10.6,11.6,28.8,15.5,42.7,2.9
+		c1.4-1.3,3.1-2.4,5.5-4.2c1.4,7.2,1.7,13.2-2.5,19.1c-8.4,11.7-14.8,24.4-20.8,37.4c-1.7,3.7-3.7,6.7-7.3,9.1
+		c-52.7,33.7-105.3,67.6-158,101.4c-1.1,0.7-2.1,1.7-4.1,1.1c0-195.1,0-390.4,0-585.6c2.1-1.2,4.1-0.9,6-0.8
+		c21.6,0.5,43.3,1.1,64.9,1.3c5.3,0,6.7,1.8,6.7,6.9c0.5,40,1.4,79.9,3.9,119.8c2.4,38.2,5.6,76.4,10.6,114.3
+		c4.6,34.7,10.1,69.1,19.2,103c1.1,4,0.6,7.1-2.3,10c-0.9,0.9-1.7,2.1-2.6,3.1c-22.1,25.1-40.6,52.7-57.5,81.5
+		c-11.6,19.7-19.9,41-28.9,61.9c14.9-23.9,25.7-50.1,41.8-73.5C497.6,1397.6,514.9,1375.3,534.9,1353.5z"/>
+	<path class="st1" d="M534.6,627.2c-40-41.2-69.6-89-94.2-140.1c0.7,6.2,4,11.2,6.2,16.6c12.7,31.7,30.5,60.6,49.6,88.7
+		c9.3,13.7,20.4,26.3,31,39.1c3.8,4.5,5,8.7,3.4,14.4c-9.6,35.4-15,71.6-19.6,107.9c-9.4,74.5-12.5,149.3-13.7,224.3c0,1-0.1,2,0,3
+		c0.6,4.7-1.2,6.5-6.1,6.4c-8.3-0.2-16.7,0.2-25,0.4c-13.7,0.3-27.3,0.4-41,1c-4.8,0.2-6.8-1.2-6.2-6.1c0.3-2,0-4,0-6
+		c0-187.9,0-375.8,0-563.7c0-3.5,0-7.1,0-10.7c4.2-0.1,6.3,2.3,8.7,3.9c50.4,32.4,100.8,64.9,151.3,97.1c5,3.2,8.4,7.1,10.8,12.4
+		c5.1,11.2,10.4,22.4,17.8,32.3c4.9,6.6,6.7,13.5,5.1,21.9c-3.6-0.1-5.6-3.1-7.9-5c-12-9.8-28.1-7.7-38.9,2.8
+		c-12.6,12.2-19,27.9-25.2,43.7C538.8,616.5,538.2,622.2,534.6,627.2z"/>
+	<path class="st1" d="M2060.9,1411.1c7.4,4.3,12.3,10.3,20.6,11.4c9.8,1.4,17.7-1.4,24.8-7.5c8.5-7.3,13.9-16.9,18.8-26.8
+		c5.1-10.4,8.6-21.5,12.7-32.5c24.7,24.7,73.3,78.9,107.2,169.7c-1.1-8.8-3.5-17.2-6.8-25.4c-13.2-33.9-29.3-66.4-49.3-96.8
+		c-12.8-19.5-27.4-37.7-42.4-55.5c-3.4-4-4.8-7.1-3.5-12.3c12.8-48.8,19.1-98.6,24-148.7c5.9-59.7,8.4-119.6,8.9-179.7
+		c0-1.3,0-2.7,0.1-4c0.2-7,0.2-7.1,7-7.4c20-0.7,40-1.3,60-1.9c2.3-0.1,4.7,0,7-0.1c3.5-0.2,4.8,1.4,4.6,4.8c-0.1,2,0,4,0,6
+		c0,188.3,0,376.6,0,565c0,3.2,0,6.4,0,9.8c-4.5-0.2-6.9-2.8-9.5-4.5c-47.7-30.6-95.2-61.4-143-91.7c-9-5.7-16-11.6-19.9-22.4
+		c-3.7-10.1-11.2-18.9-17.1-28.2C2060.8,1426.2,2058.6,1419.6,2060.9,1411.1z"/>
+	<path class="st1" d="M2244.8,454.3c-23,66.2-59.6,121.8-106.6,172.4c-4.5-11.6-7.7-22.6-12.8-32.9c-4-8.1-8.4-15.8-14.4-22.7
+		c-12.6-14.4-29.9-17.7-45.6-3.8c-1.1,1-2,2.7-4.6,2.6c-1.4-5.5-1.1-10.5,2-15.7c8.8-14.9,18.9-29,25.8-45c1-2.3,2.7-3.7,4.7-4.9
+		c52.7-33.9,105.3-67.7,158-101.5c0.5-0.3,1.3-0.2,2.1-0.4c2.3,2.1,1.2,4.9,1.2,7.3c0.1,189.6,0.1,379.3,0.1,568.9
+		c0,1.7-0.1,3.3,0,5c0.2,3.6-1.7,4.6-4.9,4.5c-5-0.2-10-0.3-15-0.5c-17-0.5-34-1-51-1.4c-7.4-0.2-7.5-0.2-7.6-7.8
+		c-0.5-35.3-1.1-70.6-3.1-105.9c-2.6-45.9-6.4-91.7-12.8-137.3c-4.2-30.3-9.4-60.5-17.3-90.2c-1.1-4-0.2-6.6,2.5-9.8
+		c27.4-31.6,51.5-65.6,70.4-102.8c11.3-22.2,21.7-44.9,28.2-69.2C2244.6,460.9,2245.4,458.8,2244.8,454.3z"/>
+	<path class="st1" d="M553.7,662.8c3.8,0.2,5.2,2.8,6.9,4.6c14.5,15.2,30.3,29,46.9,41.9c4.1,3.2,5.8,6.4,5.8,11.7
+		c-0.2,85.6-0.1,171.2-0.1,256.8c0,7.5-0.1,7.7-7.6,7.8c-25,0.4-50,0.6-74.9,1.1c-4.3,0.1-5.9-1.2-5.7-5.6c0.4-13,0.2-26,0.5-39
+		c1.5-60.3,4.7-120.5,11.8-180.4c3.7-31.7,8.3-63.3,15.4-94.5C553,665.6,553.4,664.1,553.7,662.8z"/>
+	<path class="st1" d="M553.6,1319.7c-22.4-107.4-27.4-215.3-28.7-323.3c3.3-2.5,6.4-1.5,9.3-1.5c23.3,0.3,46.7,0.7,70,1.1
+		c9.5,0.1,9.2-1.2,9.2,9.2c0,72.3,0,144.7,0,217c0,13.3-0.1,26.7,0.1,40c0,3.6-0.7,6.2-3.8,8.6c-18.7,14.5-36.3,30.3-52.7,47.4
+		C556.3,1318.5,555.6,1318.7,553.6,1319.7z"/>
+	<path class="st1" d="M2119.8,1319.2c-6.7-6.5-12.1-12.3-18-17.5c-11.8-10.2-23.8-20.2-36-30.1c-4.2-3.4-5.9-7.1-5.9-12.7
+		c0.2-84.6,0.1-169.1,0.1-253.7c0-8.4,0.1-8.5,8-8.5c24.3-0.2,48.6-0.4,72.9-0.5c7.5,0,7.7,0,7.6,7.7c-0.3,45.6-1.9,91.2-5.1,136.7
+		c-2.7,39.9-6.7,79.6-12.5,119.1C2128.1,1279,2124.6,1298.3,2119.8,1319.2z"/>
+	<path class="st1" d="M2120.4,664.4c12.3,57.5,18.7,115.7,22.8,174.2c3.1,44.9,4.7,89.8,5.2,134.8c0,2.3-0.1,4.7,0.1,7
+		c0.3,4-1.7,5.2-5.4,5.1c-6.7-0.2-13.3-0.3-20-0.3c-18.3-0.1-36.7-0.2-55-0.4c-8-0.1-8.2-0.1-8.2-8.2c0-52.3,0-104.6,0-157
+		c0-32,0.2-64-0.2-96c-0.1-6.3,2-10.3,6.8-14.3c17.1-14.3,35.1-27.7,50.2-44.2C2117.5,664.3,2118.3,663.8,2120.4,664.4z"/>
+	<path class="st1" d="M878.7,1138.5c2,4.9,3.2,9.1,5.3,12.8c4,7.2,10.5,7.1,14.6,0c3-5.3,4.3-11.2,5.2-17.1c0.8-4.7,3.2-7.1,7.8-8.9
+		c37.6-14.8,76.3-25.6,115.5-34.7c5-1.1,6.6,0.4,7.3,4.9c0.6,3.5,0.7,8.4,5.3,8.5c5.3,0.1,5.5-5.2,5.8-9c0.5-7.1,3.9-10,10.9-11.2
+		c40-7,79.8-14.9,120.4-18c2.4-0.2,5.9-2.1,6.9,1.5c1,3.7,1.8,7.8,0.4,11.7c-0.7,1.8-2.9,1.2-4.4,1.5c-9.5,1.6-19.1,2.8-28.6,4.6
+		c-39.6,7.4-78.6,17-117,29.2c-59.9,19-116.5,45.2-170.8,76.4c-41.6,24-80.2,52.3-117.2,83.1c-39.4,32.8-74.3,69.6-105.4,110.2
+		c-3,3.9-6.2,7.7-9.4,11.6c-0.1,0.2-0.6,0.2-2.1,0.5c0.2-7-1.1-13.6,1.5-20.3c7.6-19.2,12.9-39.2,16.7-59.5
+		c1.8-9.6,6.2-17.5,12-24.8c17.2-21.7,36.5-41.5,58.8-58.9c3.4,2.2,3.5,6.6,5.4,9.8c4,7,7.8,14.4,17.4,14.4
+		c9.6,0,13.8-7.2,17.3-14.4c6.5-13.3,9.4-27.7,12.2-42c0.8-3.9,2.2-6.5,5.7-8.6C808.9,1171.4,842.7,1153.5,878.7,1138.5z"/>
+	<path class="st1" d="M1032.8,891.9c-42-9-82.2-20.4-121.3-35.7c-4.8-1.9-6.9-4.4-7.7-9c-0.9-5.6-2.3-11.1-4.8-16.2
+		c-1.5-3.2-3.7-5.9-7.3-6.1c-4.1-0.2-6.8,2.5-8.3,6.1c-1.6,3.6-2.8,7.4-4.7,12.2c-11.4-5.2-22.6-10-33.5-15.2
+		c-23.1-11.1-45.4-23.7-67.3-37.1c-4.3-2.6-6.6-5.5-7.5-10.7c-2.1-11.4-4.6-22.8-8.9-33.7c-2.1-5.3-4.2-10.5-7.8-15
+		c-7.6-9.4-17.5-9.4-25.3,0.1c-4.2,5.2-6,11.7-9.8,18.2c-11.1-9.9-22.3-18.8-32.2-28.9c-10.2-10.4-19.5-21.8-28.7-33.3
+		c-5.6-7-8.9-15.3-10.6-24.4c-3.7-19.3-8.6-38.3-16-56.5c-2.9-7.1-1.4-14.1-0.5-21.8c3.9,4.8,7.9,9.4,11.6,14.3
+		c23.7,32,51,60.7,80.1,87.7c19,17.6,39.4,33.9,59.9,49.7c45,34.7,93.8,63.3,144.7,87.9c39.2,18.9,79.8,34.7,121.6,47.1
+		c41.2,12.2,82.8,22.3,125.4,28.2c1,0.1,2,0.2,3,0.4c7.3,1.2,10.5,7.7,7,14.2c-1.3,2.3-3.4,1.7-5.3,1.6c-8-0.7-15.9-1.5-23.8-2.4
+		c-32.8-3.7-65.1-10.2-97.5-15.7c-7.1-1.2-11.5-3.9-11.7-11.6c0-1.3-0.4-2.6-0.8-3.9c-0.8-2.6-1.9-5.3-5.3-4.9
+		c-2.7,0.3-3.7,2.6-4.4,4.9C1034.4,885.3,1033.7,888.2,1032.8,891.9z"/>
+	<path class="st1" d="M1953.8,1233.7c3.8,1,5.8,3.4,8,5.3c21,19.1,40.7,39.4,57.6,62.2c3,4.1,4.9,8.4,6,13.5
+		c4.5,20.5,9.3,40.9,17.1,60.4c1.3,3.1,2.4,6.3,1.8,9.7c-0.6,3.6,2.4,8.9-2.1,10.4c-2.5,0.8-5.4-3.9-7.3-6.7
+		c-35.7-51.5-81-93.6-129.4-132.6c-40.8-32.9-84.7-61.2-131-85.6c-39.5-20.8-80.6-37.9-123-52c-45.9-15.2-92.6-26.8-140.3-34.3
+		c-6.2-1-12.4-2.7-18.7-3.1c-6.5-0.4-5.5-4.5-5.2-8.4c0.4-5.4,2.1-7.1,7.5-6.6c8.3,0.7,16.5,1.9,24.8,2.7
+		c33.2,3.4,65.8,10.1,98.6,15.7c6.3,1.1,9.2,3.4,9.6,10c0.2,3.6,0.3,9.3,5.3,9.6c6,0.4,5.5-5.6,6.3-9.5c0.7-3.9,2.4-5,6.2-4
+		c39.4,9.7,78.9,19.2,116.7,34.3c4.5,1.8,6.8,4.2,7.5,9.1c0.9,5.9,2.2,11.8,5.1,17.2c1.7,3.1,3.8,5.6,7.6,5.6c3.8,0,6-2.5,7.6-5.6
+		c0.9-1.8,1.7-3.6,2.3-5.5c2.8-7.7,2.9-7.9,9.9-4.7c32.6,15.3,64.6,31.4,95,50.8c3.4,2.2,5.1,4.7,5.8,8.6
+		c2.1,12.1,4.9,24.1,9.4,35.7c1.9,5,4.1,9.8,7.5,14c7.5,9.4,17.7,9.4,25.3,0C1948.9,1245.3,1950.9,1239.5,1953.8,1233.7z"/>
+	<path class="st1" d="M2043.6,585.1c0.9,7.5,1.7,14.1-0.8,20.4c-8,20.2-13.1,41.2-17.6,62.4c-0.9,4.4-2.6,8.3-5.2,11.7
+		c-17.8,23.8-38.4,45.2-60.6,64.9c-1.5,1.3-3.2,2.3-5.1,3.8c-2.9-5.3-5.1-10.4-8.2-14.9c-8-11.8-19.3-11.8-27.5-0.1
+		c-7.4,10.6-11.4,23-12.8,35.4c-1.7,14.6-9.4,22.2-21.3,29.3c-25.5,15.2-52.1,28.1-78.7,41.2c-2.4,1.2-5,2-7.3,3.1
+		c-2.7,1.3-4.3,0.7-5.2-2.2c-1-3.2-1.9-6.4-3.4-9.3c-4.2-7.7-10.8-7.7-15.1-0.1c-2.5,4.4-3.9,9.3-4.5,14.2c-0.8,7-4.2,10.7-11,13.3
+		c-27.5,10.2-55.4,18.9-83.9,25.7c-10,2.4-20.1,4.8-30,7.5c-3.8,1-5.4,0-6.1-3.9c-0.7-4-0.2-9.9-6.2-9.6c-5.1,0.2-5.3,6-5.4,9.5
+		c-0.1,7.5-4.3,9-10.5,10.2c-39.6,7.4-79.4,13.9-119.5,18.1c-2.8,0.3-6.7,2.3-8.1-1.3c-1.4-3.6-1.7-7.9-0.3-11.7
+		c0.9-2.4,3.9-2,6.2-2.4c13.8-2.4,27.6-4.4,41.3-7c42.1-8.2,83.5-19.3,124.2-33.1c57.6-19.6,112.2-45.7,163.9-77.3
+		c38.1-23.2,73.8-50.1,107.6-79.4c37.9-32.8,72.6-68.6,102.5-109C2037.5,591.3,2039.4,587.8,2043.6,585.1z"/>
+	<path class="st1" d="M777.2,1163.8c-0.9-7.9,1.2-15.4,2-23c4.4-45.1,6.3-90.3,6.6-135.6c0.1-8.6,0.2-8.7,8.8-8.7
+		c22-0.1,44-0.1,65.9,0c8.3,0,8.4,0.1,8.5,7.9c0.2,37.7,2,75.2,6.5,112.6c0.5,3.8,0.3,6-3.9,7.5c-31.7,11.3-62.2,25.3-92.6,39.6
+		C778.8,1164.3,778.4,1164.1,777.2,1163.8z"/>
+	<path class="st1" d="M776.8,817.8c3.1-0.6,4.7,0.6,6.5,1.4c28.2,13.5,56.6,26.5,86.2,36.8c5.4,1.9,6.7,4.3,6,10.1
+		c-4.5,36.7-6.3,73.6-6.4,110.6c0,8.1,0.7,8.3-8.1,8.3c-22.6-0.1-45.3,0-67.9-0.1c-7,0-7.1-0.2-7.2-7.3c-0.4-28.6-0.9-57.3-2.7-85.9
+		c-1.3-21.3-2.9-42.5-5.5-63.7C777.2,824.9,777.1,821.6,776.8,817.8z"/>
+	<path class="st1" d="M1896.4,1163.9c-3.3,0.5-5.7-1.5-8.4-2.8c-27.2-13.3-54.8-25.7-83.3-36.1c-7.4-2.7-7.2-2.9-6.3-10.6
+		c4.4-35.8,5.3-71.8,6.4-107.7c0.1-1.7,0.1-3.3,0-5c-0.3-3.8,1.3-5.4,5.2-5.3c6.3,0.2,12.7,0.1,19,0.1c16.7,0,33.3,0,50,0.1
+		c8.4,0,8.4,0.1,8.6,8.8c0.8,30.3,1.2,60.6,3.3,90.9c1.4,19.9,2.9,39.9,5.3,59.7C1896.5,1158.6,1897.4,1161.2,1896.4,1163.9z"/>
+	<path class="st1" d="M1896.8,817.7c-1.3,13.9-2.7,27.7-3.9,41.6c-3.5,38.5-4.5,77-5.2,115.6c0,1.7-0.2,3.3-0.1,5
+		c0.4,4-1.5,5.5-5.2,5.1c-1-0.1-2,0-3,0c-22.6,0-45.3-0.1-67.9,0.2c-5,0-6.8-1.2-6.8-6.6c-0.4-28.3-1.2-56.5-3.6-84.7
+		c-0.8-8.9-1.6-17.9-2.7-26.8c-0.9-7.6-1.1-7.8,6.5-10.6c23-8.7,45.8-17.8,67.9-28.7c6-2.9,12-5.7,18-8.6
+		C1892.5,818.3,1894.3,817.4,1896.8,817.7z"/>
+	<path class="st1" d="M908.3,1109.6c-1-4.4,0.4-8.6,0.8-12.8c2.5-30.9,4-61.8,4-92.9c0-7.4,0.2-7.5,7.9-7.5
+		c33.3-0.1,66.7-0.2,100-0.2c7.8,0,8.1,0.1,8,7.4c-0.3,22,1,44,2.3,65.9c0.3,4.5-1,6.5-5.5,7.4c-38.7,8.5-76.4,20.9-114.2,32.5
+		C910.7,1109.8,909.7,1109.6,908.3,1109.6z"/>
+	<path class="st1" d="M1765.3,1110.4c-9.6-3-18.1-5.6-26.6-8.4c-29.1-9.6-58.7-17.4-88.5-24.5c-8.9-2.1-8.1-1.4-7.6-10.4
+		c1.2-21,1.6-42,2.3-62.9c0.2-7.7,0.2-7.9,7.5-7.9c33.3,0,66.7,0.1,100,0.2c7.8,0,8.2,0.3,8.2,7.7c0.2,34,1.7,68,4.8,101.8
+		C1765.4,1106.9,1765.3,1107.9,1765.3,1110.4z"/>
+	<path class="st1" d="M908.8,872.2c2.8-1.1,5,0.8,7.5,1.5c36.1,10.3,71.6,22.9,108.5,30.5c5.5,1.1,6.7,3.6,6.4,8.8
+		c-1.3,21.9-2.6,43.9-2.1,65.9c0.1,4.7-1.2,6.5-6.1,6.4c-34.7-0.2-69.3-0.2-104-0.1c-4.1,0-6.1-1-6-5.6c0.4-35.6-2-71.2-4.8-106.7
+		C908.2,872.8,908.6,872.5,908.8,872.2z"/>
+	<path class="st1" d="M1765.6,872.4c-0.9,10.7-1.8,20.9-2.5,31.2c-1.7,24.6-2.4,49.2-2.6,73.8c-0.1,7.7-0.2,7.8-7.7,7.8
+		c-33.6,0.1-67.2,0.1-100.8,0.1c-6.9,0-6.9-0.2-7.1-7.1c-0.8-21.9-1.3-43.9-2.6-65.8c-0.3-5.4,1.8-7,6.2-8
+		c14.5-3.5,29-7.1,43.5-10.8c22.5-5.7,44.5-13.5,66.7-20.3C1760.5,872.7,1762.3,871.8,1765.6,872.4z"/>
+	<path class="st1" d="M725.8,1192.5c-5.9-34.3-8.8-67.4-11-100.6c-1.8-27.6-2.2-55.2-3-82.8c-0.1-2.3,0.1-4.7-0.1-7
+		c-0.4-4.4,1.6-5.8,5.9-5.7c9,0.2,18,0,27,0c6.7,0,13.3-0.1,20,0c7.2,0.1,7.4,0.2,7.3,7.1c-0.5,23.6-1,47.2-2,70.8
+		c-1.2,27.2-3.3,54.4-6.7,81.5c-0.2,1.3-0.3,2.6-0.4,4c-0.4,9.2-3.4,16-13,19.7C741.7,1182.7,734.5,1187.7,725.8,1192.5z"/>
+	<path class="st1" d="M1946.2,1191.7c-10.1-5.6-20.4-11.5-30.8-17.2c-3.3-1.8-3.1-5-3.5-7.8c-1.4-10.2-2.4-20.5-3.5-30.7
+		c-4.7-43.8-6-87.7-6.7-131.6c-0.1-7.7,0.1-7.8,7.7-7.8c15.3-0.1,30.6,0.1,46-0.1c4.6-0.1,6.5,1.1,6.4,6.1
+		c-0.5,34-1.2,67.9-3.7,101.8c-2,26.9-4.5,53.7-8.9,80.4C1948.7,1187,1949.1,1189.5,1946.2,1191.7z"/>
+	<path class="st1" d="M1947.7,789.1c5.8,32.7,8.6,63.8,10.8,95c2.1,30.6,2.8,61.2,3.1,91.9c0.1,9.6,0.3,8.9-8.3,8.9
+		c-15,0-30-0.1-45,0.2c-5.3,0.1-6.9-1.8-6.7-7c0.9-25.3,1.3-50.6,2.5-75.9c1.3-27.3,3.9-54.4,6.9-81.6c0.9-8.3,3.3-14.2,11.5-17.8
+		C1930.8,799.2,1938.6,794.1,1947.7,789.1z"/>
+	<path class="st1" d="M726,789.2c11.3,6.2,21.7,11.9,32.2,17.5c2.9,1.6,2.8,4.2,3.2,6.8c2.7,15.8,4.1,31.7,5.5,47.6
+		c3.3,37.2,4.5,74.5,5,111.8c0,2.3,0,4.7,0.1,7c0.1,3.1-0.8,5-4.4,5c-17-0.1-34-0.1-51-0.1c-3.3,0-4.8-1.5-4.7-4.8
+		c0.2-5.7,0-11.3,0.2-17c1-42.6,2.7-85.2,7.3-127.7C720.9,820.5,722.6,805.7,726,789.2z"/>
+	<path class="st1" d="M1625.1,910.5c-0.6,9.4-1.4,18-1.7,26.5c-0.4,13-0.4,26-0.6,38.9c-0.1,9.3-0.2,9.4-9.2,9.4
+		c-30.6,0.1-61.3,0.1-91.9,0.1c-8.3,0-16.7-0.1-25,0.1c-4.2,0.1-5.9-1.3-5.9-5.7c0-15-0.1-30-0.7-44.9c-0.2-4.8,2.5-5.5,6-6
+		c10.9-1.3,21.8-3.6,32.7-3.7c23.9-0.2,46.7-6.5,70-10C1607.1,913.9,1615.2,912.2,1625.1,910.5z"/>
+	<path class="st1" d="M1625.2,1071.1c-23.5-4.2-45.7-8.1-67.9-12.1c-7.9-1.4-15.8-2.4-23.8-2.5c-12.7-0.1-25.1-2.6-37.7-4
+		c-4-0.4-5.8-1.6-5.6-6.1c0.5-14.6,0.6-29.3,0.6-43.9c0-4.5,1-6.5,6.1-6.5c39.9,0.2,79.9,0.2,119.8,0.1c5,0,6.1,2.1,6.1,6.6
+		C1622.9,1025,1623,1047.3,1625.2,1071.1z"/>
+	<path class="st1" d="M1049.3,1070.4c-1-5.9-0.1-11.8,0.3-17.7c0.9-16.3,1.4-32.6,1.4-48.9c0-7.3,0.2-7.5,6.8-7.5
+		c39.3-0.1,78.5-0.1,117.8-0.1c7,0,7.1,0.2,7.2,7.2c0.2,14,0,27.9,0.5,41.9c0.2,5.2-1.5,6.6-6.6,7c-16.9,1.3-33.8,2.7-50.6,5.2
+		c-23.3,3.4-46.7,6.8-69.7,12.3C1054.2,1070.3,1051.9,1071.4,1049.3,1070.4z"/>
+	<path class="st1" d="M1049.8,910.2c22.1,5.7,44.8,8.7,67.4,12.5c16.4,2.8,33.1,3.3,49.6,5.2c16.4,1.9,16.4,1.5,16.3,17.4
+		c-0.1,11-0.3,22-0.2,32.9c0.1,4.9-1,7.2-6.6,7.2c-39.9-0.2-79.9-0.2-119.8-0.1c-3.4,0-5.7-0.6-5.6-4.8c0.6-22.6-1.3-45.2-1.9-67.8
+		C1049,912,1049.4,911.4,1049.8,910.2z"/>
+	<path class="st1" d="M1260.5,985.4c-21.6,0-43.2,0-64.9,0c-8,0-8.2-0.1-8.1-8.4c0.2-13.6,0.6-27.3,0.8-40.9c0.1-4.2,1.4-5.6,6-5.1
+		c41.7,4.1,83.5,6.2,125.5,6.2c2.3,0,4.7,0,7,0c7.5,0.2,7.5,0.2,7.6,7.9c0.1,10.3,0.1,20.6,0.1,30.9c0,9.3,0,9.4-9.1,9.4
+		C1303.8,985.4,1282.2,985.4,1260.5,985.4z"/>
+	<path class="st1" d="M1485,931.5c0.3,16.2,0.5,32.7,0.8,49.3c0.1,3.4-1.7,4.8-4.9,4.6c-2-0.1-4,0-6,0c-41.6,0-83.2,0-124.9,0
+		c-2,0-4-0.1-6,0c-4,0.3-5.1-1.8-5-5.5c0.1-11.7,0-23.3,0.1-35c0.1-7.6,0.1-7.8,7.6-7.8c17.3,0.2,34.6-0.6,51.9-1.1
+		c26.3-0.7,52.5-3.1,78.7-5.1C1479.6,930.9,1481.8,930,1485,931.5z"/>
+	<path class="st1" d="M1485,1051.1c-35.4-3-69.9-5.3-104.5-6.4c-11.6-0.4-23.3-0.5-34.9-0.4c-4.4,0-6.7-0.8-6.5-6
+		c0.4-12,0.3-24,0-36c-0.1-4.8,1.6-6.3,6.3-6.2c45,0.1,89.9,0.1,134.9,0c3.5,0,5.8,0.7,5.7,4.8
+		C1485.5,1017.5,1485.3,1034.1,1485,1051.1z"/>
+	<path class="st1" d="M1188.4,1050.6c-0.3-17.2-0.5-33.7-0.8-50.3c0-2.9,1.6-4.1,4.4-4.1c1.7,0,3.3,0,5,0c42.6,0,85.1,0,127.7,0
+		c1.7,0,3.3,0.1,5,0c3.3-0.2,4.9,1.2,4.9,4.6c-0.1,13-0.1,25.9,0,38.9c0,3.6-1.7,4.7-4.9,4.5c-0.7,0-1.3,0-2,0
+		C1281.6,1043.7,1235.9,1047.2,1188.4,1050.6z"/>
+	<path class="st1" d="M2010.7,1230.1c-5.1-1.9-9-5.1-13.1-7.7c-9.9-6.1-19.6-12.6-29.8-18.3c-4.8-2.6-5.5-5.6-4.5-10.4
+		c3.9-20,6.2-40.1,8.2-60.4c4-41.2,5.8-82.4,6.1-123.8c0-3,0.1-6,0-9c-0.1-3,1.1-3.8,4.1-4.1c16.5-1.5,16.7-1.4,16.8,14.7
+		c0.4,66.4,3.9,132.5,10.7,198.6C2009.9,1216.3,2011.6,1222.7,2010.7,1230.1z"/>
+	<path class="st1" d="M2011.2,752.5c-1.6,16.9-3.2,33.8-4.8,50.6c-3,30.8-4.6,61.7-6.1,92.6c-1.3,27.3-1.5,54.5-1.8,81.8
+		c-0.1,7.1-0.2,7.3-7.1,7.3c-16.3,0-13.6,1.3-13.7-12.8c-0.5-54.3-3.2-108.4-10.6-162.2c-1.1-7.9-2.5-15.8-3.9-23.6
+		c-0.7-3.6-0.1-6,3.5-8.2c13.4-8,26.6-16.4,39.8-24.7C2007.6,752.8,2008.5,751.7,2011.2,752.5z"/>
+	<path class="st1" d="M662.1,1231.5c1.9-20.4,3.7-38.6,5.4-56.8c3-31.9,4.7-63.8,6-95.8c1-24.6,1.6-49.3,1.6-74
+		c0-8.3,0.1-8.3,7.9-8.4c1.3,0,2.7,0,4,0c8.6,0.1,8.7,0.1,8.8,8.7c0.4,40,1.8,80,5.4,119.8c2,22.9,4.4,45.8,9,68.3
+		c1.1,5.5,0.2,8.5-5.1,11c-12.7,6-23.9,14.5-35.5,22.2C667.7,1227.9,665.8,1229.1,662.1,1231.5z"/>
+	<path class="st1" d="M662.9,752.1c2.6-0.9,3.9,1,5.4,2.1c12.4,8.4,24.6,17.1,38,23.9c3.3,1.7,5,3.6,4.2,7.7
+		c-11.3,61.8-13.7,124.4-14.8,187c-0.2,13.5,2.2,12.1-12.3,12.1c-8.4,0-8.5-0.1-8.5-8.9c0.1-52.7-2.6-105.3-6.4-157.8
+		c-1.5-20.6-4-41.1-6-61.7C662.5,755.1,661.9,753.4,662.9,752.1z"/>
+	<path class="st1" d="M2044.3,728.6c0,3.4,0,6.3,0,9.2c0,78.9,0,157.9,0,236.8c0,1.7-0.1,3.3,0,5c0.4,3.9-1.3,5.4-5.2,5.3
+		c-5.9-0.2-11.9,0-18.3,0c0-26.3,0.3-51.6,1.3-76.8c1.7-42.3,4.4-84.5,8.9-126.5c1.6-14.9,3.9-29.7,5.6-44.6
+		C2037.2,732.4,2038.9,729.7,2044.3,728.6z"/>
+	<path class="st1" d="M2044.3,1253c-5.6-1.2-7-4.1-7.7-8.6c-3.1-19.4-5.2-38.9-7-58.5c-5.6-59-8.7-118.2-8.7-177.6
+		c0-13.6-1.4-11.7,11.5-11.8c13.3,0,11.9-1.5,11.9,11.5c0,78.6,0,157.2,0,235.8C2044.3,1246.6,2044.3,1249.3,2044.3,1253z"/>
+	<path class="st1" d="M629.2,728.7c5.8,1.6,7.4,4.6,7.9,9c2.3,21.2,5.3,42.2,7.3,63.4c5.3,55.3,7.7,110.8,8.5,166.4
+		c0,3.3,0.1,6.7,0,10c-0.1,7.5-0.1,7.5-7.7,7.7c-16,0.5-16,0.5-16-15.3c0-76.5,0-153.1,0-229.6C629.2,736.7,629.2,733,629.2,728.7z"
+		/>
+	<path class="st1" d="M629.2,1253.2c0-6,0-10.6,0-15.3c0-75.7,0-151.3,0-227c0-2,0-4,0-6c-0.1-9.2-0.3-8.7,8.7-8.6
+		c15.1,0,15.2,0,15,15.6c-0.7,62.7-3.8,125.2-10.3,187.6c-1.5,14.9-3.9,29.7-5.5,44.6C636.6,1248.7,634.5,1251.1,629.2,1253.2z"/>
+	<path class="st1" d="M2060,697.5c0-15.9,0-29.5,0-43c0-3.7,0.3-7.3-0.1-11c-2-19,6.4-34.8,15.3-50.5c1-1.7,2.3-3.2,3.6-4.7
+		c5.4-6.2,8.1-6.2,13.4,0.4c6.3,7.8,10.3,17.1,13.7,26.4c3.2,8.7,5.9,17.6,8.7,26.5c1.3,4,1,7.1-2.4,10.7
+		C2096.7,668.2,2079,681.6,2060,697.5z"/>
+	<path class="st1" d="M613.2,1283.1c0,24.5,1.5,47.6-0.5,70.3c-1.4,15.4-9,29.8-19.7,41.7c-3.8,4.2-5.7,3.9-10-0.4
+		c-4.3-4.3-7.2-9.6-9.8-15c-6.7-13.8-11.2-28.4-15.3-43.2c-0.8-3-1.3-5.6,1.5-8.1C576.3,1313,592.4,1296.7,613.2,1283.1z"/>
+	<path class="st1" d="M2060,1284.4c19.3,15.5,36.8,29.3,52.5,45.2c2.6,2.6,3.7,5.1,2.6,8.8c-5.1,17.2-10,34.5-19.7,50
+		c-7.7,12.2-12.4,12.7-19.8,0.6c-9.5-15.4-16.9-31.4-15.8-50.5C2060.7,1321.2,2060,1303.9,2060,1284.4z"/>
+	<path class="st1" d="M611.9,698.2c-8.5-6.7-17.1-12.8-25-19.7c-9.1-7.8-17.5-16.4-26.3-24.5c-2.3-2.2-4-4.1-3-7.7
+		c5.3-18.2,10.3-36.5,20.4-52.8c7.8-12.7,12.8-13,20.4-0.3c8.9,14.9,16.3,30.2,15.1,48.6c-1,15.9-0.2,31.9-0.3,47.9
+		C613.2,692.2,614,694.9,611.9,698.2z"/>
+	<path class="st1" d="M1788.2,864c4.1,27,4.9,53.5,6.1,80.1c0.5,11.3,0.2,22.6,0.7,33.9c0.2,5.2-1.6,7.4-6.7,7.3
+		c-4.7-0.1-9.3-0.3-14,0c-4.6,0.3-5.5-2-5.4-6c0.2-26.3,1.3-52.5,3.2-78.8c0.5-7.6,1.2-15.2,2.1-22.8
+		C1775.6,867.2,1775.7,867.2,1788.2,864z"/>
+	<path class="st1" d="M885.3,1118.4c-5.1-39.1-6.6-77.6-6.6-116.3c0-4.2,1.3-5.9,5.7-5.8c25,0.7,20.1-4.1,19.8,19.4
+		c-0.4,29.6-1.5,59.3-5,88.8C897.9,1115.4,898.1,1115.4,885.3,1118.4z"/>
+	<path class="st1" d="M885.7,863.5c12.4,2.1,12.2,2.1,13.5,13.8c3.8,33,4.6,66.2,5.3,99.4c0.2,8.3,0,8.5-7.9,8.4
+		c-5.6-0.1-12.6,2.4-16.3-1.2c-3.8-3.6-1.3-10.7-1.3-16.2c0.1-33.9,1.8-67.7,6.1-101.4C885.1,865.4,885.4,864.5,885.7,863.5z"/>
+	<path class="st1" d="M1788.4,1117.5c-4,0.3-6.2-1.5-8.7-2.2c-3.2-0.9-4.6-2.9-4.9-6.5c-0.7-10.3-2.2-20.5-2.9-30.8
+		c-1.5-24.6-2.8-49.2-2.8-73.9c0-7.5,0.1-7.8,7.7-7.6c5.9,0.2,13.3-2.7,17.2,1.6c3.4,3.7,1,10.9,0.9,16.5
+		C1794.3,1048.8,1792.8,1083.1,1788.4,1117.5z"/>
+	<path class="st1" d="M1481.1,1078.1c-10.7,0.3-20.5-1.5-30.3-2.6c-34.1-3.7-68.2-5.9-102.5-5.6c-1.3,0-2.7-0.1-4,0
+		c-4.9,0.5-5.1-2.5-5.2-6.2c-0.1-4,0.7-6,5.4-6c20,0.1,39.9-0.2,59.9,1c22.9,1.4,45.8,3,68.7,4.7c7.9,0.6,7.8,0.9,8.2,8.7
+		C1481.2,1073.7,1481.1,1075.4,1481.1,1078.1z"/>
+	<path class="st1" d="M1480,903c2,3.3,0.7,6.3,1,9.2c0.4,4.3-1.6,5.6-5.8,5.9c-42.2,3.3-84.3,6.2-126.7,5.9c-1.3,0-2.7-0.1-4,0
+		c-4.5,0.5-5.5-1.7-5.6-5.8c-0.1-4.5,0.9-6.6,6-6.5c29.7,0.7,59.2-1.5,88.8-3.9C1449.3,906.5,1464.7,904.6,1480,903z"/>
+	<path class="st1" d="M1192.2,1078.2c1-14.7,0.9-15,13.5-15.8c40.5-2.9,80.9-5.4,121.6-4.6c7.3,0.1,7.4,0.5,7.3,7
+		c-0.1,4-1.7,5.2-5.4,5.1c-6-0.2-12-0.1-18,0c-37.3,0.2-74.3,3.2-111.3,7.8C1197.6,1078,1195.3,1078,1192.2,1078.2z"/>
+	<path class="st1" d="M1192.6,903.2c17.2,2.2,33.7,3.8,50.2,5c28.2,2.1,56.4,4.4,84.7,3.6c3.8-0.1,7.3-0.4,7.1,5.4
+		c-0.3,6.3-0.2,6.8-6.6,6.8c-18.6,0-37.2,0.4-55.9-0.8c-19.2-1.2-38.5-2.1-57.8-3.2c-3-0.2-6-0.4-8.9-0.8
+		C1192.8,917.6,1192.8,917.6,1192.6,903.2z"/>
+	<path class="st1" d="M653.6,694.6c2.5,2.7,3.4,3.5,4.2,4.6c15.1,20.9,32.8,39.4,51.8,56.7c2.5,2.3,7.7,4.8,5,8.5
+		c-2.7,3.9-6.2-1-8.9-2.4c-14.7-7.8-28.1-17.9-42.2-26.6c-3.1-1.9-4.3-4.4-4.8-8C657.4,717,655.5,706.9,653.6,694.6z"/>
+	<path class="st1" d="M654,1284.6c0.9-5.5,2.7-11,2.6-16.4c-0.1-14.4,6.3-23.8,19.2-30.2c10.6-5.3,20.3-12.5,30.5-18.8
+		c2.5-1.5,4.9-3.3,7.8-3.6c1.6,2.1,1.5,4,0.2,5.7c-1,1.3-2.4,2.3-3.6,3.4c-18.4,17.3-36.5,34.8-51.2,55.5c-1.1,1.6-2.6,2.9-4,4.3
+		C655.3,1284.8,654.8,1284.6,654,1284.6z"/>
+	<path class="st1" d="M2019.1,698.8c-1.4,9.8-3,19.6-4.2,29.5c-0.4,3-2.2,4.7-4.4,6.2c-15.3,9.9-30.7,19.7-46.1,29.5
+		c-1.5,1-3.7,4-5.7,0.4c-1.5-2.6,0.2-4.4,2.1-5.9c11.9-9.2,21.5-20.7,32.4-30.8c8.4-7.7,14.5-17.2,21.4-26.1c0.9-1.2,2-2.4,3-3.6
+		C2018.1,698.2,2018.6,698.5,2019.1,698.8z"/>
+	<path class="st1" d="M2017.1,1283.9c-11.8-19.6-28.7-34.6-44.6-50.5c-3.7-3.7-7.8-7.2-11.8-10.6c-1.9-1.7-3.4-3.4-1.7-5.9
+		c1.7-2.4,3.5-0.5,5,0.5c15.7,10,31.3,20,46.9,30c2,1.3,3.5,2.9,3.9,5.4c1.4,10.1,2.9,20.2,4.4,30.4
+		C2018.5,1283.4,2017.8,1283.7,2017.1,1283.9z"/>
+	<path class="st1" d="M1044.6,1034c-0.6,11-1.1,21.9-1.7,32.9c-0.1,1.6-0.3,3.3-0.8,4.9c-0.3,0.8-1.4,1.6-2.2,1.7
+		c-0.9,0.1-1.9-0.5-2.1-1.7c-0.1-1-0.3-2-0.4-3c-1.2-22.2-2.3-44.5-2.4-66.8c0-3.2,0-6,4.6-5.9c3.5,0.1,5.6,0.7,5.5,4.9
+		c-0.2,11-0.1,21.9-0.1,32.9C1044.9,1034,1044.8,1034,1044.6,1034z"/>
+	<path class="st1" d="M1636.4,1033.8c0,10.6,0,21.2,0,31.8c0,2-0.3,4-0.7,5.9c-0.2,0.8-1.2,1.8-1.9,1.8c-0.8,0-1.9-0.8-2.2-1.5
+		c-0.5-1.2-0.5-2.6-0.6-3.9c-1.9-21.8-2.2-43.7-2.1-65.6c0-5.7,1.3-6.7,6-6.1c3.9,0.5,3.6,3.1,3.6,5.8c0,10.6,0,21.2,0,31.8
+		C1637.8,1033.8,1637.1,1033.8,1636.4,1033.8z"/>
+	<path class="st1" d="M1638.5,948.1c0,10.3,0,20.6,0,31c0,2.6,0.5,5.4-3.3,6.2c-4.4,0.9-6.4-0.5-6.4-5.8c0-19,0-38,1.4-56.9
+		c0.2-3,0.4-6,0.7-9c0.2-2.2-0.3-5.4,3-5.2c2.5,0.1,2.2,3,2.3,4.9c0.6,11.6,1.1,23.3,1.6,34.9C1638,948.1,1638.2,948.1,1638.5,948.1
+		z"/>
+	<path class="st1" d="M1045.1,948c0,10.6-0.1,21.2,0,31.9c0,3.6-0.9,5.5-5,5.5c-3.9-0.1-5.1-1.7-5.1-5.4c0.1-22.6,1.2-45.1,2.5-67.7
+		c0.1-1.6-0.1-3.7,2-4c2.8-0.4,3.1,1.9,3.2,3.9c0.7,11.9,1.3,23.9,1.9,35.8C1044.8,948,1044.9,948,1045.1,948z"/>
+	<path class="st1" d="M741.9,740.1c7,10.8,9.7,22,12.6,34.5c-8.1-3.6-13.5-8.9-19.7-13c-2-1.3-2-3.3-1.2-5.3
+		C735.6,751.1,737,745.6,741.9,740.1z"/>
+	<path class="st1" d="M1931.7,1241.3c-6.9-10.6-9.7-21.7-12.1-33.3c4.2,0.4,6.4,3.1,9.1,4.9
+		C1943.2,1223.1,1943.4,1224.7,1931.7,1241.3z"/>
+	<path class="st1" d="M1931.6,740c12,17.2,11.9,18.3-3.6,29.1c-2.4,1.7-4.5,3.9-8.5,4.8C1921.8,761.9,1925,750.9,1931.6,740z"/>
+	<path class="st1" d="M742.1,1241.3c-5.3-5.2-6.5-11.1-8.6-16.5c-0.8-2-0.5-3.8,1.6-5.1c6-3.9,11.2-9.1,19.2-12.6
+		C751.8,1219.5,748.7,1230.3,742.1,1241.3z"/>
+	<path class="st1" d="M599.6,517.5c0.6,0.1,1.4,0.1,1.9,0.4c12.6,7.1,13,7.9,10.7,24.4c-5.9-7.9-11-15.2-13.4-24.1
+		C599,518,599.3,517.7,599.6,517.5z"/>
+	<path class="st1" d="M597.6,1464.4c4-9.4,8.1-17.2,14-24.4C614.9,1455.5,614.5,1457.9,597.6,1464.4z"/>
+	<path class="st1" d="M2072.4,516.4c2.2,0.6,2.3,2,1.3,3.7c-3.8,6.1-7.7,12.1-11.9,18.6c-2.8-10.6-2.1-12.7,5.3-17.8
+		c1.9-1.3,4.2-2,5.3-4.3L2072.4,516.4z"/>
+	<path class="st1" d="M2061.5,1442.5c4.7,8.5,10.5,14.5,12.5,22.8C2067.2,1459.8,2056.4,1456.8,2061.5,1442.5z"/>
+	<path class="st1" d="M1781.6,836.9c4.2,9.3,4.2,9.3-1.8,12.6C1777.9,845.4,1780.6,842.1,1781.6,836.9z"/>
+	<path class="st1" d="M895.4,849.3c-7.5-2.9-7.5-2.9-3.8-11.1C893.8,841.3,893.9,844.7,895.4,849.3z"/>
+	<path class="st1" d="M630.1,1284.6c-0.2-5.4-2.7-11,1.8-17C634.2,1274.4,629.7,1279.3,630.1,1284.6z"/>
+	<path class="st1" d="M629.9,697.1c0.5,5.4,3.9,10.3,2.4,16.6C626.9,708.5,630.1,702.5,629.9,697.1z"/>
+	<path class="st1" d="M2044.5,1284.5c-2.2-5-4.3-9.2-2.8-15C2046.1,1274.3,2043.5,1279.4,2044.5,1284.5z"/>
+	<path class="st1" d="M2041.9,712.1c-2.2-6,1.1-10.1,2.2-14.6C2043.5,702.2,2046,707.4,2041.9,712.1z"/>
+	<path class="st1" d="M892.2,1143.9c-4.6-7.9-4.6-7.9,1.6-11.5C895.7,1136.5,892.6,1139.7,892.2,1143.9z"/>
+	<path class="st1" d="M1779.6,1132.2c6.1,3.1,6.1,3.1,2.7,11.1c-1.9-1-1.7-3.1-2.3-4.6C1779.2,1137,1778.6,1135.1,1779.6,1132.2z"/>
+	<path class="st1" d="M1934.8,1202.5c4.1-0.6,6.5,2.5,11.4,4.9c-6.3,0.5-8.2-3.5-11.3-5.1L1934.8,1202.5z"/>
+	<path class="st1" d="M1935.8,777c3.1,0.9,4.5-3.5,7.8-2.9c-1.8,4-4.7,3.9-7.9,2.8L1935.8,777z"/>
+	<path class="st1" d="M729.5,774c2.7-0.5,4.4,0.6,5.8,3.2C732.1,778,731,775.7,729.5,774z"/>
+	<path class="st1" d="M734,1202.5c2.3,4.6-1.2,4.8-4.2,5.9c-0.6-3.7,3.7-3.3,4.2-5.7C734,1202.7,734,1202.5,734,1202.5z"/>
+	<path class="st1" d="M1935.2,780.3c-0.4,0.2-0.8,0.5-1.2,0.7c0.3,0,0.6,0.1,0.8,0c0.2-0.1,0.4-0.3,0.6-0.5
+		C1935.5,780.5,1935.2,780.3,1935.2,780.3z"/>
+	<path class="st1" d="M740,1203.1c-1.9-0.8-4.2,2.9-6-0.5c0,0,0,0.2,0,0.1C736,1202.8,738,1202.9,740,1203.1L740,1203.1z"/>
+	<path class="st1" d="M1935.7,777c1.4,1.3,0.9,2.4-0.5,3.3c0,0,0.3,0.3,0.3,0.3c0.1-1.2,0.3-2.3,0.4-3.5
+		C1935.8,777,1935.7,777,1935.7,777z"/>
+	<path class="st1" d="M737.9,778.4c0.2,0.2,0.4,0.5,0.6,0.7c-0.2,0.2-0.5,0.5-0.7,0.5c-0.2,0-0.4-0.4-0.6-0.6
+		C737.4,778.8,737.7,778.6,737.9,778.4z"/>
+	<path class="st1" d="M740,1203.1c0-0.3,0-0.8-0.2-1c-0.8-0.8-0.7-1.1,0.4-0.8C739.5,1201.9,739.4,1202.4,740,1203.1
+		C740,1203.1,740,1203.1,740,1203.1z"/>
+	<path class="st1" d="M1934.9,1202.3c0-0.2-0.1-0.3-0.1-0.5c0,0-0.1,0.1-0.1,0.1c0.1,0.2,0.1,0.4,0.2,0.6
+		C1934.8,1202.5,1934.9,1202.3,1934.9,1202.3z"/>
+	<path class="st1" d="M2072.4,516.6c0.7,0.4,1.4,0.8,2.1,1.2c-0.2,0.2-0.4,0.7-0.6,0.7c-1.1-0.2-1.5-1-1.4-2
+		C2072.4,516.4,2072.4,516.6,2072.4,516.6z"/>
+	<path class="st1" d="M598.7,518.3c-1-0.2-1.8-0.8-1-1.6c0.8-0.9,1.4,0,1.8,0.8C599.3,517.7,599,518,598.7,518.3z"/>
+</g>
+</svg>

docs/conf.py

@@ -0,0 +1,105 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+# Warn about all references to unknown targets
+nitpicky = True
+
+# The master toctree document.
+master_doc = 'index'
+
+# -- Project information -----------------------------------------------------
+
+project = 'tractor'
+copyright = '2018, Tyler Goodlet'
+author = 'Tyler Goodlet'
+
+# The full version, including alpha/beta/rc tags
+release = '0.0.0a0.dev0'
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_book_theme'
+
+pygments_style = 'algol_nu'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+html_theme_options = {
+    # 'logo': 'tractor_logo_side.svg',
+    # 'description': 'Structured concurrent "actors"',
+    "repository_url": "https://github.com/goodboy/tractor",
+    "use_repository_button": True,
+    "home_page_in_toc": False,
+    "show_toc_level": 1,
+    "path_to_docs": "docs",
+
+}
+html_sidebars = {
+    "**": [
+        "sbt-sidebar-nav.html",
+        # "sidebar-search-bs.html",
+        # 'localtoc.html',
+    ],
+    #     'logo.html',
+    #     'github.html',
+    #     'relations.html',
+    #     'searchbox.html'
+    # ]
+}
+
+# doesn't seem to work?
+# extra_navbar = "<p>nextttt-gennnnn</p>"
+
+html_title = ''
+html_logo = '_static/tractor_logo_side.svg'
+html_favicon = '_static/tractor_logo_side.svg'
+# show_navbar_depth = 1
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
+    "pytest": ("https://docs.pytest.org/en/latest", None),
+    "setuptools": ("https://setuptools.readthedocs.io/en/latest", None),
+}

docs/dev_tips.rst

@@ -0,0 +1,51 @@
+Hot tips for ``tractor`` hackers
+================================
+
+This is a WIP guide for newcomers to the project mostly to do with
+dev, testing, CI and release gotchas, reminders and best practises.
+
+``tractor`` is a fairly novel project compared to most since it is
+effectively a new way of doing distributed computing in Python and is
+much closer to working with an "application level runtime" (like erlang
+OTP or scala's akka project) then it is a traditional Python library.
+As such, having an arsenal of tools and recipes for figuring out the
+right way to debug problems when they do arise is somewhat of
+a necessity.
+
+
+Making a Release
+----------------
+We currently do nothing special here except the traditional
+PyPa release recipe as in `documented by twine`_. I personally
+create sub-dirs within the generated `dist/` with an explicit
+release name such as `alpha3/` when there's been a sequence of
+releases I've made, but it really is up to you how you like to
+organize generated sdists locally.
+
+The resulting build cmds are approximately:
+
+.. code:: bash
+
+    python setup.py sdist -d ./dist/XXX.X/
+
+    twine upload -r testpypi dist/XXX.X/*
+
+    twine upload dist/XXX.X/*
+
+
+
+.. _documented by twine: https://twine.readthedocs.io/en/latest/#using-twine
+
+
+Debugging and monitoring actor trees
+------------------------------------
+TODO: but there are tips in the readme for some terminal commands
+which can be used to see the process trees easily on Linux.
+
+
+Using the log system to trace `trio` task flow
+----------------------------------------------
+TODO: the logging system is meant to be oriented around
+stack "layers" of the runtime such that you can track
+"logical abstraction layers" in the code such as errors, cancellation,
+IPC and streaming, and the low level transport and wire protocols.

docs/github_readme/_sphinx_readme.rst

@@ -0,0 +1,109 @@
+tractor
+=======
+The Python async-native multi-core system *you always wanted*.
+
+
+|gh_actions|
+|docs|
+
+.. _actor model: https://en.wikipedia.org/wiki/Actor_model
+.. _trio: https://github.com/python-trio/trio
+.. _multi-processing: https://en.wikipedia.org/wiki/Multiprocessing
+.. _trionic: https://trio.readthedocs.io/en/latest/design.html#high-level-design-principles
+.. _async sandwich: https://trio.readthedocs.io/en/latest/tutorial.html#async-sandwich
+.. _structured concurrent: https://trio.discourse.group/t/concise-definition-of-structured-concurrency/228
+
+
+``tractor`` is a `structured concurrent`_ "`actor model`_" built on trio_ and multi-processing_.
+
+It is an attempt to pair trionic_ `structured concurrency`_ with
+distributed Python. You can think of it as a ``trio``
+*-across-processes* or simply as an opinionated replacement for the
+stdlib's ``multiprocessing`` but built on async programming primitives
+from the ground up.
+
+Don't be scared off by this description. ``tractor`` **is just ``trio``**
+but with nurseries for process management and cancel-able IPC.
+If you understand how to work with ``trio``, ``tractor`` will give you
+the parallelism you've been missing.
+
+``tractor``'s nurseries let you spawn ``trio`` *"actors"*: new Python
+processes which each run a ``trio`` scheduled task tree (also known as
+an `async sandwich`_ - a call to ``trio.run()``). That is, each
+"*Actor*" is a new process plus a ``trio`` runtime.
+
+"Actors" communicate by exchanging asynchronous messages_ and avoid
+sharing state. The intention of this model is to allow for highly
+distributed software that, through the adherence to *structured
+concurrency*, results in systems which fail in predictable and
+recoverable ways.
+
+The first step to grok ``tractor`` is to get the basics of ``trio`` down.
+A great place to start is the `trio docs`_ and this `blog post`_.
+
+.. _messages: https://en.wikipedia.org/wiki/Message_passing
+.. _trio docs: https://trio.readthedocs.io/en/latest/
+.. _blog post: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/
+.. _structured concurrency: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/
+.. _3 axioms: https://en.wikipedia.org/wiki/Actor_model#Fundamental_concepts
+.. _unrequirements: https://en.wikipedia.org/wiki/Actor_model#Direct_communication_and_asynchrony
+.. _async generators: https://www.python.org/dev/peps/pep-0525/
+
+
+Install
+-------
+No PyPi release yet!
+
+::
+
+    pip install git+git://github.com/goodboy/tractor.git
+
+
+Alluring Features
+-----------------
+- **It's just** ``trio``, but with SC applied to processes (aka "actors")
+- Infinitely nesteable process trees
+- Built-in API for inter-process streaming
+- A (first ever?) "native" multi-core debugger for Python using `pdb++`_
+- (Soon to land) ``asyncio`` support allowing for "infected" actors where
+  `trio` drives the `asyncio` scheduler via the astounding "`guest mode`_"
+
+
+Example: self-destruct a process tree
+-------------------------------------
+.. literalinclude:: ../../examples/parallelism/we_are_processes.py
+    :language: python
+
+
+The example you're probably after...
+------------------------------------
+It seems the initial query from most new users is "how do I make a worker
+pool thing?".
+
+``tractor`` is built to handle any SC process tree you can
+imagine; the "worker pool" pattern is a trivial special case:
+
+.. literalinclude:: ../../examples/parallelism/concurrent_actors_primes.py
+    :language: python
+
+
+Feel like saying hi?
+--------------------
+This project is very much coupled to the ongoing development of
+``trio`` (i.e. ``tractor`` gets most of its ideas from that brilliant
+community). If you want to help, have suggestions or just want to
+say hi, please feel free to reach us in our `matrix channel`_.  If
+matrix seems too hip, we're also mostly all in the the `trio gitter
+channel`_!
+
+.. _trio gitter channel: https://gitter.im/python-trio/general
+.. _matrix channel: https://matrix.to/#/!tractor:matrix.org
+.. _pdb++: https://github.com/pdbpp/pdbpp
+.. _guest mode: https://trio.readthedocs.io/en/stable/reference-lowlevel.html?highlight=guest%20mode#using-guest-mode-to-run-trio-on-top-of-other-event-loops
+
+
+.. |gh_actions| image:: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fgoodboy%2Ftractor%2Fbadge&style=popout-square
+    :target: https://actions-badge.atrox.dev/goodboy/tractor/goto
+.. |docs| image:: https://readthedocs.org/projects/tractor/badge/?version=latest
+    :target: https://tractor.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status

docs/github_readme/conf.py

@@ -0,0 +1,51 @@
+# Configuration file for the Sphinx documentation builder.
+
+# this config is for the rst generation extension and thus
+# requires only basic settings:
+# https://github.com/sphinx-contrib/restbuilder
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+# Warn about all references to unknown targets
+nitpicky = True
+
+# The master toctree document.
+master_doc = '_sphinx_readme'
+
+# -- Project information -----------------------------------------------------
+
+project = 'tractor'
+copyright = '2018, Tyler Goodlet'
+author = 'Tyler Goodlet'
+
+# The full version, including alpha/beta/rc tags
+release = '0.0.0a0.dev0'
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinxcontrib.restbuilder',
+
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']

docs/index.rst

@@ -0,0 +1,612 @@
+.. tractor documentation master file, created by
+   sphinx-quickstart on Sun Feb  9 22:26:51 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+``tractor``
+===========
+
+A `structured concurrent`_, async-native "`actor model`_" built on trio_ and multiprocessing_.
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Contents:
+
+.. _actor model: https://en.wikipedia.org/wiki/Actor_model
+.. _trio: https://github.com/python-trio/trio
+.. _multiprocessing: https://en.wikipedia.org/wiki/Multiprocessing
+.. _trionic: https://trio.readthedocs.io/en/latest/design.html#high-level-design-principles
+.. _async sandwich: https://trio.readthedocs.io/en/latest/tutorial.html#async-sandwich
+.. _structured concurrent: https://trio.discourse.group/t/concise-definition-of-structured-concurrency/228
+
+
+``tractor`` is an attempt to bring trionic_ `structured concurrency`_ to
+distributed multi-core Python; it aims to be the Python multi-processing
+framework *you always wanted*.
+
+``tractor`` lets you spawn ``trio`` *"actors"*: processes which each run
+a ``trio`` scheduled task tree (also known as an `async sandwich`_).
+*Actors* communicate by exchanging asynchronous messages_ and avoid
+sharing any state. This model allows for highly distributed software
+architecture which works just as well on multiple cores as it does over
+many hosts.
+
+The first step to grok ``tractor`` is to get the basics of ``trio`` down.
+A great place to start is the `trio docs`_ and this `blog post`_.
+
+.. _messages: https://en.wikipedia.org/wiki/Message_passing
+.. _trio docs: https://trio.readthedocs.io/en/latest/
+.. _blog post: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/
+.. _structured concurrency: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/
+
+
+Install
+-------
+No PyPi release yet!
+
+::
+
+    pip install git+git://github.com/goodboy/tractor.git
+
+
+Feel like saying hi?
+--------------------
+This project is very much coupled to the ongoing development of
+``trio`` (i.e. ``tractor`` gets all its ideas from that brilliant
+community). If you want to help, have suggestions or just want to
+say hi, please feel free to ping me on the `trio gitter channel`_!
+
+.. _trio gitter channel: https://gitter.im/python-trio/general
+
+
+
+Philosophy
+----------
+Our tenets non-comprehensively include:
+
+- strict adherence to the `concept-in-progress`_ of *structured concurrency*
+- no spawning of processes *willy-nilly*; causality_ is paramount!
+- (remote) errors `always propagate`_ back to the parent supervisor
+- verbatim support for ``trio``'s cancellation_ system
+- `shared nothing architecture`_
+- no use of *proxy* objects or shared references between processes
+- an immersive debugging experience
+- anti-fragility through `chaos engineering`_
+
+``tractor`` is an actor-model-*like* system in the sense that it adheres
+to the `3 axioms`_ but does not (yet) fulfil all "unrequirements_" in
+practise. It is an experiment in applying `structured concurrency`_
+constraints on a parallel processing system where multiple Python
+processes exist over many hosts but no process can outlive its parent.
+In `erlang` parlance, it is an architecture where every process has
+a mandatory supervisor enforced by the type system. The API design is
+almost exclusively inspired by trio_'s concepts and primitives (though
+we often lag a little). As a distributed computing system `tractor`
+attempts to place sophistication at the correct layer such that
+concurrency primitives are powerful yet simple, making it easy to build
+complex systems (you can build a "worker pool" architecture but it's
+definitely not required). There is first class support for inter-actor
+streaming using `async generators`_ and ongoing work toward a functional
+reactive style for IPC.
+
+.. warning:: ``tractor`` is in alpha-alpha and is expected to change rapidly!
+    Expect nothing to be set in stone. Your ideas about where it should go
+    are greatly appreciated!
+
+.. _concept-in-progress: https://trio.discourse.group/t/structured-concurrency-kickoff/55
+.. _3 axioms: https://en.wikipedia.org/wiki/Actor_model#Fundamental_concepts
+.. _unrequirements: https://en.wikipedia.org/wiki/Actor_model#Direct_communication_and_asynchrony
+.. _async generators: https://www.python.org/dev/peps/pep-0525/
+.. _always propagate: https://trio.readthedocs.io/en/latest/design.html#exceptions-always-propagate
+.. _causality: https://vorpus.org/blog/some-thoughts-on-asynchronous-api-design-in-a-post-asyncawait-world/#c-c-c-c-causality-breaker
+.. _shared nothing architecture: https://en.wikipedia.org/wiki/Shared-nothing_architecture
+.. _cancellation: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-and-timeouts
+.. _channels: https://en.wikipedia.org/wiki/Channel_(programming)
+.. _chaos engineering: http://principlesofchaos.org/
+
+
+Examples
+--------
+Note, if you are on Windows please be sure to see the :ref:`gotchas
+<windowsgotchas>` section before trying these.
+
+
+A trynamic first scene
+**********************
+Let's direct a couple *actors* and have them run their lines for
+the hip new film we're shooting:
+
+.. literalinclude:: ../examples/a_trynamic_first_scene.py
+
+We spawn two *actors*, *donny* and *gretchen*.
+Each actor starts up and executes their *main task* defined by an
+async function, ``say_hello()``.  The function instructs each actor
+to find their partner and say hello by calling their partner's
+``hi()`` function using something called a *portal*. Each actor
+receives a response and relays that back to the parent actor (in
+this case our "director" executing ``main()``).
+
+
+Actor spawning and causality
+****************************
+``tractor`` tries to take ``trio``'s concept of causal task lifetimes
+to multi-process land. Accordingly, ``tractor``'s *actor nursery* behaves
+similar to ``trio``'s nursery_. That is, ``tractor.open_nursery()``
+opens an ``ActorNursery`` which **must** wait on spawned *actors* to complete
+(or error) in the same causal_ way ``trio`` waits on spawned subtasks.
+This includes errors from any one actor causing all other actors
+spawned by the same nursery to be cancelled_.
+
+To spawn an actor and run a function in it, open a *nursery block*
+and use the ``run_in_actor()`` method:
+
+.. literalinclude:: ../examples/actor_spawning_and_causality.py
+
+What's going on?
+
+- an initial *actor* is started with ``trio.run()`` and told to execute
+  its main task_: ``main()``
+
+- inside ``main()`` an actor is *spawned* using an ``ActorNusery`` and is told
+  to run a single function: ``cellar_door()``
+
+- a ``portal`` instance (we'll get to what it is shortly)
+  returned from ``nursery.run_in_actor()`` is used to communicate with
+  the newly spawned *sub-actor*
+
+- the second actor, *some_linguist*, in a new *process* running a new ``trio`` task_
+  then executes ``cellar_door()`` and returns its result over a *channel* back
+  to the parent actor
+
+- the parent actor retrieves the subactor's *final result* using ``portal.result()``
+  much like you'd expect from a future_.
+
+This ``run_in_actor()`` API should look very familiar to users of
+``asyncio``'s `run_in_executor()`_ which uses a ``concurrent.futures`` Executor_.
+
+Since you might also want to spawn long running *worker* or *daemon*
+actors, each actor's *lifetime* can be determined based on the spawn
+method:
+
+- if the actor is spawned using ``run_in_actor()`` it terminates when
+  its *main* task completes (i.e. when the (async) function submitted
+  to it *returns*). The ``with tractor.open_nursery()`` exits only once
+  all actors' main function/task complete (just like the nursery_ in ``trio``)
+
+- actors can be spawned to *live forever* using the ``start_actor()``
+  method and act like an RPC daemon that runs indefinitely (the
+  ``with tractor.open_nursery()`` won't exit) until cancelled_
+
+Here is a similar example using the latter method:
+
+.. literalinclude:: ../examples/actor_spawning_and_causality_with_daemon.py
+
+The ``enable_modules`` `kwarg` above is a list of module path
+strings that will be loaded and made accessible for execution in the
+remote actor through a call to ``Portal.run()``. For now this is
+a simple mechanism to restrict the functionality of the remote
+(and possibly daemonized) actor and uses Python's module system to
+limit the allowed remote function namespace(s).
+
+``tractor`` is opinionated about the underlying threading model used for
+each *actor*. Since Python has a GIL and an actor model by definition
+shares no state between actors, it fits naturally to use a multiprocessing_
+``Process``. This allows ``tractor`` programs to leverage not only multi-core
+hardware but also distribute over many hardware hosts (each *actor* can talk
+to all others with ease over standard network protocols).
+
+.. _task: https://trio.readthedocs.io/en/latest/reference-core.html#tasks-let-you-do-multiple-things-at-once
+.. _nursery: https://trio.readthedocs.io/en/latest/reference-core.html#nurseries-and-spawning
+.. _causal: https://vorpus.org/blog/some-thoughts-on-asynchronous-api-design-in-a-post-asyncawait-world/#causality
+.. _cancelled: https://trio.readthedocs.io/en/latest/reference-core.html#child-tasks-and-cancellation
+.. _run_in_executor(): https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.run_in_executor
+.. _Executor: https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Executor
+
+
+Cancellation
+************
+``tractor`` supports ``trio``'s cancellation_ system verbatim.
+Cancelling a nursery block cancels all actors spawned by it.
+Eventually ``tractor`` plans to support different `supervision strategies`_ like ``erlang``.
+
+.. _supervision strategies: http://erlang.org/doc/man/supervisor.html#sup_flags
+
+
+Remote error propagation
+************************
+Any task invoked in a remote actor should ship any error(s) back to the calling
+actor where it is raised and expected to be dealt with. This way remote actors
+are never cancelled unless explicitly asked or there's a bug in ``tractor`` itself.
+
+.. literalinclude:: ../examples/remote_error_propagation.py
+
+
+You'll notice the nursery cancellation conducts a *one-cancels-all*
+supervisory strategy `exactly like trio`_. The plan is to add more
+`erlang strategies`_ in the near future by allowing nurseries to accept
+a ``Supervisor`` type.
+
+.. _exactly like trio: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-semantics
+.. _erlang strategies: http://learnyousomeerlang.com/supervisors
+
+
+IPC using *portals*
+*******************
+``tractor`` introduces the concept of a *portal* which is an API
+borrowed_ from ``trio``. A portal may seem similar to the idea of
+a RPC future_ except a *portal* allows invoking remote *async* functions and
+generators and intermittently blocking to receive responses. This allows
+for fully async-native IPC between actors.
+
+When you invoke another actor's routines using a *portal* it looks as though
+it was called locally in the current actor. So when you see a call to
+``await portal.run()`` what you get back is what you'd expect
+to if you'd called the function directly in-process. This approach avoids
+the need to add any special RPC *proxy* objects to the library by instead just
+relying on the built-in (async) function calling semantics and protocols of Python.
+
+Depending on the function type ``Portal.run()`` tries to
+correctly interface exactly like a local version of the remote
+built-in Python *function type*. Currently async functions, generators,
+and regular functions are supported. Inspiration for this API comes
+`remote function execution`_ but without the client code being
+concerned about the underlying channels_ system or shipping code
+over the network.
+
+This *portal* approach turns out to be paricularly exciting with the
+introduction of `asynchronous generators`_ in Python 3.6! It means that
+actors can compose nicely in a data streaming pipeline.
+
+.. _exactly like trio: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-semantics
+
+Streaming
+*********
+By now you've figured out that ``tractor`` lets you spawn process based
+*actors* that can invoke cross-process (async) functions and all with
+structured concurrency built in. But the **real cool stuff** is the
+native support for cross-process *streaming*.
+
+
+Asynchronous generators
++++++++++++++++++++++++
+The default streaming function is simply an async generator definition.
+Every value *yielded* from the generator is delivered to the calling
+portal exactly like if you had invoked the function in-process meaning
+you can ``async for`` to receive each value on the calling side.
+
+As an example here's a parent actor that streams for 1 second from a
+spawned subactor:
+
+.. literalinclude:: ../examples/asynchronous_generators.py
+
+By default async generator functions are treated as inter-actor
+*streams* when invoked via a portal (how else could you really interface
+with them anyway) so no special syntax to denote the streaming *service*
+is necessary.
+
+
+Channels and Contexts
++++++++++++++++++++++
+If you aren't fond of having to write an async generator to stream data
+between actors (or need something more flexible) you can instead use
+a ``Context``. A context wraps an actor-local spawned task and
+a ``Channel`` so that tasks executing across multiple processes can
+stream data to one another using a low level, request oriented API.
+
+A ``Channel`` wraps an underlying *transport* and *interchange* format
+to enable *inter-actor-communication*. In its present state ``tractor``
+uses TCP and msgpack_.
+
+As an example if you wanted to create a streaming server without writing
+an async generator that *yields* values you instead define a decorated
+async function:
+
+.. code:: python
+
+   @tractor.stream
+   async def streamer(ctx: tractor.Context, rate: int = 2) -> None:
+      """A simple web response streaming server.
+      """
+      while True:
+         val = await web_request('http://data.feed.com')
+
+         # this is the same as ``yield`` in the async gen case
+         await ctx.send_yield(val)
+
+         await trio.sleep(1 / rate)
+
+
+You must decorate the function with ``@tractor.stream`` and declare
+a ``ctx`` argument as the first in your function signature and then
+``tractor`` will treat the async function like an async generator - as
+a stream from the calling/client side.
+
+This turns out to be handy particularly if you have multiple tasks
+pushing responses concurrently:
+
+.. code:: python
+
+   async def streamer(
+      ctx: tractor.Context,
+      rate: int = 2
+   ) -> None:
+      """A simple web response streaming server.
+      """
+      while True:
+         val = await web_request(url)
+
+         # this is the same as ``yield`` in the async gen case
+         await ctx.send_yield(val)
+
+         await trio.sleep(1 / rate)
+
+
+   @tractor.stream
+   async def stream_multiple_sources(
+      ctx: tractor.Context,
+      sources: List[str]
+   ) -> None:
+      async with trio.open_nursery() as n:
+         for url in sources:
+            n.start_soon(streamer, ctx, url)
+
+
+The context notion comes from the context_ in nanomsg_.
+
+.. _context: https://nanomsg.github.io/nng/man/tip/nng_ctx.5
+.. _msgpack: https://en.wikipedia.org/wiki/MessagePack
+
+
+
+A full fledged streaming service
+++++++++++++++++++++++++++++++++
+Alright, let's get fancy.
+
+Say you wanted to spawn two actors which each pull data feeds from
+two different sources (and wanted this work spread across 2 cpus).
+You also want to aggregate these feeds, do some processing on them and then
+deliver the final result stream to a client (or in this case parent) actor
+and print the results to your screen:
+
+.. literalinclude:: ../examples/full_fledged_streaming_service.py
+
+Here there's four actors running in separate processes (using all the
+cores on you machine). Two are streaming by *yielding* values from the
+``stream_data()`` async generator, one is aggregating values from
+those two in ``aggregate()`` (also an async generator) and shipping the
+single stream of unique values up the parent actor (the ``'MainProcess'``
+as ``multiprocessing`` calls it) which is running ``main()``. 
+
+.. _future: https://en.wikipedia.org/wiki/Futures_and_promises
+.. _borrowed:
+    https://trio.readthedocs.io/en/latest/reference-core.html#getting-back-into-the-trio-thread-from-another-thread
+.. _asynchronous generators: https://www.python.org/dev/peps/pep-0525/
+.. _remote function execution: https://codespeak.net/execnet/example/test_info.html#remote-exec-a-function-avoiding-inlined-source-part-i
+
+
+Actor local (aka *process global*) variables
+********************************************
+Although ``tractor`` uses a *shared-nothing* architecture between
+processes you can of course share state between tasks running *within*
+an actor (since a `trio.run()` runtime is single threaded). ``trio``
+tasks spawned via multiple RPC calls to an actor can modify
+*process-global-state* defined using Python module attributes:
+
+.. code:: python
+
+
+        # a per process cache
+        _actor_cache: dict[str, bool] = {}
+
+
+        def ping_endpoints(endpoints: List[str]):
+            """Start a polling process which runs completely separate
+            from our root actor/process.
+
+            """
+
+            # This runs in a new process so no changes # will propagate
+            # back to the parent actor
+            while True:
+
+                for ep in endpoints:
+                    status = await check_endpoint_is_up(ep)
+                    _actor_cache[ep] = status
+
+                await trio.sleep(0.5)
+
+
+        async def get_alive_endpoints():
+
+            nonlocal _actor_cache
+
+            return {key for key, value in _actor_cache.items() if value}
+
+
+        async def main():
+
+            async with tractor.open_nursery() as n:
+
+                portal = await n.run_in_actor(ping_endpoints)
+
+                # print the alive endpoints after 3 seconds
+                await trio.sleep(3)
+
+                # this is submitted to be run in our "ping_endpoints" actor
+                print(await portal.run(get_alive_endpoints))
+
+
+You can pass any kind of (`msgpack`) serializable data between actors using
+function call semantics but building out a state sharing system per-actor
+is totally up to you.
+
+
+Service Discovery
+*****************
+Though it will be built out much more in the near future, ``tractor``
+currently keeps track of actors by ``(name: str, id: str)`` using a
+special actor called the *arbiter*. Currently the *arbiter* must exist
+on a host (or it will be created if one can't be found) and keeps a
+simple ``dict`` of actor names to sockets for discovery by other actors.
+Obviously this can be made more sophisticated (help me with it!) but for
+now it does the trick.
+
+To find the arbiter from the current actor use the ``get_arbiter()`` function and to
+find an actor's socket address by name use the ``find_actor()`` function:
+
+.. literalinclude:: ../examples/service_discovery.py
+
+The ``name`` value you should pass to ``find_actor()`` is the one you passed as the
+*first* argument to either ``trio.run()`` or ``ActorNursery.start_actor()``.
+
+
+Running actors standalone
+*************************
+You don't have to spawn any actors using ``open_nursery()`` if you just
+want to run a single actor that connects to an existing cluster.
+All the comms and arbiter registration stuff still works. This can
+somtimes turn out being handy when debugging mult-process apps when you
+need to hop into a debugger. You just need to pass the existing
+*arbiter*'s socket address you'd like to connect to:
+
+.. code:: python
+
+    import trio
+    import tractor
+
+    async def main():
+
+        async with tractor.open_root_actor(
+            arbiter_addr=('192.168.0.10', 1616)
+        ):
+            await trio.sleep_forever()
+
+    trio.run(main)
+
+
+Choosing a process spawning backend
+***********************************
+``tractor`` is architected to support multiple actor (sub-process)
+spawning backends. Specific defaults are chosen based on your system
+but you can also explicitly select a backend of choice at startup
+via a ``start_method`` kwarg to ``tractor.open_nursery()``.
+
+Currently the options available are:
+
+- ``trio``: a ``trio``-native spawner which is an async wrapper around ``subprocess``
+- ``spawn``: one of the stdlib's ``multiprocessing`` `start methods`_
+- ``forkserver``: a faster ``multiprocessing`` variant that is Unix only
+
+.. _start methods: https://docs.python.org/3.8/library/multiprocessing.html#contexts-and-start-methods
+
+
+``trio``
+++++++++
+The ``trio`` backend offers a lightweight async wrapper around ``subprocess`` from the standard library and takes advantage of the ``trio.`` `open_process`_ API.
+
+.. _open_process: https://trio.readthedocs.io/en/stable/reference-io.html#spawning-subprocesses
+
+
+``multiprocessing``
++++++++++++++++++++
+There is support for the stdlib's ``multiprocessing`` `start methods`_.
+Note that on Windows *spawn* it the only supported method and on \*nix
+systems *forkserver* is the best method for speed but has the caveat
+that it will break easily (hangs due to broken pipes) if spawning actors
+using nested nurseries.
+
+In general, the ``multiprocessing`` backend **has not proven reliable**
+for handling errors from actors more then 2 nurseries *deep* (see `#89`_).
+If you for some reason need this consider sticking with alternative
+backends.
+
+.. _#89: https://github.com/goodboy/tractor/issues/89
+
+.. _windowsgotchas:
+
+Windows "gotchas"
+^^^^^^^^^^^^^^^^^
+On Windows (which requires the use of the stdlib's `multiprocessing`
+package) there are some gotchas. Namely, the need for calling
+`freeze_support()`_ inside the ``__main__`` context.  Additionally you
+may need place you `tractor` program entry point in a seperate
+`__main__.py` module in your package in order to avoid an error like the
+following ::
+
+    Traceback (most recent call last):
+      File "C:\ProgramData\Miniconda3\envs\tractor19030601\lib\site-packages\tractor\_actor.py", line 234, in _get_rpc_func
+        return getattr(self._mods[ns], funcname)
+    KeyError: '__mp_main__'
+
+
+To avoid this, the following is the **only code** that should be in your
+main python module of the program:
+
+.. code:: python
+
+    # application/__main__.py
+    import trio
+    import tractor
+    import multiprocessing
+    from . import tractor_app
+
+    if __name__ == '__main__':
+        multiprocessing.freeze_support()
+        trio.run(tractor_app.main)
+
+And execute as::
+
+    python -m application
+
+
+As an example we use the following code to test all documented examples
+in the test suite on windows:
+
+.. literalinclude:: ../examples/__main__.py
+
+See `#61`_ and `#79`_ for further details.
+
+.. _freeze_support(): https://docs.python.org/3/library/multiprocessing.html#multiprocessing.freeze_support
+.. _#61: https://github.com/goodboy/tractor/pull/61#issuecomment-470053512
+.. _#79: https://github.com/goodboy/tractor/pull/79
+
+
+Enabling logging
+****************
+Considering how complicated distributed software can become it helps to know
+what exactly it's doing (even at the lowest levels). Luckily ``tractor`` has
+tons of logging throughout the core. ``tractor`` isn't opinionated on
+how you use this information and users are expected to consume log messages in
+whichever way is appropriate for the system at hand. That being said, when hacking
+on ``tractor`` there is a prettified console formatter which you can enable to
+see what the heck is going on. Just put the following somewhere in your code:
+
+.. code:: python
+
+    from tractor.log import get_console_log
+    log = get_console_log('trace')
+
+
+What the future holds
+---------------------
+Stuff I'd like to see ``tractor`` do real soon:
+
+- TLS_, duh.
+- erlang-like supervisors_
+- native support for `nanomsg`_ as a channel transport
+- native `gossip protocol`_ support for service discovery and arbiter election
+- a distributed log ledger for tracking cluster behaviour
+- a slick multi-process aware debugger much like in celery_
+  but with better `pdb++`_ support
+- an extensive `chaos engineering`_ test suite
+- support for reactive programming primitives and native support for asyncitertools_ like libs
+- introduction of a `capability-based security`_ model
+
+.. _TLS: https://trio.readthedocs.io/en/latest/reference-io.html#ssl-tls-support
+.. _supervisors: https://github.com/goodboy/tractor/issues/22
+.. _nanomsg: https://nanomsg.github.io/nng/index.html
+.. _gossip protocol: https://en.wikipedia.org/wiki/Gossip_protocol
+.. _celery: http://docs.celeryproject.org/en/latest/userguide/debugging.html
+.. _asyncitertools: https://github.com/vodik/asyncitertools
+.. _pdb++: https://github.com/antocuni/pdb
+.. _capability-based security: https://en.wikipedia.org/wiki/Capability-based_security

docs/mk_gh_readme.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+sphinx-build -b rst ./github_readme ./
+
+mv _sphinx_readme.rst _README.rst

examples/__main__.py

@@ -0,0 +1,19 @@
+"""
+Needed on Windows.
+
+This module is needed as the program entry point for invocation
+with ``python -m <modulename>``. See the solution from @chrizzFTD
+here:
+
+    https://github.com/goodboy/tractor/pull/61#issuecomment-470053512
+
+"""
+if __name__ == '__main__':
+    import multiprocessing
+    multiprocessing.freeze_support()
+    # ``tests/test_docs_examples.py::test_example`` will copy each
+    # script from this examples directory into a module in a new
+    # temporary dir and name it test_example.py. We import that script
+    # module here and invoke it's ``main()``.
+    from . import test_example
+    test_example.trio.run(test_example.main)

examples/a_trynamic_first_scene.py

@@ -0,0 +1,44 @@
+import trio
+import tractor
+
+_this_module = __name__
+the_line = 'Hi my name is {}'
+
+
+tractor.log.get_console_log("INFO")
+
+
+async def hi():
+    return the_line.format(tractor.current_actor().name)
+
+
+async def say_hello(other_actor):
+    async with tractor.wait_for_actor(other_actor) as portal:
+        return await portal.run(hi)
+
+
+async def main():
+    """Main tractor entry point, the "master" process (for now
+    acts as the "director").
+    """
+    async with tractor.open_nursery() as n:
+        print("Alright... Action!")
+
+        donny = await n.run_in_actor(
+            say_hello,
+            name='donny',
+            # arguments are always named
+            other_actor='gretchen',
+        )
+        gretchen = await n.run_in_actor(
+            say_hello,
+            name='gretchen',
+            other_actor='donny',
+        )
+        print(await gretchen.result())
+        print(await donny.result())
+        print("CUTTTT CUUTT CUT!!! Donny!! You're supposed to say...")
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/actor_spawning_and_causality.py

@@ -0,0 +1,27 @@
+import trio
+import tractor
+
+
+async def cellar_door():
+    assert not tractor.is_root_process()
+    return "Dang that's beautiful"
+
+
+async def main():
+    """The main ``tractor`` routine.
+    """
+    async with tractor.open_nursery() as n:
+
+        portal = await n.run_in_actor(
+            cellar_door,
+            name='some_linguist',
+        )
+
+    # The ``async with`` will unblock here since the 'some_linguist'
+    # actor has completed its main task ``cellar_door``.
+
+    print(await portal.result())
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/actor_spawning_and_causality_with_daemon.py

@@ -0,0 +1,34 @@
+import trio
+import tractor
+
+
+async def movie_theatre_question():
+    """A question asked in a dark theatre, in a tangent
+    (errr, I mean different) process.
+    """
+    return 'have you ever seen a portal?'
+
+
+async def main():
+    """The main ``tractor`` routine.
+    """
+    async with tractor.open_nursery() as n:
+
+        portal = await n.start_actor(
+            'frank',
+            # enable the actor to run funcs from this current module
+            enable_modules=[__name__],
+        )
+
+        print(await portal.run(movie_theatre_question))
+        # call the subactor a 2nd time
+        print(await portal.run(movie_theatre_question))
+
+        # the async with will block here indefinitely waiting
+        # for our actor "frank" to complete, but since it's an
+        # "outlive_main" actor it will never end until cancelled
+        await portal.cancel_actor()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/advanced_faults/ipc_failure_during_stream.py

@@ -0,0 +1,151 @@
+'''
+Complex edge case where during real-time streaming the IPC tranport
+channels are wiped out (purposely in this example though it could have
+been an outage) and we want to ensure that despite being in debug mode
+(or not) the user can sent SIGINT once they notice the hang and the
+actor tree will eventually be cancelled without leaving any zombies.
+
+'''
+import trio
+from tractor import (
+    open_nursery,
+    context,
+    Context,
+    MsgStream,
+)
+
+
+async def break_channel_silently_then_error(
+    stream: MsgStream,
+):
+    async for msg in stream:
+        await stream.send(msg)
+
+        # XXX: close the channel right after an error is raised
+        # purposely breaking the IPC transport to make sure the parent
+        # doesn't get stuck in debug or hang on the connection join.
+        # this more or less simulates an infinite msg-receive hang on
+        # the other end.
+        await stream._ctx.chan.send(None)
+        assert 0
+
+
+async def close_stream_and_error(
+    stream: MsgStream,
+):
+    async for msg in stream:
+        await stream.send(msg)
+
+        # wipe out channel right before raising
+        await stream._ctx.chan.send(None)
+        await stream.aclose()
+        assert 0
+
+
+@context
+async def recv_and_spawn_net_killers(
+
+    ctx: Context,
+    break_ipc_after: bool | int = False,
+
+) -> None:
+    '''
+    Receive stream msgs and spawn some IPC killers mid-stream.
+
+    '''
+    await ctx.started()
+    async with (
+        ctx.open_stream() as stream,
+        trio.open_nursery() as n,
+    ):
+        async for i in stream:
+            print(f'child echoing {i}')
+            await stream.send(i)
+            if (
+                break_ipc_after
+                and i > break_ipc_after
+            ):
+                '#################################\n'
+                'Simulating child-side IPC BREAK!\n'
+                '#################################'
+                n.start_soon(break_channel_silently_then_error, stream)
+                n.start_soon(close_stream_and_error, stream)
+
+
+async def main(
+    debug_mode: bool = False,
+    start_method: str = 'trio',
+
+    # by default we break the parent IPC first (if configured to break
+    # at all), but this can be changed so the child does first (even if
+    # both are set to break).
+    break_parent_ipc_after: int | bool = False,
+    break_child_ipc_after: int | bool = False,
+
+) -> None:
+
+    async with (
+        open_nursery(
+            start_method=start_method,
+
+            # NOTE: even debugger is used we shouldn't get
+            # a hang since it never engages due to broken IPC
+            debug_mode=debug_mode,
+            loglevel='warning',
+
+        ) as an,
+    ):
+        portal = await an.start_actor(
+            'chitty_hijo',
+            enable_modules=[__name__],
+        )
+
+        async with portal.open_context(
+            recv_and_spawn_net_killers,
+            break_ipc_after=break_child_ipc_after,
+
+        ) as (ctx, sent):
+            async with ctx.open_stream() as stream:
+                for i in range(1000):
+
+                    if (
+                        break_parent_ipc_after
+                        and i > break_parent_ipc_after
+                    ):
+                        print(
+                            '#################################\n'
+                            'Simulating parent-side IPC BREAK!\n'
+                            '#################################'
+                        )
+                        await stream._ctx.chan.send(None)
+
+                    # it actually breaks right here in the
+                    # mp_spawn/forkserver backends and thus the zombie
+                    # reaper never even kicks in?
+                    print(f'parent sending {i}')
+                    await stream.send(i)
+
+                    with trio.move_on_after(2) as cs:
+
+                        # NOTE: in the parent side IPC failure case this
+                        # will raise an ``EndOfChannel`` after the child
+                        # is killed and sends a stop msg back to it's
+                        # caller/this-parent.
+                        rx = await stream.receive()
+
+                        print(f"I'm a happy user and echoed to me is {rx}")
+
+                    if cs.cancelled_caught:
+                        # pretend to be a user seeing no streaming action
+                        # thinking it's a hang, and then hitting ctl-c..
+                        print("YOO i'm a user anddd thingz hangin..")
+
+                print(
+                    "YOO i'm mad send side dun but thingz hangin..\n"
+                    'MASHING CTlR-C Ctl-c..'
+                )
+                raise KeyboardInterrupt
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/asynchronous_generators.py

@@ -0,0 +1,42 @@
+from typing import AsyncIterator
+from itertools import repeat
+
+import trio
+import tractor
+
+
+async def stream_forever() -> AsyncIterator[int]:
+
+    for i in repeat("I can see these little future bubble things"):
+        # each yielded value is sent over the ``Channel`` to the parent actor
+        yield i
+        await trio.sleep(0.01)
+
+
+async def main():
+
+    async with tractor.open_nursery() as n:
+
+        portal = await n.start_actor(
+            'donny',
+            enable_modules=[__name__],
+        )
+
+        # this async for loop streams values from the above
+        # async generator running in a separate process
+        async with portal.open_stream_from(stream_forever) as stream:
+            count = 0
+            async for letter in stream:
+                print(letter)
+                count += 1
+
+                if count > 50:
+                    break
+
+        print('stream terminated')
+
+        await portal.cancel_actor()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/fast_error_in_root_after_spawn.py

@@ -0,0 +1,54 @@
+'''
+Fast fail test with a context.
+
+Ensure the partially initialized sub-actor process
+doesn't cause a hang on error/cancel of the parent
+nursery.
+
+'''
+import trio
+import tractor
+
+
+@tractor.context
+async def sleep(
+    ctx: tractor.Context,
+):
+    await trio.sleep(0.5)
+    await ctx.started()
+    await trio.sleep_forever()
+
+
+async def open_ctx(
+    n: tractor._supervise.ActorNursery
+):
+
+    # spawn both actors
+    portal = await n.start_actor(
+        name='sleeper',
+        enable_modules=[__name__],
+    )
+
+    async with portal.open_context(
+        sleep,
+    ) as (ctx, first):
+        assert first is None
+
+
+async def main():
+
+    async with tractor.open_nursery(
+        debug_mode=True,
+        loglevel='runtime',
+    ) as an:
+
+        async with trio.open_nursery() as n:
+            n.start_soon(open_ctx, an)
+
+            await trio.sleep(0.2)
+            await trio.sleep(0.1)
+            assert 0
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/multi_daemon_subactors.py

@@ -0,0 +1,45 @@
+import tractor
+import trio
+
+
+async def breakpoint_forever():
+    "Indefinitely re-enter debugger in child actor."
+    while True:
+        yield 'yo'
+        await tractor.breakpoint()
+
+
+async def name_error():
+    "Raise a ``NameError``"
+    getattr(doggypants)  # noqa
+
+
+async def main():
+    """Test breakpoint in a streaming actor.
+    """
+    async with tractor.open_nursery(
+        debug_mode=True,
+        loglevel='error',
+    ) as n:
+
+        p0 = await n.start_actor('bp_forever', enable_modules=[__name__])
+        p1 = await n.start_actor('name_error', enable_modules=[__name__])
+
+        # retreive results
+        async with p0.open_stream_from(breakpoint_forever) as stream:
+
+            # triggers the first name error
+            try:
+                await p1.run(name_error)
+            except tractor.RemoteActorError as rae:
+                assert rae.type is NameError
+
+            async for i in stream:
+
+                # a second time try the failing subactor and this tie
+                # let error propagate up to the parent/nursery.
+                await p1.run(name_error)
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/multi_nested_subactors_error_up_through_nurseries.py

@@ -0,0 +1,98 @@
+import trio
+import tractor
+
+
+async def name_error():
+    "Raise a ``NameError``"
+    getattr(doggypants)  # noqa
+
+
+async def breakpoint_forever():
+    "Indefinitely re-enter debugger in child actor."
+    while True:
+        await tractor.breakpoint()
+
+        # NOTE: if the test never sent 'q'/'quit' commands
+        # on the pdb repl, without this checkpoint line the
+        # repl would spin in this actor forever.
+        # await trio.sleep(0)
+
+
+async def spawn_until(depth=0):
+    """"A nested nursery that triggers another ``NameError``.
+    """
+    async with tractor.open_nursery() as n:
+        if depth < 1:
+
+            await n.run_in_actor(breakpoint_forever)
+
+            p = await n.run_in_actor(
+                name_error,
+                name='name_error'
+            )
+            await trio.sleep(0.5)
+            # rx and propagate error from child
+            await p.result()
+
+        else:
+            # recusrive call to spawn another process branching layer of
+            # the tree
+            depth -= 1
+            await n.run_in_actor(
+                spawn_until,
+                depth=depth,
+                name=f'spawn_until_{depth}',
+            )
+
+
+async def main():
+    """The main ``tractor`` routine.
+
+    The process tree should look as approximately as follows when the debugger
+    first engages:
+
+    python examples/debugging/multi_nested_subactors_bp_forever.py
+    ├─ python -m tractor._child --uid ('spawner1', '7eab8462 ...)
+    │  └─ python -m tractor._child --uid ('spawn_until_3', 'afcba7a8 ...)
+    │     └─ python -m tractor._child --uid ('spawn_until_2', 'd2433d13 ...)
+    │        └─ python -m tractor._child --uid ('spawn_until_1', '1df589de ...)
+    │           └─ python -m tractor._child --uid ('spawn_until_0', '3720602b ...)
+    │
+    └─ python -m tractor._child --uid ('spawner0', '1d42012b ...)
+       └─ python -m tractor._child --uid ('spawn_until_2', '2877e155 ...)
+          └─ python -m tractor._child --uid ('spawn_until_1', '0502d786 ...)
+             └─ python -m tractor._child --uid ('spawn_until_0', 'de918e6d ...)
+
+    """
+    async with tractor.open_nursery(
+        debug_mode=True,
+        # loglevel='cancel',
+    ) as n:
+
+        # spawn both actors
+        portal = await n.run_in_actor(
+            spawn_until,
+            depth=3,
+            name='spawner0',
+        )
+        portal1 = await n.run_in_actor(
+            spawn_until,
+            depth=4,
+            name='spawner1',
+        )
+
+        # TODO: test this case as well where the parent don't see
+        # the sub-actor errors by default and instead expect a user
+        # ctrl-c to kill the root.
+        with trio.move_on_after(3):
+            await trio.sleep_forever()
+
+        # gah still an issue here.
+        await portal.result()
+
+        # should never get here
+        await portal1.result()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/multi_subactor_root_errors.py

@@ -0,0 +1,66 @@
+'''
+Test that a nested nursery will avoid clobbering
+the debugger latched by a broken child.
+
+'''
+import trio
+import tractor
+
+
+async def name_error():
+    "Raise a ``NameError``"
+    getattr(doggypants)  # noqa
+
+
+async def spawn_error():
+    """"A nested nursery that triggers another ``NameError``.
+    """
+    async with tractor.open_nursery() as n:
+        portal = await n.run_in_actor(
+            name_error,
+            name='name_error_1',
+            )
+        return await portal.result()
+
+
+async def main():
+    """The main ``tractor`` routine.
+
+    The process tree should look as approximately as follows:
+
+    python examples/debugging/multi_subactors.py
+    ├─ python -m tractor._child --uid ('name_error', 'a7caf490 ...)
+    `-python -m tractor._child --uid ('spawn_error', '52ee14a5 ...)
+       `-python -m tractor._child --uid ('name_error', '3391222c ...)
+
+    Order of failure:
+        - nested name_error sub-sub-actor
+        - root actor should then fail on assert
+        - program termination
+    """
+    async with tractor.open_nursery(
+        debug_mode=True,
+        # loglevel='cancel',
+    ) as n:
+
+        # spawn both actors
+        portal = await n.run_in_actor(
+            name_error,
+            name='name_error',
+        )
+        portal1 = await n.run_in_actor(
+            spawn_error,
+            name='spawn_error',
+        )
+
+        # trigger a root actor error
+        assert 0
+
+        # attempt to collect results (which raises error in parent)
+        # still has some issues where the parent seems to get stuck
+        await portal.result()
+        await portal1.result()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/multi_subactors.py

@@ -0,0 +1,52 @@
+import tractor
+import trio
+
+
+async def breakpoint_forever():
+    "Indefinitely re-enter debugger in child actor."
+    while True:
+        await trio.sleep(0.1)
+        await tractor.breakpoint()
+
+
+async def name_error():
+    "Raise a ``NameError``"
+    getattr(doggypants)  # noqa
+
+
+async def spawn_error():
+    """"A nested nursery that triggers another ``NameError``.
+    """
+    async with tractor.open_nursery() as n:
+        portal = await n.run_in_actor(
+            name_error,
+            name='name_error_1',
+        )
+        return await portal.result()
+
+
+async def main():
+    """The main ``tractor`` routine.
+
+    The process tree should look as approximately as follows:
+
+    -python examples/debugging/multi_subactors.py
+    |-python -m tractor._child --uid ('name_error', 'a7caf490 ...)
+    |-python -m tractor._child --uid ('bp_forever', '1f787a7e ...)
+    `-python -m tractor._child --uid ('spawn_error', '52ee14a5 ...)
+       `-python -m tractor._child --uid ('name_error', '3391222c ...)
+    """
+    async with tractor.open_nursery(
+        debug_mode=True,
+    ) as n:
+
+        # Spawn both actors, don't bother with collecting results
+        # (would result in a different debugger outcome due to parent's
+        # cancellation).
+        await n.run_in_actor(breakpoint_forever)
+        await n.run_in_actor(name_error)
+        await n.run_in_actor(spawn_error)
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/open_ctx_modnofound.py

@@ -0,0 +1,40 @@
+import trio
+import tractor
+
+
+@tractor.context
+async def just_sleep(
+
+    ctx: tractor.Context,
+    **kwargs,
+
+) -> None:
+    '''
+    Start and sleep.
+
+    '''
+    await ctx.started()
+    await trio.sleep_forever()
+
+
+async def main() -> None:
+
+    async with tractor.open_nursery(
+        debug_mode=True,
+    ) as n:
+        portal = await n.start_actor(
+            'ctx_child',
+
+            # XXX: we don't enable the current module in order
+            # to trigger `ModuleNotFound`.
+            enable_modules=[],
+        )
+
+        async with portal.open_context(
+            just_sleep,  # taken from pytest parameterization
+        ) as (ctx, sent):
+            raise KeyboardInterrupt
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/per_actor_debug.py

@@ -0,0 +1,27 @@
+import trio
+import tractor
+
+async def die():
+    raise RuntimeError
+
+
+async def main():
+    async with tractor.open_nursery() as tn:
+
+        debug_actor = await tn.start_actor(
+            'debugged_boi',
+            enable_modules=[__name__],
+            debug_mode=True,
+        )
+        crash_boi = await tn.start_actor(
+            'crash_boi',
+            enable_modules=[__name__],
+            # debug_mode=True,
+        )
+
+        async with trio.open_nursery() as n:
+            n.start_soon(debug_actor.run, die)
+            n.start_soon(crash_boi.run, die)
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/restore_builtin_breakpoint.py

@@ -0,0 +1,24 @@
+import os
+import sys
+
+import trio
+import tractor
+
+
+async def main() -> None:
+    async with tractor.open_nursery(debug_mode=True) as an:
+
+        assert os.environ['PYTHONBREAKPOINT'] == 'tractor._debug._set_trace'
+
+        # TODO: an assert that verifies the hook has indeed been, hooked
+        # XD
+        assert sys.breakpointhook is not tractor._debug._set_trace
+
+        breakpoint()
+
+    # TODO: an assert that verifies the hook is unhooked..
+    assert sys.breakpointhook
+    breakpoint()
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/root_actor_breakpoint.py

@@ -0,0 +1,19 @@
+import trio
+import tractor
+
+
+async def main():
+
+    async with tractor.open_root_actor(
+        debug_mode=True,
+    ):
+
+        await trio.sleep(0.1)
+
+        await tractor.breakpoint()
+
+        await trio.sleep(0.1)
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/root_actor_breakpoint_forever.py

@@ -0,0 +1,15 @@
+import trio
+import tractor
+
+
+async def main():
+
+    async with tractor.open_root_actor(
+        debug_mode=True,
+    ):
+        while True:
+            await tractor.breakpoint()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/root_actor_error.py

@@ -0,0 +1,13 @@
+import trio
+import tractor
+
+
+async def main():
+    async with tractor.open_root_actor(
+        debug_mode=True,
+    ):
+        assert 0
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/root_cancelled_but_child_is_in_tty_lock.py

@@ -0,0 +1,65 @@
+import trio
+import tractor
+
+
+async def name_error():
+    "Raise a ``NameError``"
+    getattr(doggypants)  # noqa
+
+
+async def spawn_until(depth=0):
+    """"A nested nursery that triggers another ``NameError``.
+    """
+    async with tractor.open_nursery() as n:
+        if depth < 1:
+            # await n.run_in_actor('breakpoint_forever', breakpoint_forever)
+            await n.run_in_actor(name_error)
+        else:
+            depth -= 1
+            await n.run_in_actor(
+                spawn_until,
+                depth=depth,
+                name=f'spawn_until_{depth}',
+            )
+
+
+async def main():
+    """The main ``tractor`` routine.
+
+    The process tree should look as approximately as follows when the debugger
+    first engages:
+
+    python examples/debugging/multi_nested_subactors_bp_forever.py
+    ├─ python -m tractor._child --uid ('spawner1', '7eab8462 ...)
+    │  └─ python -m tractor._child --uid ('spawn_until_0', '3720602b ...)
+    │     └─ python -m tractor._child --uid ('name_error', '505bf71d ...)
+    │
+    └─ python -m tractor._child --uid ('spawner0', '1d42012b ...)
+       └─ python -m tractor._child --uid ('name_error', '6c2733b8 ...)
+
+    """
+    async with tractor.open_nursery(
+        debug_mode=True,
+        loglevel='warning'
+    ) as n:
+
+        # spawn both actors
+        portal = await n.run_in_actor(
+            spawn_until,
+            depth=0,
+            name='spawner0',
+        )
+        portal1 = await n.run_in_actor(
+            spawn_until,
+            depth=1,
+            name='spawner1',
+        )
+
+        # nursery cancellation should be triggered due to propagated
+        # error from child.
+        await portal.result()
+        await portal1.result()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/root_timeout_while_child_crashed.py

@@ -0,0 +1,31 @@
+
+import trio
+import tractor
+
+
+async def key_error():
+    "Raise a ``NameError``"
+    return {}['doggy']
+
+
+async def main():
+    """Root dies 
+
+    """
+    async with tractor.open_nursery(
+        debug_mode=True,
+        loglevel='debug'
+    ) as n:
+
+        # spawn both actors
+        portal = await n.run_in_actor(key_error)
+
+        # XXX: originally a bug caused by this is where root would enter
+        # the debugger and clobber the tty used by the repl even though
+        # child should have it locked.
+        with trio.fail_after(1):
+            await trio.Event().wait()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/subactor_bp_in_ctx.py

@@ -0,0 +1,50 @@
+import tractor
+import trio
+
+
+async def gen():
+    yield 'yo'
+    await tractor.breakpoint()
+    yield 'yo'
+    await tractor.breakpoint()
+
+
+@tractor.context
+async def just_bp(
+    ctx: tractor.Context,
+) -> None:
+
+    await ctx.started()
+    await tractor.breakpoint()
+
+    # TODO: bps and errors in this call..
+    async for val in gen():
+        print(val)
+
+    # await trio.sleep(0.5)
+
+    # prematurely destroy the connection
+    await ctx.chan.aclose()
+
+    # THIS CAUSES AN UNRECOVERABLE HANG
+    # without latest ``pdbpp``:
+    assert 0
+
+
+
+async def main():
+    async with tractor.open_nursery(
+        debug_mode=True,
+    ) as n:
+        p = await n.start_actor(
+            'bp_boi',
+            enable_modules=[__name__],
+        )
+        async with p.open_context(
+            just_bp,
+        ) as (ctx, first):
+            await trio.sleep_forever()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/subactor_breakpoint.py

@@ -0,0 +1,26 @@
+import trio
+import tractor
+
+
+async def breakpoint_forever():
+    """Indefinitely re-enter debugger in child actor.
+    """
+    while True:
+        await trio.sleep(0.1)
+        await tractor.breakpoint()
+
+
+async def main():
+
+    async with tractor.open_nursery(
+        debug_mode=True,
+    ) as n:
+
+        portal = await n.run_in_actor(
+            breakpoint_forever,
+        )
+        await portal.result()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/subactor_error.py

@@ -0,0 +1,19 @@
+import trio
+import tractor
+
+
+async def name_error():
+    getattr(doggypants)
+
+
+async def main():
+    async with tractor.open_nursery(
+        debug_mode=True,
+    ) as n:
+
+        portal = await n.run_in_actor(name_error)
+        await portal.result()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/full_fledged_streaming_service.py

@@ -0,0 +1,104 @@
+import time
+import trio
+import tractor
+
+
+# this is the first 2 actors, streamer_1 and streamer_2
+async def stream_data(seed):
+    for i in range(seed):
+        yield i
+        await trio.sleep(0.0001)  # trigger scheduler
+
+
+# this is the third actor; the aggregator
+async def aggregate(seed):
+    """Ensure that the two streams we receive match but only stream
+    a single set of values to the parent.
+    """
+    async with tractor.open_nursery() as nursery:
+        portals = []
+        for i in range(1, 3):
+            # fork point
+            portal = await nursery.start_actor(
+                name=f'streamer_{i}',
+                enable_modules=[__name__],
+            )
+
+            portals.append(portal)
+
+        send_chan, recv_chan = trio.open_memory_channel(500)
+
+        async def push_to_chan(portal, send_chan):
+
+            # TODO: https://github.com/goodboy/tractor/issues/207
+            async with send_chan:
+                async with portal.open_stream_from(stream_data, seed=seed) as stream:
+                    async for value in stream:
+                        # leverage trio's built-in backpressure
+                        await send_chan.send(value)
+
+            print(f"FINISHED ITERATING {portal.channel.uid}")
+
+        # spawn 2 trio tasks to collect streams and push to a local queue
+        async with trio.open_nursery() as n:
+
+            for portal in portals:
+                n.start_soon(push_to_chan, portal, send_chan.clone())
+
+            # close this local task's reference to send side
+            await send_chan.aclose()
+
+            unique_vals = set()
+            async with recv_chan:
+                async for value in recv_chan:
+                    if value not in unique_vals:
+                        unique_vals.add(value)
+                        # yield upwards to the spawning parent actor
+                        yield value
+
+                assert value in unique_vals
+
+            print("FINISHED ITERATING in aggregator")
+
+        await nursery.cancel()
+        print("WAITING on `ActorNursery` to finish")
+    print("AGGREGATOR COMPLETE!")
+
+
+# this is the main actor and *arbiter*
+async def main():
+    # a nursery which spawns "actors"
+    async with tractor.open_nursery(
+        arbiter_addr=('127.0.0.1', 1616)
+    ) as nursery:
+
+        seed = int(1e3)
+        pre_start = time.time()
+
+        portal = await nursery.start_actor(
+            name='aggregator',
+            enable_modules=[__name__],
+        )
+
+        async with portal.open_stream_from(
+            aggregate,
+            seed=seed,
+        ) as stream:
+
+            start = time.time()
+            # the portal call returns exactly what you'd expect
+            # as if the remote "aggregate" function was called locally
+            result_stream = []
+            async for value in stream:
+                result_stream.append(value)
+
+        await portal.cancel_actor()
+
+        print(f"STREAM TIME = {time.time() - start}")
+        print(f"STREAM + SPAWN TIME = {time.time() - pre_start}")
+        assert result_stream == list(range(seed))
+        return result_stream
+
+
+if __name__ == '__main__':
+    final_stream = trio.run(main)

examples/infected_asyncio_echo_server.py

@@ -0,0 +1,92 @@
+'''
+An SC compliant infected ``asyncio`` echo server.
+
+'''
+import asyncio
+from statistics import mean
+import time
+
+import trio
+import tractor
+
+
+async def aio_echo_server(
+    to_trio: trio.MemorySendChannel,
+    from_trio: asyncio.Queue,
+
+) -> None:
+
+    # a first message must be sent **from** this ``asyncio``
+    # task or the ``trio`` side will never unblock from
+    # ``tractor.to_asyncio.open_channel_from():``
+    to_trio.send_nowait('start')
+
+    # XXX: this uses an ``from_trio: asyncio.Queue`` currently but we
+    # should probably offer something better.
+    while True:
+        # echo the msg back
+        to_trio.send_nowait(await from_trio.get())
+        await asyncio.sleep(0)
+
+
+@tractor.context
+async def trio_to_aio_echo_server(
+    ctx: tractor.Context,
+):
+    # this will block until the ``asyncio`` task sends a "first"
+    # message.
+    async with tractor.to_asyncio.open_channel_from(
+        aio_echo_server,
+    ) as (first, chan):
+
+        assert first == 'start'
+        await ctx.started(first)
+
+        async with ctx.open_stream() as stream:
+
+            async for msg in stream:
+                await chan.send(msg)
+
+                out = await chan.receive()
+                # echo back to parent actor-task
+                await stream.send(out)
+
+
+async def main():
+
+    async with tractor.open_nursery() as n:
+        p = await n.start_actor(
+            'aio_server',
+            enable_modules=[__name__],
+            infect_asyncio=True,
+        )
+        async with p.open_context(
+            trio_to_aio_echo_server,
+        ) as (ctx, first):
+
+            assert first == 'start'
+
+            count = 0
+            async with ctx.open_stream() as stream:
+
+                delays = []
+                send = time.time()
+
+                await stream.send(count)
+                async for msg in stream:
+                    recv = time.time()
+                    delays.append(recv - send)
+                    assert msg == count
+                    count += 1
+                    send = time.time()
+                    await stream.send(count)
+
+                    if count >= 1e3:
+                        break
+
+        print(f'mean round trip rate (Hz): {1/mean(delays)}')
+        await p.cancel_actor()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/integration/open_context_and_sleep.py

@@ -0,0 +1,49 @@
+import trio
+import click
+import tractor
+import pydantic
+# from multiprocessing import shared_memory
+
+
+@tractor.context
+async def just_sleep(
+
+    ctx: tractor.Context,
+    **kwargs,
+
+) -> None:
+    '''
+    Test a small ping-pong 2-way streaming server.
+
+    '''
+    await ctx.started()
+    await trio.sleep_forever()
+
+
+async def main() -> None:
+
+    proc = await trio.open_process( (
+        'python',
+        '-c',
+        'import trio; trio.run(trio.sleep_forever)',
+    ))
+    await proc.wait()
+    # await trio.sleep_forever()
+    # async with tractor.open_nursery() as n:
+
+    #     portal = await n.start_actor(
+    #         'rpc_server',
+    #         enable_modules=[__name__],
+    #     )
+
+    #     async with portal.open_context(
+    #         just_sleep,  # taken from pytest parameterization
+    #     ) as (ctx, sent):
+    #         await trio.sleep_forever()
+
+
+
+if __name__ == '__main__':
+    import time
+    # time.sleep(999)
+    trio.run(main)

examples/multiple_streams_one_portal.py

@@ -0,0 +1,46 @@
+import trio
+import tractor
+
+
+log = tractor.log.get_logger('multiportal')
+
+
+async def stream_data(seed=10):
+    log.info("Starting stream task")
+
+    for i in range(seed):
+        yield i
+        await trio.sleep(0)  # trigger scheduler
+
+
+async def stream_from_portal(p, consumed):
+
+    async with p.open_stream_from(stream_data) as stream:
+        async for item in stream:
+            if item in consumed:
+                consumed.remove(item)
+            else:
+                consumed.append(item)
+
+
+async def main():
+
+    async with tractor.open_nursery(loglevel='info') as an:
+
+        p = await an.start_actor('stream_boi', enable_modules=[__name__])
+
+        consumed = []
+
+        async with trio.open_nursery() as n:
+            for i in range(2):
+                n.start_soon(stream_from_portal, p, consumed)
+
+        # both streaming consumer tasks have completed and so we should
+        # have nothing in our list thanks to single threadedness
+        assert not consumed
+
+        await an.cancel()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/parallelism/_concurrent_futures_primes.py

@@ -0,0 +1,43 @@
+import time
+import concurrent.futures
+import math
+
+PRIMES = [
+    112272535095293,
+    112582705942171,
+    112272535095293,
+    115280095190773,
+    115797848077099,
+    1099726899285419]
+
+
+def is_prime(n):
+    if n < 2:
+        return False
+    if n == 2:
+        return True
+    if n % 2 == 0:
+        return False
+
+    sqrt_n = int(math.floor(math.sqrt(n)))
+    for i in range(3, sqrt_n + 1, 2):
+        if n % i == 0:
+            return False
+    return True
+
+
+def main():
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        start = time.time()
+
+        for number, prime in zip(PRIMES, executor.map(is_prime, PRIMES)):
+            print('%d is prime: %s' % (number, prime))
+
+        print(f'processing took {time.time() - start} seconds')
+
+
+if __name__ == '__main__':
+
+    start = time.time()
+    main()
+    print(f'script took {time.time() - start} seconds')

examples/parallelism/concurrent_actors_primes.py

@@ -0,0 +1,119 @@
+"""
+Demonstration of the prime number detector example from the
+``concurrent.futures`` docs:
+
+https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor-example
+
+This uses no extra threads, fancy semaphores or futures; all we need
+is ``tractor``'s channels.
+
+"""
+from contextlib import asynccontextmanager
+from typing import Callable
+import itertools
+import math
+import time
+
+import tractor
+import trio
+from async_generator import aclosing
+
+
+PRIMES = [
+    112272535095293,
+    112582705942171,
+    112272535095293,
+    115280095190773,
+    115797848077099,
+    1099726899285419,
+]
+
+
+async def is_prime(n):
+    if n < 2:
+        return False
+    if n == 2:
+        return True
+    if n % 2 == 0:
+        return False
+
+    sqrt_n = int(math.floor(math.sqrt(n)))
+    for i in range(3, sqrt_n + 1, 2):
+        if n % i == 0:
+            return False
+    return True
+
+
+@asynccontextmanager
+async def worker_pool(workers=4):
+    """Though it's a trivial special case for ``tractor``, the well
+    known "worker pool" seems to be the defacto "but, I want this
+    process pattern!" for most parallelism pilgrims.
+
+    Yes, the workers stay alive (and ready for work) until you close
+    the context.
+    """
+    async with tractor.open_nursery() as tn:
+
+        portals = []
+        snd_chan, recv_chan = trio.open_memory_channel(len(PRIMES))
+
+        for i in range(workers):
+
+            # this starts a new sub-actor (process + trio runtime) and
+            # stores it's "portal" for later use to "submit jobs" (ugh).
+            portals.append(
+                await tn.start_actor(
+                    f'worker_{i}',
+                    enable_modules=[__name__],
+                )
+            )
+
+        async def _map(
+            worker_func: Callable[[int], bool],
+            sequence: list[int]
+        ) -> list[bool]:
+
+            # define an async (local) task to collect results from workers
+            async def send_result(func, value, portal):
+                await snd_chan.send((value, await portal.run(func, n=value)))
+
+            async with trio.open_nursery() as n:
+
+                for value, portal in zip(sequence, itertools.cycle(portals)):
+                    n.start_soon(
+                        send_result,
+                        worker_func,
+                        value,
+                        portal
+                    )
+
+                # deliver results as they arrive
+                for _ in range(len(sequence)):
+                    yield await recv_chan.receive()
+
+        # deliver the parallel "worker mapper" to user code
+        yield _map
+
+        # tear down all "workers" on pool close
+        await tn.cancel()
+
+
+async def main():
+
+    async with worker_pool() as actor_map:
+
+        start = time.time()
+
+        async with aclosing(actor_map(is_prime, PRIMES)) as results:
+            async for number, prime in results:
+
+                print(f'{number} is prime: {prime}')
+
+        print(f'processing took {time.time() - start} seconds')
+
+
+if __name__ == '__main__':
+    start = time.time()
+    trio.run(main)
+    print(f'script took {time.time() - start} seconds')

examples/parallelism/single_func.py

@@ -0,0 +1,43 @@
+"""
+Run with a process monitor from a terminal using::
+
+    $TERM -e watch -n 0.1  "pstree -a $$" \
+        & python examples/parallelism/single_func.py \
+        && kill $!
+
+"""
+import os
+
+import tractor
+import trio
+
+
+async def burn_cpu():
+
+    pid = os.getpid()
+
+    # burn a core @ ~ 50kHz
+    for _ in range(50000):
+        await trio.sleep(1/50000/50)
+
+    return os.getpid()
+
+
+async def main():
+
+    async with tractor.open_nursery() as n:
+
+        portal = await n.run_in_actor(burn_cpu)
+
+        #  burn rubber in the parent too
+        await burn_cpu()
+
+        # wait on result from target function
+        pid = await portal.result()
+
+    # end of nursery block
+    print(f"Collected subproc {pid}")
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/parallelism/we_are_processes.py

@@ -0,0 +1,43 @@
+"""
+Run with a process monitor from a terminal using::
+
+    $TERM -e watch -n 0.1  "pstree -a $$" \
+        & python examples/parallelism/we_are_processes.py \
+        && kill $!
+
+"""
+from multiprocessing import cpu_count
+import os
+
+import tractor
+import trio
+
+
+async def target():
+    print(
+        f"Yo, i'm '{tractor.current_actor().name}' "
+        f"running in pid {os.getpid()}"
+    )
+
+    await trio.sleep_forever()
+
+
+async def main():
+
+    async with tractor.open_nursery() as n:
+
+        for i in range(cpu_count()):
+            await n.run_in_actor(target, name=f'worker_{i}')
+
+        print('This process tree will self-destruct in 1 sec...')
+        await trio.sleep(1)
+
+        # you could have done this yourself
+        raise Exception('Self Destructed')
+
+
+if __name__ == '__main__':
+    try:
+        trio.run(main)
+    except Exception:
+        print('Zombies Contained')

examples/quick_cluster.py

@@ -0,0 +1,44 @@
+
+import trio
+import tractor
+
+
+async def sleepy_jane():
+    uid = tractor.current_actor().uid
+    print(f'Yo i am actor {uid}')
+    await trio.sleep_forever()
+
+
+async def main():
+    '''
+    Spawn a flat actor cluster, with one process per
+    detected core.
+
+    '''
+    portal_map: dict[str, tractor.Portal]
+    results: dict[str, str]
+
+    # look at this hip new syntax!
+    async with (
+
+        tractor.open_actor_cluster(
+            modules=[__name__]
+        ) as portal_map,
+
+        trio.open_nursery() as n,
+    ):
+
+        for (name, portal) in portal_map.items():
+            n.start_soon(portal.run, sleepy_jane)
+
+        await trio.sleep(0.5)
+
+        # kill the cluster with a cancel
+        raise KeyboardInterrupt
+
+
+if __name__ == '__main__':
+    try:
+        trio.run(main)
+    except KeyboardInterrupt:
+        pass

examples/remote_error_propagation.py

@@ -0,0 +1,30 @@
+import trio
+import tractor
+
+
+async def assert_err():
+    assert 0
+
+
+async def main():
+    async with tractor.open_nursery() as n:
+        real_actors = []
+        for i in range(3):
+            real_actors.append(await n.start_actor(
+                f'actor_{i}',
+                enable_modules=[__name__],
+            ))
+
+        # start one actor that will fail immediately
+        await n.run_in_actor(assert_err)
+
+    # should error here with a ``RemoteActorError`` containing
+    # an ``AssertionError`` and all the other actors have been cancelled
+
+
+if __name__ == '__main__':
+    try:
+        # also raises
+        trio.run(main)
+    except tractor.RemoteActorError:
+        print("Look Maa that actor failed hard, hehhh!")

examples/rpc_bidir_streaming.py

@@ -0,0 +1,72 @@
+import trio
+import tractor
+
+
+@tractor.context
+async def simple_rpc(
+
+    ctx: tractor.Context,
+    data: int,
+
+) -> None:
+    '''Test a small ping-pong 2-way streaming server.
+
+    '''
+    # signal to parent that we're up much like
+    # ``trio_typing.TaskStatus.started()``
+    await ctx.started(data + 1)
+
+    async with ctx.open_stream() as stream:
+
+        count = 0
+        async for msg in stream:
+
+            assert msg == 'ping'
+            await stream.send('pong')
+            count += 1
+
+        else:
+            assert count == 10
+
+
+async def main() -> None:
+
+    async with tractor.open_nursery() as n:
+
+        portal = await n.start_actor(
+            'rpc_server',
+            enable_modules=[__name__],
+        )
+
+        # XXX: syntax requires py3.9
+        async with (
+
+            portal.open_context(
+                simple_rpc,  # taken from pytest parameterization
+                data=10,
+
+            ) as (ctx, sent),
+
+            ctx.open_stream() as stream,
+        ):
+
+            assert sent == 11
+
+            count = 0
+            # receive msgs using async for style
+            await stream.send('ping')
+
+            async for msg in stream:
+                assert msg == 'pong'
+                await stream.send('ping')
+                count += 1
+
+                if count >= 9:
+                    break
+
+        # explicitly teardown the daemon-actor
+        await portal.cancel_actor()
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/service_discovery.py

@@ -0,0 +1,22 @@
+import trio
+import tractor
+
+tractor.log.get_console_log("INFO")
+
+
+async def main(service_name):
+
+    async with tractor.open_nursery() as an:
+        await an.start_actor(service_name)
+
+        async with tractor.get_arbiter('127.0.0.1', 1616) as portal:
+            print(f"Arbiter is listening on {portal.channel}")
+
+        async with tractor.wait_for_actor(service_name) as sockaddr:
+            print(f"my_service is found at {sockaddr}")
+
+        await an.cancel()
+
+
+if __name__ == '__main__':
+    trio.run(main, 'some_actor_name')

mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+plugins = trio_typing.plugin

nooz/.gitignore

@@ -0,0 +1 @@
+!.gitignore

nooz/322.trivial.rst

@@ -0,0 +1,16 @@
+Strictly support Python 3.10+, start runtime machinery reorg
+
+Since we want to push forward using the new `match:` syntax for our
+internal RPC-msg loops, we officially drop 3.9 support for the next
+release which should coincide well with the first release of 3.11.
+
+This patch set also officially removes the ``tractor.run()`` API (which
+has been deprecated for some time) as well as starts an initial re-org
+of the internal runtime core by:
+- renaming ``tractor._actor`` -> ``._runtime``
+- moving the ``._runtime.ActorActor._process_messages()`` and
+  ``._async_main()`` to be module level singleton-task-functions since
+  they are only started once for each connection and actor spawn
+  respectively; this internal API thus looks more similar to (at the
+  time of writing) the ``trio``-internals in ``trio._core._run``.
+- officially remove ``tractor.run()``, now deprecated for some time.

nooz/324.bugfix.rst

@@ -0,0 +1,4 @@
+Only set `._debug.Lock.local_pdb_complete` if has been created.
+
+This can be triggered by a very rare race condition (and thus we have no
+working test yet) but it is known to exist in (a) consumer project(s).

nooz/333.feature.rst

@@ -0,0 +1,25 @@
+Add support for ``trio >= 0.22`` and support for the new Python 3.11
+``[Base]ExceptionGroup`` from `pep 654`_ via the backported
+`exceptiongroup`_ package and some final fixes to the debug mode
+subsystem.
+
+This port ended up driving some (hopefully) final fixes to our debugger
+subsystem including the solution to all lingering stdstreams locking
+race-conditions and deadlock scenarios. This includes extending the
+debugger tests suite as well as cancellation and ``asyncio`` mode cases.
+Some of the notable details:
+
+- always reverting to the ``trio`` SIGINT handler when leaving debug
+  mode.
+- bypassing child attempts to acquire the debug lock when detected
+  to be amdist actor-runtime-cancellation.
+- allowing the root actor to cancel local but IPC-stale subactor
+  requests-tasks for the debug lock when in a "no IPC peers" state.
+
+Further we refined our ``ActorNursery`` semantics to be more similar to
+``trio`` in the sense that parent task errors are always packed into the
+actor-nursery emitted exception group and adjusted all tests and
+examples accordingly.
+
+.. _pep 654: https://peps.python.org/pep-0654/#handling-exception-groups
+.. _exceptiongroup: https://github.com/python-trio/exceptiongroup

nooz/335.trivial.rst

@@ -0,0 +1,5 @@
+Establish an explicit "backend spawning" method table; use it from CI
+
+More clearly lays out the current set of (3) backends: ``['trio',
+'mp_spawn', 'mp_forkserver']`` and adjusts the ``._spawn.py`` internals
+as well as the test suite to accommodate.

nooz/336.trivial.rst

@@ -0,0 +1,4 @@
+Add ``key: Callable[..., Hashable]`` support to ``.trionics.maybe_open_context()``
+
+Gives users finer grained control over cache hit behaviour using
+a callable which receives the input ``kwargs: dict``.

nooz/337.feature.rst

@@ -0,0 +1,41 @@
+Add support for debug-lock blocking using a ``._debug.Lock._blocked:
+set[tuple]`` and add ids when no-more IPC connections with the
+root actor are detected.
+
+This is an enhancement which (mostly) solves a lingering debugger
+locking race case we needed to handle:
+
+- child crashes acquires TTY lock in root and attaches to ``pdb``
+- child IPC goes down such that all channels to the root are broken
+  / non-functional.
+- root is stuck thinking the child is still in debug even though it
+  can't be contacted and the child actor machinery hasn't been
+  cancelled by its parent.
+- root get's stuck in deadlock with child since it won't send a cancel
+  request until the child is finished debugging (to avoid clobbering
+  a child that is actually using the debugger), but the child can't
+  unlock the debugger bc IPC is down and it can't contact the root.
+
+To avoid this scenario add debug lock blocking list via
+`._debug.Lock._blocked: set[tuple]` which holds actor uids for any actor
+that is detected by the root as having no transport channel connections
+(of which at least one should exist if this sub-actor at some point
+acquired the debug lock). The root consequently checks this list for any
+actor that tries to (re)acquire the lock and blocks with
+a ``ContextCancelled``. Further, when a debug condition is tested in
+``._runtime._invoke``, the context's ``._enter_debugger_on_cancel`` is
+set to `False` if the actor was put on the block list then all
+post-mortem / crash handling will be bypassed for that task.
+
+In theory this approach to block list management may cause problems
+where some nested child actor acquires and releases the lock multiple
+times and it gets stuck on the block list after the first use? If this
+turns out to be an issue we can try changing the strat so blocks are
+only added when the root has zero IPC peers left?
+
+Further, this adds a root-locking-task side cancel scope,
+``Lock._root_local_task_cs_in_debug``, which can be ``.cancel()``-ed by the root
+runtime when a stale lock is detected during the IPC channel testing.
+However, right now we're NOT using this since it seems to cause test
+failures likely due to causing pre-mature cancellation and maybe needs
+a bit more experimenting?

nooz/343.trivial.rst

@@ -0,0 +1,19 @@
+Rework our ``.trionics.BroadcastReceiver`` internals to avoid method
+recursion and approach a design and interface closer to ``trio``'s
+``MemoryReceiveChannel``.
+
+The details of the internal changes include:
+
+- implementing a ``BroadcastReceiver.receive_nowait()`` and using it
+  within the async ``.receive()`` thus avoiding recursion from
+  ``.receive()``.
+- failing over to an internal ``._receive_from_underlying()`` when the
+  ``_nowait()`` call raises ``trio.WouldBlock``
+- adding ``BroadcastState.statistics()`` for debugging and testing both
+  internals and by users.
+- add an internal ``BroadcastReceiver._raise_on_lag: bool`` which can be
+  set to avoid ``Lagged`` raising for possible use cases where a user
+  wants to choose between a [cheap or nasty
+  pattern](https://zguide.zeromq.org/docs/chapter7/#The-Cheap-or-Nasty-Pattern)
+  the the particular stream (we use this in ``piker``'s dark clearing
+  engine to avoid fast feeds breaking during HFT periods).

nooz/344.bugfix.rst

@@ -0,0 +1,11 @@
+Always ``list``-cast the ``mngrs`` input to
+``.trionics.gather_contexts()`` and ensure its size otherwise raise
+a ``ValueError``.
+
+Turns out that trying to pass an inline-style generator comprehension
+doesn't seem to work inside the ``async with`` expression? Further, in
+such a case we can get a hang waiting on the all-entered event
+completion when the internal mngrs iteration is a noop. Instead we
+always greedily check a size and error on empty input; the lazy
+iteration of a generator input is not beneficial anyway since we're
+entering all manager instances in concurrent tasks.

nooz/346.bugfix.rst

@@ -0,0 +1,15 @@
+Fixes to ensure IPC (channel) breakage doesn't result in hung actor
+trees; the zombie reaping and general supervision machinery will always
+clean up and terminate.
+
+This includes not only the (mostly minor) fixes to solve these cases but
+also a new extensive test suite in `test_advanced_faults.py` with an
+accompanying highly configurable example module-script in
+`examples/advanced_faults/ipc_failure_during_stream.py`. Tests ensure we
+never get hang or zombies despite operating in debug mode and attempt to
+simulate all possible IPC transport failure cases for a local-host actor
+tree.
+
+Further we simplify `Context.open_stream.__aexit__()` to just call
+`MsgStream.aclose()` directly more or less avoiding a pure duplicate
+code path.

nooz/349.trivial.rst

@@ -0,0 +1,10 @@
+Always redraw the `pdbpp` prompt on `SIGINT` during REPL use.
+
+There was recent changes todo with Python 3.10 that required us to pin
+to a specific commit in `pdbpp` which have recently been fixed minus
+this last issue with `SIGINT` shielding: not clobbering or not
+showing the `(Pdb++)` prompt on ctlr-c by the user. This repairs all
+that by firstly removing the standard KBI intercepting of the std lib's
+`pdb.Pdb._cmdloop()` as well as ensuring that only the actor with REPL
+control ever reports `SIGINT` handler log msgs and prompt redraws. With
+this we move back to using pypi `pdbpp` release.

nooz/356.trivial.rst

@@ -0,0 +1,7 @@
+Drop `trio.Process.aclose()` usage, copy into our spawning code.
+
+The details are laid out in https://github.com/goodboy/tractor/issues/330.
+`trio` changed is process running quite some time ago, this just copies
+out the small bit we needed (from the old `.aclose()`) for hard kills
+where a soft runtime cancel request fails and our "zombie killer"
+implementation kicks in.

nooz/358.feature.rst

@@ -0,0 +1,15 @@
+Switch to using the fork & fix of `pdb++`, `pdbp`:
+https://github.com/mdmintz/pdbp
+
+Allows us to sidestep a variety of issues that aren't being maintained
+in the upstream project thanks to the hard work of @mdmintz!
+
+We also include some default settings adjustments as per recent
+development on the fork:
+
+- sticky mode is still turned on by default but now activates when
+  a using the `ll` repl command.
+- turn off line truncation by default to avoid inter-line gaps when
+  resizing the terimnal during use.
+- when using the backtrace cmd either by `w` or `bt`, the config
+  automatically switches to non-sticky mode.

nooz/HOWTO.rst

@@ -0,0 +1,8 @@
+See both the `towncrier docs`_ and the `pluggy release readme`_ for hot
+tips. We basically have the most minimal setup and release process right
+now and use the default `fragment set`_.
+
+
+.. _towncrier docs: https://github.com/twisted/towncrier#quick-start
+.. _pluggy release readme: https://github.com/pytest-dev/pluggy/blob/main/changelog/README.rst
+.. _fragment set: https://github.com/twisted/towncrier#news-fragments

nooz/_template.rst

@@ -0,0 +1,37 @@
+{% for section in sections %}
+{% set underline = "-" %}
+{% if section %}
+{{section}}
+{{ underline * section|length }}{% set underline = "~" %}
+
+{% endif %}
+{% if sections[section] %}
+{% for category, val in definitions.items() if category in sections[section] %}
+
+{{ definitions[category]['name'] }}
+{{ underline * definitions[category]['name']|length }}
+
+{% if definitions[category]['showcontent'] %}
+{% for text, values in sections[section][category]|dictsort(by='value') %}
+{% set issue_joiner = joiner(', ') %}
+- {% for value in values|sort %}{{ issue_joiner() }}`{{ value }} <https://github.com/goodboy/tractor/issues/{{ value[1:] }}>`_{% endfor %}: {{ text }}
+
+{% endfor %}
+{% else %}
+- {{ sections[section][category]['']|sort|join(', ') }}
+
+
+{% endif %}
+{% if sections[section][category]|length == 0 %}
+
+No significant changes.
+
+{% else %}
+{% endif %}
+{% endfor %}
+{% else %}
+
+No significant changes.
+
+{% endif %}
+{% endfor %}

pyproject.toml

@@ -0,0 +1,28 @@
+[tool.towncrier]
+package = "tractor"
+filename = "NEWS.rst"
+directory = "nooz/"
+version = "0.1.0a6"
+title_format = "tractor {version} ({project_date})"
+template = "nooz/_template.rst"
+all_bullets = true
+
+  [[tool.towncrier.type]]
+  directory = "feature"
+  name = "Features"
+  showcontent = true
+
+  [[tool.towncrier.type]]
+  directory = "bugfix"
+  name = "Bug Fixes"
+  showcontent = true
+
+  [[tool.towncrier.type]]
+  directory = "doc"
+  name = "Improved Documentation"
+  showcontent = true
+
+  [[tool.towncrier.type]]
+  directory = "trivial"
+  name = "Trivial/Internal Changes"
+  showcontent = true

requirements-docs.txt

@@ -0,0 +1,2 @@
+sphinx
+sphinx_book_theme

requirements-test.txt

@@ -1,4 +1,8 @@
 pytest
 pytest-trio
-pdbpp
+pytest-timeout
+pdbp
 mypy
+trio_typing
+pexpect
+towncrier

setup.py

@@ -1,57 +1,97 @@
 #!/usr/bin/env python
 #
-# tractor: a trionic actor model built on `multiprocessing` and `trio`
+# tractor: structured concurrent "actors".
 #
-# Copyright (C) 2018  Tyler Goodlet
+# Copyright 2018-eternity Tyler Goodlet.
 
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from setuptools import setup
 
-with open('README.rst', encoding='utf-8') as f:
+with open('docs/README.rst', encoding='utf-8') as f:
     readme = f.read()
 
 
 setup(
     name="tractor",
-    version='0.1.0.alpha0',
-    description='A trionic actor model built on `multiprocessing` and `trio`',
+    version='0.1.0a6dev0',  # alpha zone
+    description='structured concurrrent `trio`-"actors"',
     long_description=readme,
-    license='GPLv3',
+    license='AGPLv3',
     author='Tyler Goodlet',
     maintainer='Tyler Goodlet',
-    maintainer_email='tgoodlet@gmail.com',
-    url='https://github.com/tgoodlet/tractor',
-    platforms=['linux'],
+    maintainer_email='goodboy_foss@protonmail.com',
+    url='https://github.com/goodboy/tractor',
+    platforms=['linux', 'windows'],
     packages=[
         'tractor',
+        'tractor.experimental',
+        'tractor.trionics',
+    ],
+    install_requires=[
+
+        # trio related
+        # proper range spec:
+        # https://packaging.python.org/en/latest/discussions/install-requires-vs-requirements/#id5
+        'trio >= 0.22',
+        'async_generator',
+        'trio_typing',
+        'exceptiongroup',
+
+        # tooling
+        'tricycle',
+        'trio_typing',
+        'colorlog',
+        'wrapt',
+
+        # IPC serialization
+        'msgspec',
+
+        # debug mode REPL
+        'pdbp',
+
+        # pip ref docs on these specs:
+        # https://pip.pypa.io/en/stable/reference/requirement-specifiers/#examples
+        # and pep:
+        # https://peps.python.org/pep-0440/#version-specifiers
+
+        # windows deps workaround for ``pdbpp``
+        # https://github.com/pdbpp/pdbpp/issues/498
+        # https://github.com/pdbpp/fancycompleter/issues/37
+        'pyreadline3 ; platform_system == "Windows"',
+
     ],
-    install_requires=['msgpack', 'trio', 'async_generator', 'colorlog'],
     tests_require=['pytest'],
-    python_requires=">=3.6",
+    python_requires=">=3.10",
     keywords=[
-        "async", "concurrency", "actor model", "distributed",
-        'trio', 'multiprocessing'
+        'trio',
+        'async',
+        'concurrency',
+        'structured concurrency',
+        'actor model',
+        'distributed',
+        'multiprocessing'
     ],
     classifiers=[
-        'Development Status :: 3 - Alpha',
-        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)'
-        'Operating System :: POSIX :: Linux',
+        "Development Status :: 3 - Alpha",
+        "Operating System :: POSIX :: Linux",
+        "Operating System :: Microsoft :: Windows",
         "Framework :: Trio",
+        "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
         "Programming Language :: Python :: Implementation :: CPython",
-        "Programming Language :: Python :: Implementation :: PyPy",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.10",
         "Intended Audience :: Science/Research",
         "Intended Audience :: Developers",
         "Topic :: System :: Distributed Computing",

tests/conftest.py

@@ -1,32 +1,22 @@
 """
 ``tractor`` testing!!
 """
+import sys
+import subprocess
+import os
 import random
+import signal
+import platform
+import pathlib
+import time
+import inspect
 from functools import partial, wraps
 
 import pytest
+import trio
 import tractor
 
-
-_arb_addr = '127.0.0.1', random.randint(1000, 9999)
-
-
-def pytest_addoption(parser):
-    parser.addoption("--ll", action="store", dest='loglevel',
-                     default=None, help="logging level to set when testing")
-
-
-@pytest.fixture(scope='session', autouse=True)
-def loglevel(request):
-    orig = tractor.log._default_loglevel
-    level = tractor.log._default_loglevel = request.config.option.loglevel
-    yield level
-    tractor.log._default_loglevel = orig
-
-
-@pytest.fixture(scope='session')
-def arb_addr():
-    return _arb_addr
+pytest_plugins = ['pytester']
 
 
 def tractor_test(fn):
@@ -36,11 +26,225 @@ def tractor_test(fn):
     @tractor_test
     async def test_whatever():
         await ...
+
+    If fixtures:
+
+        - ``arb_addr`` (a socket addr tuple where arbiter is listening)
+        - ``loglevel`` (logging level passed to tractor internals)
+        - ``start_method`` (subprocess spawning backend)
+
+    are defined in the `pytest` fixture space they will be automatically
+    injected to tests declaring these funcargs.
     """
     @wraps(fn)
-    def wrapper(*args, **kwargs):
+    def wrapper(
+        *args,
+        loglevel=None,
+        arb_addr=None,
+        start_method=None,
+        **kwargs
+    ):
         # __tracebackhide__ = True
-        return tractor.run(
-            partial(fn, *args, **kwargs), arbiter_addr=_arb_addr)
+
+        if 'arb_addr' in inspect.signature(fn).parameters:
+            # injects test suite fixture value to test as well
+            # as `run()`
+            kwargs['arb_addr'] = arb_addr
+
+        if 'loglevel' in inspect.signature(fn).parameters:
+            # allows test suites to define a 'loglevel' fixture
+            # that activates the internal logging
+            kwargs['loglevel'] = loglevel
+
+        if start_method is None:
+            if platform.system() == "Windows":
+                start_method = 'trio'
+
+        if 'start_method' in inspect.signature(fn).parameters:
+            # set of subprocess spawning backends
+            kwargs['start_method'] = start_method
+
+        if kwargs:
+
+            # use explicit root actor start
+
+            async def _main():
+                async with tractor.open_root_actor(
+                    # **kwargs,
+                    arbiter_addr=arb_addr,
+                    loglevel=loglevel,
+                    start_method=start_method,
+
+                    # TODO: only enable when pytest is passed --pdb
+                    # debug_mode=True,
+
+                ):
+                    await fn(*args, **kwargs)
+
+            main = _main
+
+        else:
+            # use implicit root actor start
+            main = partial(fn, *args, **kwargs)
+
+        return trio.run(main)
 
     return wrapper
+
+
+_arb_addr = '127.0.0.1', random.randint(1000, 9999)
+
+
+# Sending signal.SIGINT on subprocess fails on windows. Use CTRL_* alternatives
+if platform.system() == 'Windows':
+    _KILL_SIGNAL = signal.CTRL_BREAK_EVENT
+    _INT_SIGNAL = signal.CTRL_C_EVENT
+    _INT_RETURN_CODE = 3221225786
+    _PROC_SPAWN_WAIT = 2
+else:
+    _KILL_SIGNAL = signal.SIGKILL
+    _INT_SIGNAL = signal.SIGINT
+    _INT_RETURN_CODE = 1 if sys.version_info < (3, 8) else -signal.SIGINT.value
+    _PROC_SPAWN_WAIT = 0.6 if sys.version_info < (3, 7) else 0.4
+
+
+no_windows = pytest.mark.skipif(
+    platform.system() == "Windows",
+    reason="Test is unsupported on windows",
+)
+
+
+def repodir() -> pathlib.Path:
+    '''
+    Return the abspath to the repo directory.
+
+    '''
+    # 2 parents up to step up through tests/<repo_dir>
+    return pathlib.Path(__file__).parent.parent.absolute()
+
+
+def examples_dir() -> pathlib.Path:
+    '''
+    Return the abspath to the examples directory as `pathlib.Path`.
+
+    '''
+    return repodir() / 'examples'
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--ll", action="store", dest='loglevel',
+        default='ERROR', help="logging level to set when testing"
+    )
+
+    parser.addoption(
+        "--spawn-backend", action="store", dest='spawn_backend',
+        default='trio',
+        help="Processing spawning backend to use for test run",
+    )
+
+
+def pytest_configure(config):
+    backend = config.option.spawn_backend
+    tractor._spawn.try_set_start_method(backend)
+
+
+@pytest.fixture(scope='session', autouse=True)
+def loglevel(request):
+    orig = tractor.log._default_loglevel
+    level = tractor.log._default_loglevel = request.config.option.loglevel
+    tractor.log.get_console_log(level)
+    yield level
+    tractor.log._default_loglevel = orig
+
+
+@pytest.fixture(scope='session')
+def spawn_backend(request) -> str:
+    return request.config.option.spawn_backend
+
+
+_ci_env: bool = os.environ.get('CI', False)
+
+
+@pytest.fixture(scope='session')
+def ci_env() -> bool:
+    """Detect CI envoirment.
+    """
+    return _ci_env
+
+
+@pytest.fixture(scope='session')
+def arb_addr():
+    return _arb_addr
+
+
+def pytest_generate_tests(metafunc):
+    spawn_backend = metafunc.config.option.spawn_backend
+
+    if not spawn_backend:
+        # XXX some weird windows bug with `pytest`?
+        spawn_backend = 'trio'
+
+    # TODO: maybe just use the literal `._spawn.SpawnMethodKey`?
+    assert spawn_backend in (
+        'mp_spawn',
+        'mp_forkserver',
+        'trio',
+    )
+
+    # NOTE: used to be used to dyanmically parametrize tests for when
+    # you just passed --spawn-backend=`mp` on the cli, but now we expect
+    # that cli input to be manually specified, BUT, maybe we'll do
+    # something like this again in the future?
+    if 'start_method' in metafunc.fixturenames:
+        metafunc.parametrize("start_method", [spawn_backend], scope='module')
+
+
+def sig_prog(proc, sig):
+    "Kill the actor-process with ``sig``."
+    proc.send_signal(sig)
+    time.sleep(0.1)
+    if not proc.poll():
+        # TODO: why sometimes does SIGINT not work on teardown?
+        # seems to happen only when trace logging enabled?
+        proc.send_signal(_KILL_SIGNAL)
+    ret = proc.wait()
+    assert ret
+
+
+@pytest.fixture
+def daemon(
+    loglevel: str,
+    testdir,
+    arb_addr: tuple[str, int],
+):
+    '''
+    Run a daemon actor as a "remote arbiter".
+
+    '''
+    if loglevel in ('trace', 'debug'):
+        # too much logging will lock up the subproc (smh)
+        loglevel = 'info'
+
+    cmdargs = [
+        sys.executable, '-c',
+        "import tractor; tractor.run_daemon([], registry_addr={}, loglevel={})"
+        .format(
+            arb_addr,
+            "'{}'".format(loglevel) if loglevel else None)
+    ]
+    kwargs = dict()
+    if platform.system() == 'Windows':
+        # without this, tests hang on windows forever
+        kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
+
+    proc = testdir.popen(
+        cmdargs,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        **kwargs,
+    )
+    assert not proc.returncode
+    time.sleep(_PROC_SPAWN_WAIT)
+    yield proc
+    sig_prog(proc, _INT_SIGNAL)

tests/test_2way.py

@@ -0,0 +1,129 @@
+"""
+Bidirectional streaming.
+
+"""
+import pytest
+import trio
+import tractor
+
+
+@tractor.context
+async def simple_rpc(
+
+    ctx: tractor.Context,
+    data: int,
+
+) -> None:
+    '''
+    Test a small ping-pong server.
+
+    '''
+    # signal to parent that we're up
+    await ctx.started(data + 1)
+
+    print('opening stream in callee')
+    async with ctx.open_stream() as stream:
+
+        count = 0
+        while True:
+            try:
+                await stream.receive() == 'ping'
+            except trio.EndOfChannel:
+                assert count == 10
+                break
+            else:
+                print('pong')
+                await stream.send('pong')
+                count += 1
+
+
+@tractor.context
+async def simple_rpc_with_forloop(
+
+    ctx: tractor.Context,
+    data: int,
+
+) -> None:
+    """Same as previous test but using ``async for`` syntax/api.
+
+    """
+
+    # signal to parent that we're up
+    await ctx.started(data + 1)
+
+    print('opening stream in callee')
+    async with ctx.open_stream() as stream:
+
+        count = 0
+        async for msg in stream:
+
+            assert msg == 'ping'
+            print('pong')
+            await stream.send('pong')
+            count += 1
+
+        else:
+            assert count == 10
+
+
+@pytest.mark.parametrize(
+    'use_async_for',
+    [True, False],
+)
+@pytest.mark.parametrize(
+    'server_func',
+    [simple_rpc, simple_rpc_with_forloop],
+)
+def test_simple_rpc(server_func, use_async_for):
+    '''
+    The simplest request response pattern.
+
+    '''
+    async def main():
+        async with tractor.open_nursery() as n:
+
+            portal = await n.start_actor(
+                'rpc_server',
+                enable_modules=[__name__],
+            )
+
+            async with portal.open_context(
+                server_func,  # taken from pytest parameterization
+                data=10,
+            ) as (ctx, sent):
+
+                assert sent == 11
+
+                async with ctx.open_stream() as stream:
+
+                    if use_async_for:
+
+                        count = 0
+                        # receive msgs using async for style
+                        print('ping')
+                        await stream.send('ping')
+
+                        async for msg in stream:
+                            assert msg == 'pong'
+                            print('ping')
+                            await stream.send('ping')
+                            count += 1
+
+                            if count >= 9:
+                                break
+
+                    else:
+                        # classic send/receive style
+                        for _ in range(10):
+
+                            print('ping')
+                            await stream.send('ping')
+                            assert await stream.receive() == 'pong'
+
+                # stream should terminate here
+
+            # final context result(s) should be consumed here in __aexit__()
+
+            await portal.cancel_actor()
+
+    trio.run(main)

tests/test_advanced_faults.py

@@ -0,0 +1,193 @@
+'''
+Sketchy network blackoutz, ugly byzantine gens, puedes eschuchar la
+cancelacion?..
+
+'''
+from functools import partial
+
+import pytest
+from _pytest.pathlib import import_path
+import trio
+import tractor
+
+from conftest import (
+    examples_dir,
+)
+
+
+@pytest.mark.parametrize(
+    'debug_mode',
+    [False, True],
+    ids=['no_debug_mode', 'debug_mode'],
+)
+@pytest.mark.parametrize(
+    'ipc_break',
+    [
+        # no breaks
+        {
+            'break_parent_ipc_after': False,
+            'break_child_ipc_after': False,
+        },
+
+        # only parent breaks
+        {
+            'break_parent_ipc_after': 500,
+            'break_child_ipc_after': False,
+        },
+
+        # only child breaks
+        {
+            'break_parent_ipc_after': False,
+            'break_child_ipc_after': 500,
+        },
+
+        # both: break parent first
+        {
+            'break_parent_ipc_after': 500,
+            'break_child_ipc_after': 800,
+        },
+        # both: break child first
+        {
+            'break_parent_ipc_after': 800,
+            'break_child_ipc_after': 500,
+        },
+
+    ],
+    ids=[
+        'no_break',
+        'break_parent',
+        'break_child',
+        'break_both_parent_first',
+        'break_both_child_first',
+    ],
+)
+def test_ipc_channel_break_during_stream(
+    debug_mode: bool,
+    spawn_backend: str,
+    ipc_break: dict | None,
+):
+    '''
+    Ensure we can have an IPC channel break its connection during
+    streaming and it's still possible for the (simulated) user to kill
+    the actor tree using SIGINT.
+
+    We also verify the type of connection error expected in the parent
+    depending on which side if the IPC breaks first.
+
+    '''
+    if spawn_backend != 'trio':
+        if debug_mode:
+            pytest.skip('`debug_mode` only supported on `trio` spawner')
+
+        # non-`trio` spawners should never hit the hang condition that
+        # requires the user to do ctl-c to cancel the actor tree.
+        expect_final_exc = trio.ClosedResourceError
+
+    mod = import_path(
+        examples_dir() / 'advanced_faults' / 'ipc_failure_during_stream.py',
+        root=examples_dir(),
+    )
+
+    expect_final_exc = KeyboardInterrupt
+
+    # when ONLY the child breaks we expect the parent to get a closed
+    # resource error on the next `MsgStream.receive()` and then fail out
+    # and cancel the child from there.
+    if (
+
+        # only child breaks
+        (
+            ipc_break['break_child_ipc_after']
+            and ipc_break['break_parent_ipc_after'] is False
+        )
+
+        # both break but, parent breaks first
+        or (
+            ipc_break['break_child_ipc_after'] is not False
+            and (
+                ipc_break['break_parent_ipc_after']
+                > ipc_break['break_child_ipc_after']
+            )
+        )
+
+    ):
+        expect_final_exc = trio.ClosedResourceError
+
+    # when the parent IPC side dies (even if the child's does as well
+    # but the child fails BEFORE the parent) we expect the channel to be
+    # sent a stop msg from the child at some point which will signal the
+    # parent that the stream has been terminated.
+    # NOTE: when the parent breaks "after" the child you get this same
+    # case as well, the child breaks the IPC channel with a stop msg
+    # before any closure takes place.
+    elif (
+        # only parent breaks
+        (
+            ipc_break['break_parent_ipc_after']
+            and ipc_break['break_child_ipc_after'] is False
+        )
+
+        # both break but, child breaks first
+        or (
+            ipc_break['break_parent_ipc_after'] is not False
+            and (
+                ipc_break['break_child_ipc_after']
+                > ipc_break['break_parent_ipc_after']
+            )
+        )
+    ):
+        expect_final_exc = trio.EndOfChannel
+
+    with pytest.raises(expect_final_exc):
+        trio.run(
+            partial(
+                mod.main,
+                debug_mode=debug_mode,
+                start_method=spawn_backend,
+                **ipc_break,
+            )
+        )
+
+
+@tractor.context
+async def break_ipc_after_started(
+    ctx: tractor.Context,
+) -> None:
+    await ctx.started()
+    async with ctx.open_stream() as stream:
+        await stream.aclose()
+        await trio.sleep(0.2)
+        await ctx.chan.send(None)
+        print('child broke IPC and terminating')
+
+
+def test_stream_closed_right_after_ipc_break_and_zombie_lord_engages():
+    '''
+    Verify that is a subactor's IPC goes down just after bringing up a stream
+    the parent can trigger a SIGINT and the child will be reaped out-of-IPC by
+    the localhost process supervision machinery: aka "zombie lord".
+
+    '''
+    async def main():
+        async with tractor.open_nursery() as n:
+            portal = await n.start_actor(
+                'ipc_breaker',
+                enable_modules=[__name__],
+            )
+
+            with trio.move_on_after(1):
+                async with (
+                    portal.open_context(
+                        break_ipc_after_started
+                    ) as (ctx, sent),
+                ):
+                    async with ctx.open_stream():
+                        await trio.sleep(0.5)
+
+                    print('parent waiting on context')
+
+            print('parent exited context')
+            raise KeyboardInterrupt
+
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(main)

tests/test_advanced_streaming.py

@@ -0,0 +1,380 @@
+'''
+Advanced streaming patterns using bidirectional streams and contexts.
+
+'''
+from collections import Counter
+import itertools
+import platform
+
+import trio
+import tractor
+
+
+def is_win():
+    return platform.system() == 'Windows'
+
+
+_registry: dict[str, set[tractor.MsgStream]] = {
+    'even': set(),
+    'odd': set(),
+}
+
+
+async def publisher(
+
+    seed: int = 0,
+
+) -> None:
+
+    global _registry
+
+    def is_even(i):
+        return i % 2 == 0
+
+    for val in itertools.count(seed):
+
+        sub = 'even' if is_even(val) else 'odd'
+
+        for sub_stream in _registry[sub].copy():
+            await sub_stream.send(val)
+
+        # throttle send rate to ~1kHz
+        # making it readable to a human user
+        await trio.sleep(1/1000)
+
+
+@tractor.context
+async def subscribe(
+
+    ctx: tractor.Context,
+
+) -> None:
+
+    global _registry
+
+    # syn caller
+    await ctx.started(None)
+
+    async with ctx.open_stream() as stream:
+
+        # update subs list as consumer requests
+        async for new_subs in stream:
+
+            new_subs = set(new_subs)
+            remove = new_subs - _registry.keys()
+
+            print(f'setting sub to {new_subs} for {ctx.chan.uid}')
+
+            # remove old subs
+            for sub in remove:
+                _registry[sub].remove(stream)
+
+            # add new subs for consumer
+            for sub in new_subs:
+                _registry[sub].add(stream)
+
+
+async def consumer(
+
+    subs: list[str],
+
+) -> None:
+
+    uid = tractor.current_actor().uid
+
+    async with tractor.wait_for_actor('publisher') as portal:
+        async with portal.open_context(subscribe) as (ctx, first):
+            async with ctx.open_stream() as stream:
+
+                # flip between the provided subs dynamically
+                if len(subs) > 1:
+
+                    for sub in itertools.cycle(subs):
+                        print(f'setting dynamic sub to {sub}')
+                        await stream.send([sub])
+
+                        count = 0
+                        async for value in stream:
+                            print(f'{uid} got: {value}')
+                            if count > 5:
+                                break
+                            count += 1
+
+                else:  # static sub
+
+                    await stream.send(subs)
+                    async for value in stream:
+                        print(f'{uid} got: {value}')
+
+
+def test_dynamic_pub_sub():
+
+    global _registry
+
+    from multiprocessing import cpu_count
+    cpus = cpu_count()
+
+    async def main():
+        async with tractor.open_nursery() as n:
+
+            # name of this actor will be same as target func
+            await n.run_in_actor(publisher)
+
+            for i, sub in zip(
+                range(cpus - 2),
+                itertools.cycle(_registry.keys())
+            ):
+                await n.run_in_actor(
+                    consumer,
+                    name=f'consumer_{sub}',
+                    subs=[sub],
+                )
+
+            # make one dynamic subscriber
+            await n.run_in_actor(
+                consumer,
+                name='consumer_dynamic',
+                subs=list(_registry.keys()),
+            )
+
+            # block until cancelled by user
+            with trio.fail_after(3):
+                await trio.sleep_forever()
+
+    try:
+        trio.run(main)
+    except trio.TooSlowError:
+        pass
+
+
+@tractor.context
+async def one_task_streams_and_one_handles_reqresp(
+
+    ctx: tractor.Context,
+
+) -> None:
+
+    await ctx.started()
+
+    async with ctx.open_stream() as stream:
+
+        async def pingpong():
+            '''Run a simple req/response service.
+
+            '''
+            async for msg in stream:
+                print('rpc server ping')
+                assert msg == 'ping'
+                print('rpc server pong')
+                await stream.send('pong')
+
+        async with trio.open_nursery() as n:
+            n.start_soon(pingpong)
+
+            for _ in itertools.count():
+                await stream.send('yo')
+                await trio.sleep(0.01)
+
+
+def test_reqresp_ontopof_streaming():
+    '''
+    Test a subactor that both streams with one task and
+    spawns another which handles a small requests-response
+    dialogue over the same bidir-stream.
+
+    '''
+    async def main():
+
+        # flat to make sure we get at least one pong
+        got_pong: bool = False
+        timeout: int = 2
+
+        if is_win():  # smh
+            timeout = 4
+
+        with trio.move_on_after(timeout):
+            async with tractor.open_nursery() as n:
+
+                # name of this actor will be same as target func
+                portal = await n.start_actor(
+                    'dual_tasks',
+                    enable_modules=[__name__]
+                )
+
+                async with portal.open_context(
+                    one_task_streams_and_one_handles_reqresp,
+
+                ) as (ctx, first):
+
+                    assert first is None
+
+                    async with ctx.open_stream() as stream:
+
+                        await stream.send('ping')
+
+                        async for msg in stream:
+                            print(f'client received: {msg}')
+
+                            assert msg in {'pong', 'yo'}
+
+                            if msg == 'pong':
+                                got_pong = True
+                                await stream.send('ping')
+                                print('client sent ping')
+
+        assert got_pong
+
+    try:
+        trio.run(main)
+    except trio.TooSlowError:
+        pass
+
+
+async def async_gen_stream(sequence):
+    for i in sequence:
+        yield i
+        await trio.sleep(0.1)
+
+
+@tractor.context
+async def echo_ctx_stream(
+    ctx: tractor.Context,
+) -> None:
+    await ctx.started()
+
+    async with ctx.open_stream() as stream:
+        async for msg in stream:
+            await stream.send(msg)
+
+
+def test_sigint_both_stream_types():
+    '''Verify that running a bi-directional and recv only stream
+    side-by-side will cancel correctly from SIGINT.
+
+    '''
+    timeout: float = 2
+    if is_win():  # smh
+        timeout += 1
+
+    async def main():
+        with trio.fail_after(timeout):
+            async with tractor.open_nursery() as n:
+                # name of this actor will be same as target func
+                portal = await n.start_actor(
+                    '2_way',
+                    enable_modules=[__name__]
+                )
+
+                async with portal.open_context(echo_ctx_stream) as (ctx, _):
+                    async with ctx.open_stream() as stream:
+                        async with portal.open_stream_from(
+                            async_gen_stream,
+                            sequence=list(range(1)),
+                        ) as gen_stream:
+
+                            msg = await gen_stream.receive()
+                            await stream.send(msg)
+                            resp = await stream.receive()
+                            assert resp == msg
+                            raise KeyboardInterrupt
+
+    try:
+        trio.run(main)
+        assert 0, "Didn't receive KBI!?"
+    except KeyboardInterrupt:
+        pass
+
+
+@tractor.context
+async def inf_streamer(
+    ctx: tractor.Context,
+
+) -> None:
+    '''
+    Stream increasing ints until terminated with a 'done' msg.
+
+    '''
+    await ctx.started()
+
+    async with (
+        ctx.open_stream() as stream,
+        trio.open_nursery() as n,
+    ):
+        async def bail_on_sentinel():
+            async for msg in stream:
+                if msg == 'done':
+                    await stream.aclose()
+                else:
+                    print(f'streamer received {msg}')
+
+        # start termination detector
+        n.start_soon(bail_on_sentinel)
+
+        for val in itertools.count():
+            try:
+                await stream.send(val)
+            except trio.ClosedResourceError:
+                # close out the stream gracefully
+                break
+
+    print('terminating streamer')
+
+
+def test_local_task_fanout_from_stream():
+    '''
+    Single stream with multiple local consumer tasks using the
+    ``MsgStream.subscribe()` api.
+
+    Ensure all tasks receive all values after stream completes sending.
+
+    '''
+    consumers = 22
+
+    async def main():
+
+        counts = Counter()
+
+        async with tractor.open_nursery() as tn:
+            p = await tn.start_actor(
+                'inf_streamer',
+                enable_modules=[__name__],
+            )
+            async with (
+                p.open_context(inf_streamer) as (ctx, _),
+                ctx.open_stream() as stream,
+            ):
+
+                async def pull_and_count(name: str):
+                    # name = trio.lowlevel.current_task().name
+                    async with stream.subscribe() as recver:
+                        assert isinstance(
+                            recver,
+                            tractor.trionics.BroadcastReceiver
+                        )
+                        async for val in recver:
+                            # print(f'{name}: {val}')
+                            counts[name] += 1
+
+                        print(f'{name} bcaster ended')
+
+                    print(f'{name} completed')
+
+                with trio.fail_after(3):
+                    async with trio.open_nursery() as nurse:
+                        for i in range(consumers):
+                            nurse.start_soon(pull_and_count, i)
+
+                        await trio.sleep(0.5)
+                        print('\nterminating')
+                        await stream.send('done')
+
+            print('closed stream connection')
+
+            assert len(counts) == consumers
+            mx = max(counts.values())
+            # make sure each task received all stream values
+            assert all(val == mx for val in counts.values())
+
+            await p.cancel_actor()
+
+    trio.run(main)

tests/test_cancellation.py

@@ -1,61 +1,207 @@
 """
 Cancellation and error propagation
+
 """
+import os
+import signal
+import platform
+import time
 from itertools import repeat
 
+from exceptiongroup import (
+    BaseExceptionGroup,
+    ExceptionGroup,
+)
 import pytest
 import trio
 import tractor
 
-from conftest import tractor_test
+from conftest import tractor_test, no_windows
 
 
-async def assert_err():
+def is_win():
+    return platform.system() == 'Windows'
+
+
+async def assert_err(delay=0):
+    await trio.sleep(delay)
     assert 0
 
 
-def test_remote_error(arb_addr):
-    """Verify an error raises in a subactor is propagated to the parent.
-    """
-    async def main():
-        async with tractor.open_nursery() as nursery:
+async def sleep_forever():
+    await trio.sleep_forever()
 
-            portal = await nursery.run_in_actor('errorer', assert_err)
+
+async def do_nuthin():
+    # just nick the scheduler
+    await trio.sleep(0)
+
+
+@pytest.mark.parametrize(
+    'args_err',
+    [
+        # expected to be thrown in assert_err
+        ({}, AssertionError),
+        # argument mismatch raised in _invoke()
+        ({'unexpected': 10}, TypeError)
+    ],
+    ids=['no_args', 'unexpected_args'],
+)
+def test_remote_error(arb_addr, args_err):
+    """Verify an error raised in a subactor that is propagated
+    to the parent nursery, contains the underlying boxed builtin
+    error type info and causes cancellation and reraising all the
+    way up the stack.
+    """
+    args, errtype = args_err
+
+    async def main():
+        async with tractor.open_nursery(
+            arbiter_addr=arb_addr,
+        ) as nursery:
+
+            # on a remote type error caused by bad input args
+            # this should raise directly which means we **don't** get
+            # an exception group outside the nursery since the error
+            # here and the far end task error are one in the same?
+            portal = await nursery.run_in_actor(
+                assert_err, name='errorer', **args
+            )
 
             # get result(s) from main task
             try:
-                return await portal.result()
-            except tractor.RemoteActorError:
+                # this means the root actor will also raise a local
+                # parent task error and thus an eg will propagate out
+                # of this actor nursery.
+                await portal.result()
+            except tractor.RemoteActorError as err:
+                assert err.type == errtype
                 print("Look Maa that actor failed hard, hehh")
                 raise
-            except Exception:
-                pass
-            assert 0, "Remote error was not raised?"
 
-    with pytest.raises(tractor.RemoteActorError):
-        # also raises
-        tractor.run(main, arbiter_addr=arb_addr)
+    # ensure boxed errors
+    if args:
+        with pytest.raises(tractor.RemoteActorError) as excinfo:
+            trio.run(main)
+
+        assert excinfo.value.type == errtype
+
+    else:
+        # the root task will also error on the `.result()` call
+        # so we expect an error from there AND the child.
+        with pytest.raises(BaseExceptionGroup) as excinfo:
+            trio.run(main)
+
+        # ensure boxed errors
+        for exc in excinfo.value.exceptions:
+            assert exc.type == errtype
 
 
-def do_nothing():
+def test_multierror(arb_addr):
+    '''
+    Verify we raise a ``BaseExceptionGroup`` out of a nursery where
+    more then one actor errors.
+
+    '''
+    async def main():
+        async with tractor.open_nursery(
+            arbiter_addr=arb_addr,
+        ) as nursery:
+
+            await nursery.run_in_actor(assert_err, name='errorer1')
+            portal2 = await nursery.run_in_actor(assert_err, name='errorer2')
+
+            # get result(s) from main task
+            try:
+                await portal2.result()
+            except tractor.RemoteActorError as err:
+                assert err.type == AssertionError
+                print("Look Maa that first actor failed hard, hehh")
+                raise
+
+        # here we should get a ``BaseExceptionGroup`` containing exceptions
+        # from both subactors
+
+    with pytest.raises(BaseExceptionGroup):
+        trio.run(main)
+
+
+@pytest.mark.parametrize('delay', (0, 0.5))
+@pytest.mark.parametrize(
+    'num_subactors', range(25, 26),
+)
+def test_multierror_fast_nursery(arb_addr, start_method, num_subactors, delay):
+    """Verify we raise a ``BaseExceptionGroup`` out of a nursery where
+    more then one actor errors and also with a delay before failure
+    to test failure during an ongoing spawning.
+    """
+    async def main():
+        async with tractor.open_nursery(
+            arbiter_addr=arb_addr,
+        ) as nursery:
+
+            for i in range(num_subactors):
+                await nursery.run_in_actor(
+                    assert_err,
+                    name=f'errorer{i}',
+                    delay=delay
+                )
+
+    # with pytest.raises(trio.MultiError) as exc_info:
+    with pytest.raises(BaseExceptionGroup) as exc_info:
+        trio.run(main)
+
+    assert exc_info.type == ExceptionGroup
+    err = exc_info.value
+    exceptions = err.exceptions
+
+    if len(exceptions) == 2:
+        # sometimes oddly now there's an embedded BrokenResourceError ?
+        for exc in exceptions:
+            excs = getattr(exc, 'exceptions', None)
+            if excs:
+                exceptions = excs
+                break
+
+    assert len(exceptions) == num_subactors
+
+    for exc in exceptions:
+        assert isinstance(exc, tractor.RemoteActorError)
+        assert exc.type == AssertionError
+
+
+async def do_nothing():
     pass
 
 
-def test_cancel_single_subactor(arb_addr):
-
-    async def main():
-
-        async with tractor.open_nursery() as nursery:
+@pytest.mark.parametrize('mechanism', ['nursery_cancel', KeyboardInterrupt])
+def test_cancel_single_subactor(arb_addr, mechanism):
+    """Ensure a ``ActorNursery.start_actor()`` spawned subactor
+    cancels when the nursery is cancelled.
+    """
+    async def spawn_actor():
+        """Spawn an actor that blocks indefinitely.
+        """
+        async with tractor.open_nursery(
+            arbiter_addr=arb_addr,
+        ) as nursery:
 
             portal = await nursery.start_actor(
-                'nothin', rpc_module_paths=[__name__],
+                'nothin', enable_modules=[__name__],
             )
-            assert (await portal.run(__name__, 'do_nothing')) is None
+            assert (await portal.run(do_nothing)) is None
 
-            # would hang otherwise
-            await nursery.cancel()
+            if mechanism == 'nursery_cancel':
+                # would hang otherwise
+                await nursery.cancel()
+            else:
+                raise mechanism
 
-    tractor.run(main, arbiter_addr=arb_addr)
+    if mechanism == 'nursery_cancel':
+        trio.run(spawn_actor)
+    else:
+        with pytest.raises(mechanism):
+            trio.run(spawn_actor)
 
 
 async def stream_forever():
@@ -67,50 +213,389 @@ async def stream_forever():
 
 
 @tractor_test
-async def test_cancel_infinite_streamer():
+async def test_cancel_infinite_streamer(start_method):
 
     # stream for at most 1 seconds
     with trio.move_on_after(1) as cancel_scope:
         async with tractor.open_nursery() as n:
             portal = await n.start_actor(
-                f'donny',
-                rpc_module_paths=[__name__],
+                'donny',
+                enable_modules=[__name__],
             )
 
             # this async for loop streams values from the above
             # async generator running in a separate process
-            async for letter in await portal.run(__name__, 'stream_forever'):
-                print(letter)
+            async with portal.open_stream_from(stream_forever) as stream:
+                async for letter in stream:
+                    print(letter)
 
     # we support trio's cancellation system
     assert cancel_scope.cancelled_caught
     assert n.cancelled
 
 
+@pytest.mark.parametrize(
+    'num_actors_and_errs',
+    [
+        # daemon actors sit idle while single task actors error out
+        (1, tractor.RemoteActorError, AssertionError, (assert_err, {}), None),
+        (2, BaseExceptionGroup, AssertionError, (assert_err, {}), None),
+        (3, BaseExceptionGroup, AssertionError, (assert_err, {}), None),
+
+        # 1 daemon actor errors out while single task actors sleep forever
+        (3, tractor.RemoteActorError, AssertionError, (sleep_forever, {}),
+         (assert_err, {}, True)),
+        # daemon actors error out after brief delay while single task
+        # actors complete quickly
+        (3, tractor.RemoteActorError, AssertionError,
+         (do_nuthin, {}), (assert_err, {'delay': 1}, True)),
+        # daemon complete quickly delay while single task
+        # actors error after brief delay
+        (3, BaseExceptionGroup, AssertionError,
+         (assert_err, {'delay': 1}), (do_nuthin, {}, False)),
+    ],
+    ids=[
+        '1_run_in_actor_fails',
+        '2_run_in_actors_fail',
+        '3_run_in_actors_fail',
+        '1_daemon_actors_fail',
+        '1_daemon_actors_fail_all_run_in_actors_dun_quick',
+        'no_daemon_actors_fail_all_run_in_actors_sleep_then_fail',
+    ],
+)
 @tractor_test
-async def test_one_cancels_all():
-    """Verify one failed actor causes all others in the nursery
-    to be cancelled just like in trio.
+async def test_some_cancels_all(num_actors_and_errs, start_method, loglevel):
+    """Verify a subset of failed subactors causes all others in
+    the nursery to be cancelled just like the strategy in trio.
 
     This is the first and only supervisory strategy at the moment.
     """
+    num_actors, first_err, err_type, ria_func, da_func = num_actors_and_errs
     try:
         async with tractor.open_nursery() as n:
-            real_actors = []
-            for i in range(3):
-                real_actors.append(await n.start_actor(
-                    f'actor_{i}',
-                    rpc_module_paths=[__name__],
+
+            # spawn the same number of deamon actors which should be cancelled
+            dactor_portals = []
+            for i in range(num_actors):
+                dactor_portals.append(await n.start_actor(
+                    f'deamon_{i}',
+                    enable_modules=[__name__],
                 ))
 
-            # start one actor that will fail immediately
-            await n.run_in_actor('extra', assert_err)
+            func, kwargs = ria_func
+            riactor_portals = []
+            for i in range(num_actors):
+                # start actor(s) that will fail immediately
+                riactor_portals.append(
+                    await n.run_in_actor(
+                        func,
+                        name=f'actor_{i}',
+                        **kwargs
+                    )
+                )
 
-        # should error here with a ``RemoteActorError`` containing
-        # an ``AssertionError`
+            if da_func:
+                func, kwargs, expect_error = da_func
+                for portal in dactor_portals:
+                    # if this function fails then we should error here
+                    # and the nursery should teardown all other actors
+                    try:
+                        await portal.run(func, **kwargs)
+
+                    except tractor.RemoteActorError as err:
+                        assert err.type == err_type
+                        # we only expect this first error to propogate
+                        # (all other daemons are cancelled before they
+                        # can be scheduled)
+                        num_actors = 1
+                        # reraise so nursery teardown is triggered
+                        raise
+                    else:
+                        if expect_error:
+                            pytest.fail(
+                                "Deamon call should fail at checkpoint?")
+
+        # should error here with a ``RemoteActorError`` or ``MultiError``
+
+    except first_err as err:
+        if isinstance(err, BaseExceptionGroup):
+            assert len(err.exceptions) == num_actors
+            for exc in err.exceptions:
+                if isinstance(exc, tractor.RemoteActorError):
+                    assert exc.type == err_type
+                else:
+                    assert isinstance(exc, trio.Cancelled)
+        elif isinstance(err, tractor.RemoteActorError):
+            assert err.type == err_type
 
-    except tractor.RemoteActorError:
         assert n.cancelled is True
         assert not n._children
     else:
         pytest.fail("Should have gotten a remote assertion error?")
+
+
+async def spawn_and_error(breadth, depth) -> None:
+    name = tractor.current_actor().name
+    async with tractor.open_nursery() as nursery:
+        for i in range(breadth):
+
+            if depth > 0:
+
+                args = (
+                    spawn_and_error,
+                )
+                kwargs = {
+                    'name': f'spawner_{i}_depth_{depth}',
+                    'breadth': breadth,
+                    'depth': depth - 1,
+                }
+            else:
+                args = (
+                    assert_err,
+                )
+                kwargs = {
+                    'name': f'{name}_errorer_{i}',
+                }
+            await nursery.run_in_actor(*args, **kwargs)
+
+
+@tractor_test
+async def test_nested_multierrors(loglevel, start_method):
+    '''
+    Test that failed actor sets are wrapped in `BaseExceptionGroup`s. This
+    test goes only 2 nurseries deep but we should eventually have tests
+    for arbitrary n-depth actor trees.
+
+    '''
+    if start_method == 'trio':
+        depth = 3
+        subactor_breadth = 2
+    else:
+        # XXX: multiprocessing can't seem to handle any more then 2 depth
+        # process trees for whatever reason.
+        # Any more process levels then this and we see bugs that cause
+        # hangs and broken pipes all over the place...
+        if start_method == 'forkserver':
+            pytest.skip("Forksever sux hard at nested spawning...")
+        depth = 1  # means an additional actor tree of spawning (2 levels deep)
+        subactor_breadth = 2
+
+    with trio.fail_after(120):
+        try:
+            async with tractor.open_nursery() as nursery:
+                for i in range(subactor_breadth):
+                    await nursery.run_in_actor(
+                        spawn_and_error,
+                        name=f'spawner_{i}',
+                        breadth=subactor_breadth,
+                        depth=depth,
+                    )
+        except BaseExceptionGroup as err:
+            assert len(err.exceptions) == subactor_breadth
+            for subexc in err.exceptions:
+
+                # verify first level actor errors are wrapped as remote
+                if is_win():
+
+                    # windows is often too slow and cancellation seems
+                    # to happen before an actor is spawned
+                    if isinstance(subexc, trio.Cancelled):
+                        continue
+
+                    elif isinstance(subexc, tractor.RemoteActorError):
+                        # on windows it seems we can't exactly be sure wtf
+                        # will happen..
+                        assert subexc.type in (
+                            tractor.RemoteActorError,
+                            trio.Cancelled,
+                            BaseExceptionGroup,
+                        )
+
+                    elif isinstance(subexc, BaseExceptionGroup):
+                        for subsub in subexc.exceptions:
+
+                            if subsub in (tractor.RemoteActorError,):
+                                subsub = subsub.type
+
+                            assert type(subsub) in (
+                                trio.Cancelled,
+                                BaseExceptionGroup,
+                            )
+                else:
+                    assert isinstance(subexc, tractor.RemoteActorError)
+
+                if depth > 0 and subactor_breadth > 1:
+                    # XXX not sure what's up with this..
+                    # on windows sometimes spawning is just too slow and
+                    # we get back the (sent) cancel signal instead
+                    if is_win():
+                        if isinstance(subexc, tractor.RemoteActorError):
+                            assert subexc.type in (
+                                BaseExceptionGroup,
+                                tractor.RemoteActorError
+                            )
+                        else:
+                            assert isinstance(subexc, BaseExceptionGroup)
+                    else:
+                        assert subexc.type is ExceptionGroup
+                else:
+                    assert subexc.type in (
+                        tractor.RemoteActorError,
+                        trio.Cancelled
+                    )
+
+
+@no_windows
+def test_cancel_via_SIGINT(
+    loglevel,
+    start_method,
+    spawn_backend,
+):
+    """Ensure that a control-C (SIGINT) signal cancels both the parent and
+    child processes in trionic fashion
+    """
+    pid = os.getpid()
+
+    async def main():
+        with trio.fail_after(2):
+            async with tractor.open_nursery() as tn:
+                await tn.start_actor('sucka')
+                if 'mp' in spawn_backend:
+                    time.sleep(0.1)
+                os.kill(pid, signal.SIGINT)
+                await trio.sleep_forever()
+
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(main)
+
+
+@no_windows
+def test_cancel_via_SIGINT_other_task(
+    loglevel,
+    start_method,
+    spawn_backend,
+):
+    """Ensure that a control-C (SIGINT) signal cancels both the parent
+    and child processes in trionic fashion even a subprocess is started
+    from a seperate ``trio`` child  task.
+    """
+    pid = os.getpid()
+    timeout: float = 2
+    if is_win():  # smh
+        timeout += 1
+
+    async def spawn_and_sleep_forever(task_status=trio.TASK_STATUS_IGNORED):
+        async with tractor.open_nursery() as tn:
+            for i in range(3):
+                await tn.run_in_actor(
+                    sleep_forever,
+                    name='namesucka',
+                )
+            task_status.started()
+            await trio.sleep_forever()
+
+    async def main():
+        # should never timeout since SIGINT should cancel the current program
+        with trio.fail_after(timeout):
+            async with trio.open_nursery() as n:
+                await n.start(spawn_and_sleep_forever)
+                if 'mp' in spawn_backend:
+                    time.sleep(0.1)
+                os.kill(pid, signal.SIGINT)
+
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(main)
+
+
+async def spin_for(period=3):
+    "Sync sleep."
+    time.sleep(period)
+
+
+async def spawn():
+    async with tractor.open_nursery() as tn:
+        await tn.run_in_actor(
+            spin_for,
+            name='sleeper',
+        )
+
+
+@no_windows
+def test_cancel_while_childs_child_in_sync_sleep(
+    loglevel,
+    start_method,
+    spawn_backend,
+):
+    """Verify that a child cancelled while executing sync code is torn
+    down even when that cancellation is triggered by the parent
+    2 nurseries "up".
+    """
+    if start_method == 'forkserver':
+        pytest.skip("Forksever sux hard at resuming from sync sleep...")
+
+    async def main():
+        with trio.fail_after(2):
+            async with tractor.open_nursery() as tn:
+                await tn.run_in_actor(
+                    spawn,
+                    name='spawn',
+                )
+                await trio.sleep(1)
+                assert 0
+
+    with pytest.raises(AssertionError):
+        trio.run(main)
+
+
+def test_fast_graceful_cancel_when_spawn_task_in_soft_proc_wait_for_daemon(
+    start_method,
+):
+    '''
+    This is a very subtle test which demonstrates how cancellation
+    during process collection can result in non-optimal teardown
+    performance on daemon actors. The fix for this test was to handle
+    ``trio.Cancelled`` specially in the spawn task waiting in
+    `proc.wait()` such that ``Portal.cancel_actor()`` is called before
+    executing the "hard reap" sequence (which has an up to 3 second
+    delay currently).
+
+    In other words, if we can cancel the actor using a graceful remote
+    cancellation, and it's faster, we might as well do it.
+
+    '''
+    kbi_delay = 0.5
+    timeout: float = 2.9
+
+    if is_win():  # smh
+        timeout += 1
+
+    async def main():
+        start = time.time()
+        try:
+            async with trio.open_nursery() as nurse:
+                async with tractor.open_nursery() as tn:
+                    p = await tn.start_actor(
+                        'fast_boi',
+                        enable_modules=[__name__],
+                    )
+
+                    async def delayed_kbi():
+                        await trio.sleep(kbi_delay)
+                        print(f'RAISING KBI after {kbi_delay} s')
+                        raise KeyboardInterrupt
+
+                    # start task which raises a kbi **after**
+                    # the actor nursery ``__aexit__()`` has
+                    # been run.
+                    nurse.start_soon(delayed_kbi)
+
+                    await p.run(do_nuthin)
+        finally:
+            duration = time.time() - start
+            if duration > timeout:
+                raise trio.TooSlowError(
+                    'daemon cancel was slower then necessary..'
+                )
+
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(main)

tests/test_child_manages_service_nursery.py

@@ -0,0 +1,173 @@
+'''
+Test a service style daemon that maintains a nursery for spawning
+"remote async tasks" including both spawning other long living
+sub-sub-actor daemons.
+
+'''
+from typing import Optional
+import asyncio
+from contextlib import asynccontextmanager as acm
+
+import pytest
+import trio
+from trio_typing import TaskStatus
+import tractor
+from tractor import RemoteActorError
+from async_generator import aclosing
+
+
+async def aio_streamer(
+    from_trio: asyncio.Queue,
+    to_trio: trio.abc.SendChannel,
+) -> trio.abc.ReceiveChannel:
+
+    # required first msg to sync caller
+    to_trio.send_nowait(None)
+
+    from itertools import cycle
+    for i in cycle(range(10)):
+        to_trio.send_nowait(i)
+        await asyncio.sleep(0.01)
+
+
+async def trio_streamer():
+    from itertools import cycle
+    for i in cycle(range(10)):
+        yield i
+        await trio.sleep(0.01)
+
+
+async def trio_sleep_and_err(delay: float = 0.5):
+    await trio.sleep(delay)
+    # name error
+    doggy()  # noqa
+
+
+_cached_stream: Optional[
+    trio.abc.ReceiveChannel
+] = None
+
+
+@acm
+async def wrapper_mngr(
+):
+    from tractor.trionics import broadcast_receiver
+    global _cached_stream
+    in_aio = tractor.current_actor().is_infected_aio()
+
+    if in_aio:
+        if _cached_stream:
+
+            from_aio = _cached_stream
+
+            # if we already have a cached feed deliver a rx side clone
+            # to consumer
+            async with broadcast_receiver(from_aio, 6) as from_aio:
+                yield from_aio
+                return
+        else:
+            async with tractor.to_asyncio.open_channel_from(
+                aio_streamer,
+            ) as (first, from_aio):
+                assert not first
+
+                # cache it so next task uses broadcast receiver
+                _cached_stream = from_aio
+
+                yield from_aio
+    else:
+        async with aclosing(trio_streamer()) as stream:
+            # cache it so next task uses broadcast receiver
+            _cached_stream = stream
+            yield stream
+
+
+_nursery: trio.Nursery = None
+
+
+@tractor.context
+async def trio_main(
+    ctx: tractor.Context,
+):
+    # sync
+    await ctx.started()
+
+    # stash a "service nursery" as "actor local" (aka a Python global)
+    global _nursery
+    n = _nursery
+    assert n
+
+    async def consume_stream():
+        async with wrapper_mngr() as stream:
+            async for msg in stream:
+                print(msg)
+
+    # run 2 tasks to ensure broadcaster chan use
+    n.start_soon(consume_stream)
+    n.start_soon(consume_stream)
+
+    n.start_soon(trio_sleep_and_err)
+
+    await trio.sleep_forever()
+
+
+@tractor.context
+async def open_actor_local_nursery(
+    ctx: tractor.Context,
+):
+    global _nursery
+    async with trio.open_nursery() as n:
+        _nursery = n
+        await ctx.started()
+        await trio.sleep(10)
+        # await trio.sleep(1)
+
+        # XXX: this causes the hang since
+        # the caller does not unblock from its own
+        # ``trio.sleep_forever()``.
+
+        # TODO: we need to test a simple ctx task starting remote tasks
+        # that error and then blocking on a ``Nursery.start()`` which
+        # never yields back.. aka a scenario where the
+        # ``tractor.context`` task IS NOT in the service n's cancel
+        # scope.
+        n.cancel_scope.cancel()
+
+
+@pytest.mark.parametrize(
+    'asyncio_mode',
+    [True, False],
+    ids='asyncio_mode={}'.format,
+)
+def test_actor_managed_trio_nursery_task_error_cancels_aio(
+    asyncio_mode: bool,
+    arb_addr
+):
+    '''
+    Verify that a ``trio`` nursery created managed in a child actor
+    correctly relays errors to the parent actor when one of its spawned
+    tasks errors even when running in infected asyncio mode and using
+    broadcast receivers for multi-task-per-actor subscription.
+
+    '''
+    async def main():
+
+        # cancel the nursery shortly after boot
+        async with tractor.open_nursery() as n:
+            p = await n.start_actor(
+                'nursery_mngr',
+                infect_asyncio=asyncio_mode,
+                enable_modules=[__name__],
+            )
+            async with (
+                p.open_context(open_actor_local_nursery) as (ctx, first),
+                p.open_context(trio_main) as (ctx, first),
+            ):
+                await trio.sleep_forever()
+
+    with pytest.raises(RemoteActorError) as excinfo:
+        trio.run(main)
+
+    # verify boxed error
+    err = excinfo.value
+    assert isinstance(err.type(), NameError)

tests/test_clustering.py

@@ -0,0 +1,84 @@
+import itertools
+
+import pytest
+import trio
+import tractor
+from tractor import open_actor_cluster
+from tractor.trionics import gather_contexts
+
+from conftest import tractor_test
+
+
+MESSAGE = 'tractoring at full speed'
+
+
+def test_empty_mngrs_input_raises() -> None:
+
+    async def main():
+        with trio.fail_after(1):
+            async with (
+                open_actor_cluster(
+                    modules=[__name__],
+
+                    # NOTE: ensure we can passthrough runtime opts
+                    loglevel='info',
+                    # debug_mode=True,
+
+                ) as portals,
+
+                gather_contexts(
+                    # NOTE: it's the use of inline-generator syntax
+                    # here that causes the empty input.
+                    mngrs=(
+                        p.open_context(worker) for p in portals.values()
+                    ),
+                ),
+            ):
+                assert 0
+
+    with pytest.raises(ValueError):
+        trio.run(main)
+
+
+@tractor.context
+async def worker(
+    ctx: tractor.Context,
+
+) -> None:
+
+    await ctx.started()
+
+    async with ctx.open_stream(
+        backpressure=True,
+    ) as stream:
+
+        # TODO: this with the below assert causes a hang bug?
+        # with trio.move_on_after(1):
+
+        async for msg in stream:
+            # do something with msg
+            print(msg)
+            assert msg == MESSAGE
+
+        # TODO: does this ever cause a hang
+        # assert 0
+
+
+@tractor_test
+async def test_streaming_to_actor_cluster() -> None:
+
+    async with (
+        open_actor_cluster(modules=[__name__]) as portals,
+
+        gather_contexts(
+            mngrs=[p.open_context(worker) for p in portals.values()],
+        ) as contexts,
+
+        gather_contexts(
+            mngrs=[ctx[0].open_stream() for ctx in contexts],
+        ) as streams,
+
+    ):
+        with trio.move_on_after(1):
+            for stream in itertools.cycle(streams):
+                await stream.send(MESSAGE)

tests/test_context_stream_semantics.py

@@ -0,0 +1,798 @@
+'''
+``async with ():`` inlined context-stream cancellation testing.
+
+Verify the we raise errors when streams are opened prior to sync-opening
+a ``tractor.Context`` beforehand.
+
+'''
+from contextlib import asynccontextmanager as acm
+from itertools import count
+import platform
+from typing import Optional
+
+import pytest
+import trio
+import tractor
+from tractor._exceptions import StreamOverrun
+
+from conftest import tractor_test
+
+# ``Context`` semantics are as follows,
+#  ------------------------------------
+
+# - standard setup/teardown:
+#   ``Portal.open_context()`` starts a new
+#   remote task context in another actor. The target actor's task must
+#   call ``Context.started()`` to unblock this entry on the caller side.
+#   the callee task executes until complete and returns a final value
+#   which is delivered to the caller side and retreived via
+#   ``Context.result()``.
+
+# - cancel termination:
+#   context can be cancelled on either side where either end's task can
+#   call ``Context.cancel()`` which raises a local ``trio.Cancelled``
+#   and sends a task cancel request to the remote task which in turn
+#   raises a ``trio.Cancelled`` in that scope, catches it, and re-raises
+#   as ``ContextCancelled``. This is then caught by
+#   ``Portal.open_context()``'s exit and we get a graceful termination
+#   of the linked tasks.
+
+# - error termination:
+#   error is caught after all context-cancel-scope tasks are cancelled
+#   via regular ``trio`` cancel scope semantics, error is sent to other
+#   side and unpacked as a `RemoteActorError`.
+
+
+# ``Context.open_stream() as stream: MsgStream:`` msg semantics are:
+#  -----------------------------------------------------------------
+
+# - either side can ``.send()`` which emits a 'yield' msgs and delivers
+#   a value to the a ``MsgStream.receive()`` call.
+
+# - stream closure: one end relays a 'stop' message which terminates an
+#   ongoing ``MsgStream`` iteration.
+
+# - cancel/error termination: as per the context semantics above but
+#   with implicit stream closure on the cancelling end.
+
+
+_state: bool = False
+
+
+@tractor.context
+async def too_many_starteds(
+    ctx: tractor.Context,
+) -> None:
+    '''
+    Call ``Context.started()`` more then once (an error).
+
+    '''
+    await ctx.started()
+    try:
+        await ctx.started()
+    except RuntimeError:
+        raise
+
+
+@tractor.context
+async def not_started_but_stream_opened(
+    ctx: tractor.Context,
+) -> None:
+    '''
+    Enter ``Context.open_stream()`` without calling ``.started()``.
+
+    '''
+    try:
+        async with ctx.open_stream():
+            assert 0
+    except RuntimeError:
+        raise
+
+
+@pytest.mark.parametrize(
+    'target',
+    [too_many_starteds, not_started_but_stream_opened],
+    ids='misuse_type={}'.format,
+)
+def test_started_misuse(target):
+
+    async def main():
+        async with tractor.open_nursery() as n:
+            portal = await n.start_actor(
+                target.__name__,
+                enable_modules=[__name__],
+            )
+
+            async with portal.open_context(target) as (ctx, sent):
+                await trio.sleep(1)
+
+    with pytest.raises(tractor.RemoteActorError):
+        trio.run(main)
+
+
+@tractor.context
+async def simple_setup_teardown(
+
+    ctx: tractor.Context,
+    data: int,
+    block_forever: bool = False,
+
+) -> None:
+
+    # startup phase
+    global _state
+    _state = True
+
+    # signal to parent that we're up
+    await ctx.started(data + 1)
+
+    try:
+        if block_forever:
+            # block until cancelled
+            await trio.sleep_forever()
+        else:
+            return 'yo'
+    finally:
+        _state = False
+
+
+async def assert_state(value: bool):
+    global _state
+    assert _state == value
+
+
+@pytest.mark.parametrize(
+    'error_parent',
+    [False, ValueError, KeyboardInterrupt],
+)
+@pytest.mark.parametrize(
+    'callee_blocks_forever',
+    [False, True],
+    ids=lambda item: f'callee_blocks_forever={item}'
+)
+@pytest.mark.parametrize(
+    'pointlessly_open_stream',
+    [False, True],
+    ids=lambda item: f'open_stream={item}'
+)
+def test_simple_context(
+    error_parent,
+    callee_blocks_forever,
+    pointlessly_open_stream,
+):
+
+    timeout = 1.5 if not platform.system() == 'Windows' else 4
+
+    async def main():
+
+        with trio.fail_after(timeout):
+            async with tractor.open_nursery() as nursery:
+
+                portal = await nursery.start_actor(
+                    'simple_context',
+                    enable_modules=[__name__],
+                )
+
+                try:
+                    async with portal.open_context(
+                        simple_setup_teardown,
+                        data=10,
+                        block_forever=callee_blocks_forever,
+                    ) as (ctx, sent):
+
+                        assert sent == 11
+
+                        if callee_blocks_forever:
+                            await portal.run(assert_state, value=True)
+                        else:
+                            assert await ctx.result() == 'yo'
+
+                        if not error_parent:
+                            await ctx.cancel()
+
+                        if pointlessly_open_stream:
+                            async with ctx.open_stream():
+                                if error_parent:
+                                    raise error_parent
+
+                                if callee_blocks_forever:
+                                    await ctx.cancel()
+                                else:
+                                    # in this case the stream will send a
+                                    # 'stop' msg to the far end which needs
+                                    # to be ignored
+                                    pass
+                        else:
+                            if error_parent:
+                                raise error_parent
+
+                finally:
+
+                    # after cancellation
+                    if not error_parent:
+                        await portal.run(assert_state, value=False)
+
+                    # shut down daemon
+                    await portal.cancel_actor()
+
+    if error_parent:
+        try:
+            trio.run(main)
+        except error_parent:
+            pass
+        except trio.MultiError as me:
+            # XXX: on windows it seems we may have to expect the group error
+            from tractor._exceptions import is_multi_cancelled
+            assert is_multi_cancelled(me)
+    else:
+        trio.run(main)
+
+
+# basic stream terminations:
+# - callee context closes without using stream
+# - caller context closes without using stream
+# - caller context calls `Context.cancel()` while streaming
+#   is ongoing resulting in callee being cancelled
+# - callee calls `Context.cancel()` while streaming and caller
+#   sees stream terminated in `RemoteActorError`
+
+# TODO: future possible features
+# - restart request: far end raises `ContextRestart`
+
+
+@tractor.context
+async def close_ctx_immediately(
+
+    ctx: tractor.Context,
+
+) -> None:
+
+    await ctx.started()
+    global _state
+
+    async with ctx.open_stream():
+        pass
+
+
+@tractor_test
+async def test_callee_closes_ctx_after_stream_open():
+    'callee context closes without using stream'
+
+    async with tractor.open_nursery() as n:
+
+        portal = await n.start_actor(
+            'fast_stream_closer',
+            enable_modules=[__name__],
+        )
+
+        with trio.fail_after(2):
+            async with portal.open_context(
+                close_ctx_immediately,
+
+                # flag to avoid waiting the final result
+                # cancel_on_exit=True,
+
+            ) as (ctx, sent):
+
+                assert sent is None
+
+                with trio.fail_after(0.5):
+                    async with ctx.open_stream() as stream:
+
+                        # should fall through since ``StopAsyncIteration``
+                        # should be raised through translation of
+                        # a ``trio.EndOfChannel`` by
+                        # ``trio.abc.ReceiveChannel.__anext__()``
+                        async for _ in stream:
+                            assert 0
+                        else:
+
+                            # verify stream is now closed
+                            try:
+                                await stream.receive()
+                            except trio.EndOfChannel:
+                                pass
+
+                # TODO: should be just raise the closed resource err
+                # directly here to enforce not allowing a re-open
+                # of a stream to the context (at least until a time of
+                # if/when we decide that's a good idea?)
+                try:
+                    with trio.fail_after(0.5):
+                        async with ctx.open_stream() as stream:
+                            pass
+                except trio.ClosedResourceError:
+                    pass
+
+        await portal.cancel_actor()
+
+
+@tractor.context
+async def expect_cancelled(
+
+    ctx: tractor.Context,
+
+) -> None:
+    global _state
+    _state = True
+
+    await ctx.started()
+
+    try:
+        async with ctx.open_stream() as stream:
+            async for msg in stream:
+                await stream.send(msg)  # echo server
+
+    except trio.Cancelled:
+        # expected case
+        _state = False
+        raise
+
+    else:
+        assert 0, "Wasn't cancelled!?"
+
+
+@pytest.mark.parametrize(
+    'use_ctx_cancel_method',
+    [False, True],
+)
+@tractor_test
+async def test_caller_closes_ctx_after_callee_opens_stream(
+    use_ctx_cancel_method: bool,
+):
+    'caller context closes without using stream'
+
+    async with tractor.open_nursery() as n:
+
+        portal = await n.start_actor(
+            'ctx_cancelled',
+            enable_modules=[__name__],
+        )
+
+        async with portal.open_context(
+            expect_cancelled,
+        ) as (ctx, sent):
+            await portal.run(assert_state, value=True)
+
+            assert sent is None
+
+            # call cancel explicitly
+            if use_ctx_cancel_method:
+
+                await ctx.cancel()
+
+                try:
+                    async with ctx.open_stream() as stream:
+                        async for msg in stream:
+                            pass
+
+                except tractor.ContextCancelled:
+                    raise  # XXX: must be propagated to __aexit__
+
+                else:
+                    assert 0, "Should have context cancelled?"
+
+                # channel should still be up
+                assert portal.channel.connected()
+
+                # ctx is closed here
+                await portal.run(assert_state, value=False)
+
+            else:
+                try:
+                    with trio.fail_after(0.2):
+                        await ctx.result()
+                        assert 0, "Callee should have blocked!?"
+                except trio.TooSlowError:
+                    await ctx.cancel()
+        try:
+            async with ctx.open_stream() as stream:
+                async for msg in stream:
+                    pass
+        except tractor.ContextCancelled:
+            pass
+        else:
+            assert 0, "Should have received closed resource error?"
+
+        # ctx is closed here
+        await portal.run(assert_state, value=False)
+
+        # channel should not have been destroyed yet, only the
+        # inter-actor-task context
+        assert portal.channel.connected()
+
+        # teardown the actor
+        await portal.cancel_actor()
+
+
+@tractor_test
+async def test_multitask_caller_cancels_from_nonroot_task():
+
+    async with tractor.open_nursery() as n:
+
+        portal = await n.start_actor(
+            'ctx_cancelled',
+            enable_modules=[__name__],
+        )
+
+        async with portal.open_context(
+            expect_cancelled,
+        ) as (ctx, sent):
+
+            await portal.run(assert_state, value=True)
+            assert sent is None
+
+            async with ctx.open_stream() as stream:
+
+                async def send_msg_then_cancel():
+                    await stream.send('yo')
+                    await portal.run(assert_state, value=True)
+                    await ctx.cancel()
+                    await portal.run(assert_state, value=False)
+
+                async with trio.open_nursery() as n:
+                    n.start_soon(send_msg_then_cancel)
+
+                    try:
+                        async for msg in stream:
+                            assert msg == 'yo'
+
+                    except tractor.ContextCancelled:
+                        raise  # XXX: must be propagated to __aexit__
+
+                # channel should still be up
+                assert portal.channel.connected()
+
+                # ctx is closed here
+                await portal.run(assert_state, value=False)
+
+        # channel should not have been destroyed yet, only the
+        # inter-actor-task context
+        assert portal.channel.connected()
+
+        # teardown the actor
+        await portal.cancel_actor()
+
+
+@tractor.context
+async def cancel_self(
+
+    ctx: tractor.Context,
+
+) -> None:
+    global _state
+    _state = True
+
+    await ctx.cancel()
+
+    # should inline raise immediately
+    try:
+        async with ctx.open_stream():
+            pass
+    except tractor.ContextCancelled:
+        # suppress for now so we can do checkpoint tests below
+        pass
+    else:
+        raise RuntimeError('Context didnt cancel itself?!')
+
+    # check a real ``trio.Cancelled`` is raised on a checkpoint
+    try:
+        with trio.fail_after(0.1):
+            await trio.sleep_forever()
+    except trio.Cancelled:
+        raise
+
+    except trio.TooSlowError:
+        # should never get here
+        assert 0
+
+
+@tractor_test
+async def test_callee_cancels_before_started():
+    '''
+    Callee calls `Context.cancel()` while streaming and caller
+    sees stream terminated in `ContextCancelled`.
+
+    '''
+    async with tractor.open_nursery() as n:
+
+        portal = await n.start_actor(
+            'cancels_self',
+            enable_modules=[__name__],
+        )
+        try:
+
+            async with portal.open_context(
+                cancel_self,
+            ) as (ctx, sent):
+                async with ctx.open_stream():
+
+                    await trio.sleep_forever()
+
+        # raises a special cancel signal
+        except tractor.ContextCancelled as ce:
+            ce.type == trio.Cancelled
+
+            # the traceback should be informative
+            assert 'cancelled itself' in ce.msgdata['tb_str']
+
+        # teardown the actor
+        await portal.cancel_actor()
+
+
+@tractor.context
+async def never_open_stream(
+
+    ctx:  tractor.Context,
+
+) -> None:
+    '''
+    Context which never opens a stream and blocks.
+
+    '''
+    await ctx.started()
+    await trio.sleep_forever()
+
+
+@tractor.context
+async def keep_sending_from_callee(
+
+    ctx:  tractor.Context,
+    msg_buffer_size: Optional[int] = None,
+
+) -> None:
+    '''
+    Send endlessly on the calleee stream.
+
+    '''
+    await ctx.started()
+    async with ctx.open_stream(
+        msg_buffer_size=msg_buffer_size,
+    ) as stream:
+        for msg in count():
+            print(f'callee sending {msg}')
+            await stream.send(msg)
+            await trio.sleep(0.01)
+
+
+@pytest.mark.parametrize(
+    'overrun_by',
+    [
+        ('caller', 1, never_open_stream),
+        ('cancel_caller_during_overrun', 1, never_open_stream),
+        ('callee', 0, keep_sending_from_callee),
+    ],
+    ids='overrun_condition={}'.format,
+)
+def test_one_end_stream_not_opened(overrun_by):
+    '''
+    This should exemplify the bug from:
+    https://github.com/goodboy/tractor/issues/265
+
+    '''
+    overrunner, buf_size_increase, entrypoint = overrun_by
+    from tractor._runtime import Actor
+    buf_size = buf_size_increase + Actor.msg_buffer_size
+
+    async def main():
+        async with tractor.open_nursery() as n:
+            portal = await n.start_actor(
+                entrypoint.__name__,
+                enable_modules=[__name__],
+            )
+
+            async with portal.open_context(
+                entrypoint,
+            ) as (ctx, sent):
+                assert sent is None
+
+                if 'caller' in overrunner:
+
+                    async with ctx.open_stream() as stream:
+                        for i in range(buf_size):
+                            print(f'sending {i}')
+                            await stream.send(i)
+
+                        if 'cancel' in overrunner:
+                            # without this we block waiting on the child side
+                            await ctx.cancel()
+
+                        else:
+                            # expect overrun error to be relayed back
+                            # and this sleep interrupted
+                            await trio.sleep_forever()
+
+                else:
+                    # callee overruns caller case so we do nothing here
+                    await trio.sleep_forever()
+
+            await portal.cancel_actor()
+
+    # 2 overrun cases and the no overrun case (which pushes right up to
+    # the msg limit)
+    if overrunner == 'caller' or 'cance' in overrunner:
+        with pytest.raises(tractor.RemoteActorError) as excinfo:
+            trio.run(main)
+
+        assert excinfo.value.type == StreamOverrun
+
+    elif overrunner == 'callee':
+        with pytest.raises(tractor.RemoteActorError) as excinfo:
+            trio.run(main)
+
+        # TODO: embedded remote errors so that we can verify the source
+        # error? the callee delivers an error which is an overrun
+        # wrapped in a remote actor error.
+        assert excinfo.value.type == tractor.RemoteActorError
+
+    else:
+        trio.run(main)
+
+
+@tractor.context
+async def echo_back_sequence(
+
+    ctx:  tractor.Context,
+    seq: list[int],
+    msg_buffer_size: Optional[int] = None,
+
+) -> None:
+    '''
+    Send endlessly on the calleee stream.
+
+    '''
+    await ctx.started()
+    async with ctx.open_stream(
+        msg_buffer_size=msg_buffer_size,
+    ) as stream:
+
+        seq = list(seq)  # bleh, `msgpack`...
+        count = 0
+        while count < 3:
+            batch = []
+            async for msg in stream:
+                batch.append(msg)
+                if batch == seq:
+                    break
+
+            for msg in batch:
+                print(f'callee sending {msg}')
+                await stream.send(msg)
+
+            count += 1
+
+        return 'yo'
+
+
+def test_stream_backpressure():
+    '''
+    Demonstrate small overruns of each task back and forth
+    on a stream not raising any errors by default.
+
+    '''
+    async def main():
+        async with tractor.open_nursery() as n:
+            portal = await n.start_actor(
+                'callee_sends_forever',
+                enable_modules=[__name__],
+            )
+            seq = list(range(3))
+            async with portal.open_context(
+                echo_back_sequence,
+                seq=seq,
+                msg_buffer_size=1,
+            ) as (ctx, sent):
+                assert sent is None
+
+                async with ctx.open_stream(msg_buffer_size=1) as stream:
+                    count = 0
+                    while count < 3:
+                        for msg in seq:
+                            print(f'caller sending {msg}')
+                            await stream.send(msg)
+                            await trio.sleep(0.1)
+
+                        batch = []
+                        async for msg in stream:
+                            batch.append(msg)
+                            if batch == seq:
+                                break
+
+                        count += 1
+
+            # here the context should return
+            assert await ctx.result() == 'yo'
+
+            # cancel the daemon
+            await portal.cancel_actor()
+
+    trio.run(main)
+
+
+@tractor.context
+async def sleep_forever(
+    ctx: tractor.Context,
+) -> None:
+    await ctx.started()
+    async with ctx.open_stream():
+        await trio.sleep_forever()
+
+
+@acm
+async def attach_to_sleep_forever():
+    '''
+    Cancel a context **before** any underlying error is raised in order
+    to trigger a local reception of a ``ContextCancelled`` which **should not**
+    be re-raised in the local surrounding ``Context`` *iff* the cancel was
+    requested by **this** side of the context.
+
+    '''
+    async with tractor.wait_for_actor('sleeper') as p2:
+        async with (
+            p2.open_context(sleep_forever) as (peer_ctx, first),
+            peer_ctx.open_stream(),
+        ):
+            try:
+                yield
+            finally:
+                # XXX: previously this would trigger local
+                # ``ContextCancelled`` to be received and raised in the
+                # local context overriding any local error due to
+                # logic inside ``_invoke()`` which checked for
+                # an error set on ``Context._error`` and raised it in
+                # under a cancellation scenario.
+
+                # The problem is you can have a remote cancellation
+                # that is part of a local error and we shouldn't raise
+                # ``ContextCancelled`` **iff** we weren't the side of
+                # the context to initiate it, i.e.
+                # ``Context._cancel_called`` should **NOT** have been
+                # set. The special logic to handle this case is now
+                # inside ``Context._may_raise_from_remote_msg()`` XD
+                await peer_ctx.cancel()
+
+
+@tractor.context
+async def error_before_started(
+    ctx: tractor.Context,
+) -> None:
+    '''
+    This simulates exactly an original bug discovered in:
+    https://github.com/pikers/piker/issues/244
+
+    '''
+    async with attach_to_sleep_forever():
+        # send an unserializable type which should raise a type error
+        # here and **NOT BE SWALLOWED** by the surrounding acm!!?!
+        await ctx.started(object())
+
+
+def test_do_not_swallow_error_before_started_by_remote_contextcancelled():
+    '''
+    Verify that an error raised in a remote context which itself opens another
+    remote context, which it cancels, does not ovverride the original error that
+    caused the cancellation of the secondardy context.
+
+    '''
+    async def main():
+        async with tractor.open_nursery() as n:
+            portal = await n.start_actor(
+                'errorer',
+                enable_modules=[__name__],
+            )
+            await n.start_actor(
+                'sleeper',
+                enable_modules=[__name__],
+            )
+
+            async with (
+                portal.open_context(
+                    error_before_started
+                ) as (ctx, sent),
+            ):
+                await trio.sleep_forever()
+
+    with pytest.raises(tractor.RemoteActorError) as excinfo:
+        trio.run(main)
+
+    assert excinfo.value.type == TypeError

tests/test_debugger.py

@@ -0,0 +1,933 @@
+"""
+That "native" debug mode better work!
+
+All these tests can be understood (somewhat) by running the equivalent
+`examples/debugging/` scripts manually.
+
+TODO:
+    - none of these tests have been run successfully on windows yet but
+      there's been manual testing that verified it works.
+    - wonder if any of it'll work on OS X?
+
+"""
+import itertools
+from os import path
+from typing import Optional
+import platform
+import pathlib
+import sys
+import time
+
+import pytest
+import pexpect
+from pexpect.exceptions import (
+    TIMEOUT,
+    EOF,
+)
+
+from conftest import (
+    examples_dir,
+    _ci_env,
+)
+
+# TODO: The next great debugger audit could be done by you!
+# - recurrent entry to breakpoint() from single actor *after* and an
+#   error in another task?
+# - root error before child errors
+# - root error after child errors
+# - root error before child breakpoint
+# - root error after child breakpoint
+# - recurrent root errors
+
+
+if platform.system() == 'Windows':
+    pytest.skip(
+        'Debugger tests have no windows support (yet)',
+        allow_module_level=True,
+    )
+
+
+def mk_cmd(ex_name: str) -> str:
+    '''
+    Generate a command suitable to pass to ``pexpect.spawn()``.
+
+    '''
+    script_path: pathlib.Path = examples_dir() / 'debugging' / f'{ex_name}.py'
+    return ' '.join(['python', str(script_path)])
+
+
+# TODO: was trying to this xfail style but some weird bug i see in CI
+# that's happening at collect time.. pretty soon gonna dump actions i'm
+# thinkin...
+# in CI we skip tests which >= depth 1 actor trees due to there
+# still being an oustanding issue with relaying the debug-mode-state
+# through intermediary parents.
+has_nested_actors = pytest.mark.has_nested_actors
+# .xfail(
+#     os.environ.get('CI', False),
+#     reason=(
+#         'This test uses nested actors and fails in CI\n'
+#         'The test seems to run fine locally but until we solve the '
+#         'following issue this CI test will be xfail:\n'
+#         'https://github.com/goodboy/tractor/issues/320'
+#     )
+# )
+
+
+@pytest.fixture
+def spawn(
+    start_method,
+    testdir,
+    arb_addr,
+) -> 'pexpect.spawn':
+
+    if start_method != 'trio':
+        pytest.skip(
+            "Debugger tests are only supported on the trio backend"
+        )
+
+    def _spawn(cmd):
+        return testdir.spawn(
+            cmd=mk_cmd(cmd),
+            expect_timeout=3,
+        )
+
+    return _spawn
+
+
+PROMPT = r"\(Pdb\+\)"
+
+
+def expect(
+    child,
+
+    # prompt by default
+    patt: str = PROMPT,
+
+    **kwargs,
+
+) -> None:
+    '''
+    Expect wrapper that prints last seen console
+    data before failing.
+
+    '''
+    try:
+        child.expect(
+            patt,
+            **kwargs,
+        )
+    except TIMEOUT:
+        before = str(child.before.decode())
+        print(before)
+        raise
+
+
+def assert_before(
+    child,
+    patts: list[str],
+
+) -> None:
+
+    before = str(child.before.decode())
+
+    for patt in patts:
+        try:
+            assert patt in before
+        except AssertionError:
+            print(before)
+            raise
+
+
+@pytest.fixture(
+    params=[False, True],
+    ids='ctl-c={}'.format,
+)
+def ctlc(
+    request,
+    ci_env: bool,
+
+) -> bool:
+
+    use_ctlc = request.param
+
+    node = request.node
+    markers = node.own_markers
+    for mark in markers:
+        if mark.name == 'has_nested_actors':
+            pytest.skip(
+                f'Test {node} has nested actors and fails with Ctrl-C.\n'
+                f'The test can sometimes run fine locally but until'
+                ' we solve' 'this issue this CI test will be xfail:\n'
+                'https://github.com/goodboy/tractor/issues/320'
+            )
+
+    if use_ctlc:
+        # XXX: disable pygments highlighting for auto-tests
+        # since some envs (like actions CI) will struggle
+        # the the added color-char encoding..
+        from tractor._debug import TractorConfig
+        TractorConfig.use_pygements = False
+
+    yield use_ctlc
+
+
+@pytest.mark.parametrize(
+    'user_in_out',
+    [
+        ('c', 'AssertionError'),
+        ('q', 'AssertionError'),
+    ],
+    ids=lambda item: f'{item[0]} -> {item[1]}',
+)
+def test_root_actor_error(spawn, user_in_out):
+    '''
+    Demonstrate crash handler entering pdb from basic error in root actor.
+
+    '''
+    user_input, expect_err_str = user_in_out
+
+    child = spawn('root_actor_error')
+
+    # scan for the prompt
+    expect(child, PROMPT)
+
+    before = str(child.before.decode())
+
+    # make sure expected logging and error arrives
+    assert "Attaching to pdb in crashed actor: ('root'" in before
+    assert 'AssertionError' in before
+
+    # send user command
+    child.sendline(user_input)
+
+    # process should exit
+    expect(child, EOF)
+    assert expect_err_str in str(child.before)
+
+
+@pytest.mark.parametrize(
+    'user_in_out',
+    [
+        ('c', None),
+        ('q', 'bdb.BdbQuit'),
+    ],
+    ids=lambda item: f'{item[0]} -> {item[1]}',
+)
+def test_root_actor_bp(spawn, user_in_out):
+    """Demonstrate breakpoint from in root actor.
+    """
+    user_input, expect_err_str = user_in_out
+    child = spawn('root_actor_breakpoint')
+
+    # scan for the prompt
+    child.expect(PROMPT)
+
+    assert 'Error' not in str(child.before)
+
+    # send user command
+    child.sendline(user_input)
+    child.expect('\r\n')
+
+    # process should exit
+    child.expect(pexpect.EOF)
+
+    if expect_err_str is None:
+        assert 'Error' not in str(child.before)
+    else:
+        assert expect_err_str in str(child.before)
+
+
+def do_ctlc(
+    child,
+    count: int = 3,
+    delay: float = 0.1,
+    patt: Optional[str] = None,
+
+    # expect repl UX to reprint the prompt after every
+    # ctrl-c send.
+    # XXX: no idea but, in CI this never seems to work even on 3.10 so
+    # needs some further investigation potentially...
+    expect_prompt: bool = not _ci_env,
+
+) -> None:
+
+    # make sure ctl-c sends don't do anything but repeat output
+    for _ in range(count):
+        time.sleep(delay)
+        child.sendcontrol('c')
+
+        # TODO: figure out why this makes CI fail..
+        # if you run this test manually it works just fine..
+        if expect_prompt:
+            before = str(child.before.decode())
+            time.sleep(delay)
+            child.expect(PROMPT)
+            time.sleep(delay)
+
+            if patt:
+                # should see the last line on console
+                assert patt in before
+
+
+def test_root_actor_bp_forever(
+    spawn,
+    ctlc: bool,
+):
+    "Re-enter a breakpoint from the root actor-task."
+    child = spawn('root_actor_breakpoint_forever')
+
+    # do some "next" commands to demonstrate recurrent breakpoint
+    # entries
+    for _ in range(10):
+
+        child.expect(PROMPT)
+
+        if ctlc:
+            do_ctlc(child)
+
+        child.sendline('next')
+
+    # do one continue which should trigger a
+    # new task to lock the tty
+    child.sendline('continue')
+    child.expect(PROMPT)
+
+    # seems that if we hit ctrl-c too fast the
+    # sigint guard machinery might not kick in..
+    time.sleep(0.001)
+
+    if ctlc:
+        do_ctlc(child)
+
+    # XXX: this previously caused a bug!
+    child.sendline('n')
+    child.expect(PROMPT)
+
+    child.sendline('n')
+    child.expect(PROMPT)
+
+    # quit out of the loop
+    child.sendline('q')
+    child.expect(pexpect.EOF)
+
+
+@pytest.mark.parametrize(
+    'do_next',
+    (True, False),
+    ids='do_next={}'.format,
+)
+def test_subactor_error(
+    spawn,
+    ctlc: bool,
+    do_next: bool,
+):
+    '''
+    Single subactor raising an error
+
+    '''
+    child = spawn('subactor_error')
+
+    # scan for the prompt
+    child.expect(PROMPT)
+
+    before = str(child.before.decode())
+    assert "Attaching to pdb in crashed actor: ('name_error'" in before
+
+    if do_next:
+        child.sendline('n')
+
+    else:
+        # make sure ctl-c sends don't do anything but repeat output
+        if ctlc:
+            do_ctlc(
+                child,
+            )
+
+        # send user command and (in this case it's the same for 'continue'
+        # vs. 'quit') the debugger should enter a second time in the nursery
+        # creating actor
+        child.sendline('continue')
+
+    child.expect(PROMPT)
+    before = str(child.before.decode())
+
+    # root actor gets debugger engaged
+    assert "Attaching to pdb in crashed actor: ('root'" in before
+    # error is a remote error propagated from the subactor
+    assert "RemoteActorError: ('name_error'" in before
+
+    # another round
+    if ctlc:
+        do_ctlc(child)
+
+    child.sendline('c')
+    child.expect('\r\n')
+
+    # process should exit
+    child.expect(pexpect.EOF)
+
+
+def test_subactor_breakpoint(
+    spawn,
+    ctlc: bool,
+):
+    "Single subactor with an infinite breakpoint loop"
+
+    child = spawn('subactor_breakpoint')
+
+    # scan for the prompt
+    child.expect(PROMPT)
+
+    before = str(child.before.decode())
+    assert "Attaching pdb to actor: ('breakpoint_forever'" in before
+
+    # do some "next" commands to demonstrate recurrent breakpoint
+    # entries
+    for _ in range(10):
+        child.sendline('next')
+        child.expect(PROMPT)
+
+        if ctlc:
+            do_ctlc(child)
+
+    # now run some "continues" to show re-entries
+    for _ in range(5):
+        child.sendline('continue')
+        child.expect(PROMPT)
+        before = str(child.before.decode())
+        assert "Attaching pdb to actor: ('breakpoint_forever'" in before
+
+        if ctlc:
+            do_ctlc(child)
+
+    # finally quit the loop
+    child.sendline('q')
+
+    # child process should exit but parent will capture pdb.BdbQuit
+    child.expect(PROMPT)
+
+    before = str(child.before.decode())
+    assert "RemoteActorError: ('breakpoint_forever'" in before
+    assert 'bdb.BdbQuit' in before
+
+    if ctlc:
+        do_ctlc(child)
+
+    # quit the parent
+    child.sendline('c')
+
+    # process should exit
+    child.expect(pexpect.EOF)
+
+    before = str(child.before.decode())
+    assert "RemoteActorError: ('breakpoint_forever'" in before
+    assert 'bdb.BdbQuit' in before
+
+
+@has_nested_actors
+def test_multi_subactors(
+    spawn,
+    ctlc: bool,
+):
+    '''
+    Multiple subactors, both erroring and
+    breakpointing as well as a nested subactor erroring.
+
+    '''
+    child = spawn(r'multi_subactors')
+
+    # scan for the prompt
+    child.expect(PROMPT)
+
+    before = str(child.before.decode())
+    assert "Attaching pdb to actor: ('breakpoint_forever'" in before
+
+    if ctlc:
+        do_ctlc(child)
+
+    # do some "next" commands to demonstrate recurrent breakpoint
+    # entries
+    for _ in range(10):
+        child.sendline('next')
+        child.expect(PROMPT)
+
+        if ctlc:
+            do_ctlc(child)
+
+    # continue to next error
+    child.sendline('c')
+
+    # first name_error failure
+    child.expect(PROMPT)
+    before = str(child.before.decode())
+    assert "Attaching to pdb in crashed actor: ('name_error'" in before
+    assert "NameError" in before
+
+    if ctlc:
+        do_ctlc(child)
+
+    # continue again
+    child.sendline('c')
+
+    # 2nd name_error failure
+    child.expect(PROMPT)
+
+    # TODO: will we ever get the race where this crash will show up?
+    # blocklist strat now prevents this crash
+    # assert_before(child, [
+    #     "Attaching to pdb in crashed actor: ('name_error_1'",
+    #     "NameError",
+    # ])
+
+    if ctlc:
+        do_ctlc(child)
+
+    # breakpoint loop should re-engage
+    child.sendline('c')
+    child.expect(PROMPT)
+    before = str(child.before.decode())
+    assert "Attaching pdb to actor: ('breakpoint_forever'" in before
+
+    if ctlc:
+        do_ctlc(child)
+
+    # wait for spawn error to show up
+    spawn_err = "Attaching to pdb in crashed actor: ('spawn_error'"
+    start = time.time()
+    while (
+        spawn_err not in before
+        and (time.time() - start) < 3  # timeout eventually
+    ):
+        child.sendline('c')
+        time.sleep(0.1)
+        child.expect(PROMPT)
+        before = str(child.before.decode())
+
+        if ctlc:
+            do_ctlc(child)
+
+    # 2nd depth nursery should trigger
+    # (XXX: this below if guard is technically a hack that makes the
+    # nested case seem to work locally on linux but ideally in the long
+    # run this can be dropped.)
+    if not ctlc:
+        assert_before(child, [
+            spawn_err,
+            "RemoteActorError: ('name_error_1'",
+        ])
+
+    # now run some "continues" to show re-entries
+    for _ in range(5):
+        child.sendline('c')
+        child.expect(PROMPT)
+
+    # quit the loop and expect parent to attach
+    child.sendline('q')
+    child.expect(PROMPT)
+    before = str(child.before.decode())
+
+    assert_before(child, [
+        # debugger attaches to root
+        "Attaching to pdb in crashed actor: ('root'",
+
+        # expect a multierror with exceptions for each sub-actor
+        "RemoteActorError: ('breakpoint_forever'",
+        "RemoteActorError: ('name_error'",
+        "RemoteActorError: ('spawn_error'",
+        "RemoteActorError: ('name_error_1'",
+        'bdb.BdbQuit',
+    ])
+
+    if ctlc:
+        do_ctlc(child)
+
+    # process should exit
+    child.sendline('c')
+    child.expect(pexpect.EOF)
+
+    # repeat of previous multierror for final output
+    assert_before(child, [
+        "RemoteActorError: ('breakpoint_forever'",
+        "RemoteActorError: ('name_error'",
+        "RemoteActorError: ('spawn_error'",
+        "RemoteActorError: ('name_error_1'",
+        'bdb.BdbQuit',
+    ])
+
+
+def test_multi_daemon_subactors(
+    spawn,
+    loglevel: str,
+    ctlc: bool
+):
+    '''
+    Multiple daemon subactors, both erroring and breakpointing within a
+    stream.
+
+    '''
+    child = spawn('multi_daemon_subactors')
+
+    child.expect(PROMPT)
+
+    # there can be a race for which subactor will acquire
+    # the root's tty lock first so anticipate either crash
+    # message on the first entry.
+
+    bp_forever_msg = "Attaching pdb to actor: ('bp_forever'"
+    name_error_msg = "NameError: name 'doggypants' is not defined"
+
+    before = str(child.before.decode())
+    if bp_forever_msg in before:
+        next_msg = name_error_msg
+
+    elif name_error_msg in before:
+        next_msg = bp_forever_msg
+
+    else:
+        raise ValueError("Neither log msg was found !?")
+
+    if ctlc:
+        do_ctlc(child)
+
+    # NOTE: previously since we did not have clobber prevention
+    # in the root actor this final resume could result in the debugger
+    # tearing down since both child actors would be cancelled and it was
+    # unlikely that `bp_forever` would re-acquire the tty lock again.
+    # Now, we should have a final resumption in the root plus a possible
+    # second entry by `bp_forever`.
+
+    child.sendline('c')
+    child.expect(PROMPT)
+    assert_before(child, [next_msg])
+
+    # XXX: hooray the root clobbering the child here was fixed!
+    # IMO, this demonstrates the true power of SC system design.
+
+    # now the root actor won't clobber the bp_forever child
+    # during it's first access to the debug lock, but will instead
+    # wait for the lock to release, by the edge triggered
+    # ``_debug.Lock.no_remote_has_tty`` event before sending cancel messages
+    # (via portals) to its underlings B)
+
+    # at some point here there should have been some warning msg from
+    # the root announcing it avoided a clobber of the child's lock, but
+    # it seems unreliable in testing here to gnab it:
+    # assert "in use by child ('bp_forever'," in before
+
+    if ctlc:
+        do_ctlc(child)
+
+    # expect another breakpoint actor entry
+    child.sendline('c')
+    child.expect(PROMPT)
+
+    try:
+        assert_before(child, [bp_forever_msg])
+    except AssertionError:
+        assert_before(child, [name_error_msg])
+
+    else:
+        if ctlc:
+            do_ctlc(child)
+
+        # should crash with the 2nd name error (simulates
+        # a retry) and then the root eventually (boxed) errors
+        # after 1 or more further bp actor entries.
+
+        child.sendline('c')
+        child.expect(PROMPT)
+        assert_before(child, [name_error_msg])
+
+    # wait for final error in root
+    # where it crashs with boxed error
+    while True:
+        try:
+            child.sendline('c')
+            child.expect(PROMPT)
+            assert_before(
+                child,
+                [bp_forever_msg]
+            )
+        except AssertionError:
+            break
+
+    assert_before(
+        child,
+        [
+            # boxed error raised in root task
+            "Attaching to pdb in crashed actor: ('root'",
+            "_exceptions.RemoteActorError: ('name_error'",
+        ]
+    )
+
+    child.sendline('c')
+    child.expect(pexpect.EOF)
+
+
+@has_nested_actors
+def test_multi_subactors_root_errors(
+    spawn,
+    ctlc: bool
+):
+    '''
+    Multiple subactors, both erroring and breakpointing as well as
+    a nested subactor erroring.
+
+    '''
+    child = spawn('multi_subactor_root_errors')
+
+    # scan for the prompt
+    child.expect(PROMPT)
+
+    # at most one subactor should attach before the root is cancelled
+    before = str(child.before.decode())
+    assert "NameError: name 'doggypants' is not defined" in before
+
+    if ctlc:
+        do_ctlc(child)
+
+    # continue again to catch 2nd name error from
+    # actor 'name_error_1' (which is 2nd depth).
+    child.sendline('c')
+
+    # due to block list strat from #337, this will no longer
+    # propagate before the root errors and cancels the spawner sub-tree.
+    child.expect(PROMPT)
+
+    # only if the blocking condition doesn't kick in fast enough
+    before = str(child.before.decode())
+    if "Debug lock blocked for ['name_error_1'" not in before:
+
+        assert_before(child, [
+            "Attaching to pdb in crashed actor: ('name_error_1'",
+            "NameError",
+        ])
+
+        if ctlc:
+            do_ctlc(child)
+
+        child.sendline('c')
+        child.expect(PROMPT)
+
+    # check if the spawner crashed or was blocked from debug
+    # and if this intermediary attached check the boxed error
+    before = str(child.before.decode())
+    if "Attaching to pdb in crashed actor: ('spawn_error'" in before:
+
+        assert_before(child, [
+            # boxed error from spawner's child
+            "RemoteActorError: ('name_error_1'",
+            "NameError",
+        ])
+
+        if ctlc:
+            do_ctlc(child)
+
+        child.sendline('c')
+        child.expect(PROMPT)
+
+    # expect a root actor crash
+    assert_before(child, [
+        "RemoteActorError: ('name_error'",
+        "NameError",
+
+        # error from root actor and root task that created top level nursery
+        "Attaching to pdb in crashed actor: ('root'",
+        "AssertionError",
+    ])
+
+    child.sendline('c')
+    child.expect(pexpect.EOF)
+
+    assert_before(child, [
+        # "Attaching to pdb in crashed actor: ('root'",
+        # boxed error from previous step
+        "RemoteActorError: ('name_error'",
+        "NameError",
+        "AssertionError",
+        'assert 0',
+    ])
+
+
+@has_nested_actors
+def test_multi_nested_subactors_error_through_nurseries(
+    spawn,
+
+    # TODO: address debugger issue for nested tree:
+    # https://github.com/goodboy/tractor/issues/320
+    # ctlc: bool,
+):
+    """Verify deeply nested actors that error trigger debugger entries
+    at each actor nurserly (level) all the way up the tree.
+
+    """
+    # NOTE: previously, inside this script was a bug where if the
+    # parent errors before a 2-levels-lower actor has released the lock,
+    # the parent tries to cancel it but it's stuck in the debugger?
+    # A test (below) has now been added to explicitly verify this is
+    # fixed.
+
+    child = spawn('multi_nested_subactors_error_up_through_nurseries')
+
+    timed_out_early: bool = False
+
+    for send_char in itertools.cycle(['c', 'q']):
+        try:
+            child.expect(PROMPT)
+            child.sendline(send_char)
+            time.sleep(0.01)
+
+        except EOF:
+            break
+
+    assert_before(child, [
+
+        # boxed source errors
+        "NameError: name 'doggypants' is not defined",
+        "tractor._exceptions.RemoteActorError: ('name_error'",
+        "bdb.BdbQuit",
+
+        # first level subtrees
+        "tractor._exceptions.RemoteActorError: ('spawner0'",
+        # "tractor._exceptions.RemoteActorError: ('spawner1'",
+
+        # propagation of errors up through nested subtrees
+        "tractor._exceptions.RemoteActorError: ('spawn_until_0'",
+        "tractor._exceptions.RemoteActorError: ('spawn_until_1'",
+        "tractor._exceptions.RemoteActorError: ('spawn_until_2'",
+    ])
+
+
+@pytest.mark.timeout(15)
+@has_nested_actors
+def test_root_nursery_cancels_before_child_releases_tty_lock(
+    spawn,
+    start_method,
+    ctlc: bool,
+):
+    '''
+    Test that when the root sends a cancel message before a nested child
+    has unblocked (which can happen when it has the tty lock and is
+    engaged in pdb) it is indeed cancelled after exiting the debugger.
+
+    '''
+    timed_out_early = False
+
+    child = spawn('root_cancelled_but_child_is_in_tty_lock')
+
+    child.expect(PROMPT)
+
+    before = str(child.before.decode())
+    assert "NameError: name 'doggypants' is not defined" in before
+    assert "tractor._exceptions.RemoteActorError: ('name_error'" not in before
+    time.sleep(0.5)
+
+    if ctlc:
+        do_ctlc(child)
+
+    child.sendline('c')
+
+    for i in range(4):
+        time.sleep(0.5)
+        try:
+            child.expect(PROMPT)
+
+        except (
+            EOF,
+            TIMEOUT,
+        ):
+            # races all over..
+
+            print(f"Failed early on {i}?")
+            before = str(child.before.decode())
+
+            timed_out_early = True
+
+            # race conditions on how fast the continue is sent?
+            break
+
+        before = str(child.before.decode())
+        assert "NameError: name 'doggypants' is not defined" in before
+
+        if ctlc:
+            do_ctlc(child)
+
+        child.sendline('c')
+        time.sleep(0.1)
+
+    for i in range(3):
+        try:
+            child.expect(pexpect.EOF, timeout=0.5)
+            break
+        except TIMEOUT:
+            child.sendline('c')
+            time.sleep(0.1)
+            print('child was able to grab tty lock again?')
+    else:
+        print('giving up on child releasing, sending `quit` cmd')
+        child.sendline('q')
+        expect(child, EOF)
+
+    if not timed_out_early:
+        before = str(child.before.decode())
+        assert_before(child, [
+            "tractor._exceptions.RemoteActorError: ('spawner0'",
+            "tractor._exceptions.RemoteActorError: ('name_error'",
+            "NameError: name 'doggypants' is not defined",
+        ])
+
+
+def test_root_cancels_child_context_during_startup(
+    spawn,
+    ctlc: bool,
+):
+    '''Verify a fast fail in the root doesn't lock up the child reaping
+    and all while using the new context api.
+
+    '''
+    child = spawn('fast_error_in_root_after_spawn')
+
+    child.expect(PROMPT)
+
+    before = str(child.before.decode())
+    assert "AssertionError" in before
+
+    if ctlc:
+        do_ctlc(child)
+
+    child.sendline('c')
+    child.expect(pexpect.EOF)
+
+
+def test_different_debug_mode_per_actor(
+    spawn,
+    ctlc: bool,
+):
+    child = spawn('per_actor_debug')
+    child.expect(PROMPT)
+
+    # only one actor should enter the debugger
+    before = str(child.before.decode())
+    assert "Attaching to pdb in crashed actor: ('debugged_boi'" in before
+    assert "RuntimeError" in before
+
+    if ctlc:
+        do_ctlc(child)
+
+    child.sendline('c')
+    child.expect(pexpect.EOF)
+
+    before = str(child.before.decode())
+
+    # NOTE: this debugged actor error currently WON'T show up since the
+    # root will actually cancel and terminate the nursery before the error
+    # msg reported back from the debug mode actor is processed.
+    # assert "tractor._exceptions.RemoteActorError: ('debugged_boi'" in before
+
+    assert "tractor._exceptions.RemoteActorError: ('crash_boi'" in before
+
+    # the crash boi should not have made a debugger request but
+    # instead crashed completely
+    assert "tractor._exceptions.RemoteActorError: ('crash_boi'" in before
+    assert "RuntimeError" in before

tests/test_discovery.py

@@ -1,6 +1,12 @@
 """
 Actor "discovery" testing
 """
+import os
+import signal
+import platform
+from functools import partial
+import itertools
+
 import pytest
 import tractor
 import trio
@@ -14,26 +20,29 @@ async def test_reg_then_unreg(arb_addr):
     assert actor.is_arbiter
     assert len(actor._registry) == 1  # only self is registered
 
-    async with tractor.open_nursery() as n:
-        portal = await n.start_actor('actor', rpc_module_paths=[__name__])
+    async with tractor.open_nursery(
+        arbiter_addr=arb_addr,
+    ) as n:
+
+        portal = await n.start_actor('actor', enable_modules=[__name__])
         uid = portal.channel.uid
 
         async with tractor.get_arbiter(*arb_addr) as aportal:
-            # local actor should be the arbiter
+            # this local actor should be the arbiter
             assert actor is aportal.actor
 
-            # sub-actor uid should be in the registry
-            await trio.sleep(0.1)  # registering is async, so..
-            assert uid in aportal.actor._registry
-            sockaddrs = actor._registry[uid]
-            # XXX: can we figure out what the listen addr will be?
-            assert sockaddrs
+            async with tractor.wait_for_actor('actor'):
+                # sub-actor uid should be in the registry
+                assert uid in aportal.actor._registry
+                sockaddrs = actor._registry[uid]
+                # XXX: can we figure out what the listen addr will be?
+                assert sockaddrs
 
         await n.cancel()  # tear down nursery
 
         await trio.sleep(0.1)
         assert uid not in aportal.actor._registry
-        sockaddrs = actor._registry[uid]
+        sockaddrs = actor._registry.get(uid)
         assert not sockaddrs
 
 
@@ -45,20 +54,22 @@ async def hi():
 
 
 async def say_hello(other_actor):
-    await trio.sleep(0.4)  # wait for other actor to spawn
+    await trio.sleep(1)  # wait for other actor to spawn
     async with tractor.find_actor(other_actor) as portal:
+        assert portal is not None
         return await portal.run(__name__, 'hi')
 
 
 async def say_hello_use_wait(other_actor):
     async with tractor.wait_for_actor(other_actor) as portal:
+        assert portal is not None
         result = await portal.run(__name__, 'hi')
         return result
 
 
 @tractor_test
 @pytest.mark.parametrize('func', [say_hello, say_hello_use_wait])
-async def test_trynamic_trio(func):
+async def test_trynamic_trio(func, start_method, arb_addr):
     """Main tractor entry point, the "master" process (for now
     acts as the "director").
     """
@@ -66,15 +77,292 @@ async def test_trynamic_trio(func):
         print("Alright... Action!")
 
         donny = await n.run_in_actor(
-            'donny',
             func,
             other_actor='gretchen',
+            name='donny',
         )
         gretchen = await n.run_in_actor(
-            'gretchen',
             func,
             other_actor='donny',
+            name='gretchen',
         )
         print(await gretchen.result())
         print(await donny.result())
         print("CUTTTT CUUTT CUT!!?! Donny!! You're supposed to say...")
+
+
+async def stream_forever():
+    for i in itertools.count():
+        yield i
+        await trio.sleep(0.01)
+
+
+async def cancel(use_signal, delay=0):
+    # hold on there sally
+    await trio.sleep(delay)
+
+    # trigger cancel
+    if use_signal:
+        if platform.system() == 'Windows':
+            pytest.skip("SIGINT not supported on windows")
+        os.kill(os.getpid(), signal.SIGINT)
+    else:
+        raise KeyboardInterrupt
+
+
+async def stream_from(portal):
+    async with portal.open_stream_from(stream_forever) as stream:
+        async for value in stream:
+            print(value)
+
+
+async def unpack_reg(actor_or_portal):
+    '''
+    Get and unpack a "registry" RPC request from the "arbiter" registry
+    system.
+
+    '''
+    if getattr(actor_or_portal, 'get_registry', None):
+        msg = await actor_or_portal.get_registry()
+    else:
+        msg = await actor_or_portal.run_from_ns('self', 'get_registry')
+
+    return {tuple(key.split('.')): val for key, val in msg.items()}
+
+
+async def spawn_and_check_registry(
+    arb_addr: tuple,
+    use_signal: bool,
+    remote_arbiter: bool = False,
+    with_streaming: bool = False,
+
+) -> None:
+
+    async with tractor.open_root_actor(
+        arbiter_addr=arb_addr,
+    ):
+        async with tractor.get_arbiter(*arb_addr) as portal:
+            # runtime needs to be up to call this
+            actor = tractor.current_actor()
+
+            if remote_arbiter:
+                assert not actor.is_arbiter
+
+            if actor.is_arbiter:
+                extra = 1  # arbiter is local root actor
+                get_reg = partial(unpack_reg, actor)
+
+            else:
+                get_reg = partial(unpack_reg, portal)
+                extra = 2  # local root actor + remote arbiter
+
+            # ensure current actor is registered
+            registry = await get_reg()
+            assert actor.uid in registry
+
+            try:
+                async with tractor.open_nursery() as n:
+                    async with trio.open_nursery() as trion:
+
+                        portals = {}
+                        for i in range(3):
+                            name = f'a{i}'
+                            if with_streaming:
+                                portals[name] = await n.start_actor(
+                                    name=name, enable_modules=[__name__])
+
+                            else:  # no streaming
+                                portals[name] = await n.run_in_actor(
+                                    trio.sleep_forever, name=name)
+
+                        # wait on last actor to come up
+                        async with tractor.wait_for_actor(name):
+                            registry = await get_reg()
+                            for uid in n._children:
+                                assert uid in registry
+
+                        assert len(portals) + extra == len(registry)
+
+                        if with_streaming:
+                            await trio.sleep(0.1)
+
+                            pts = list(portals.values())
+                            for p in pts[:-1]:
+                                trion.start_soon(stream_from, p)
+
+                            # stream for 1 sec
+                            trion.start_soon(cancel, use_signal, 1)
+
+                            last_p = pts[-1]
+                            await stream_from(last_p)
+
+                        else:
+                            await cancel(use_signal)
+
+            finally:
+                await trio.sleep(0.5)
+
+                # all subactors should have de-registered
+                registry = await get_reg()
+                assert len(registry) == extra
+                assert actor.uid in registry
+
+
+@pytest.mark.parametrize('use_signal', [False, True])
+@pytest.mark.parametrize('with_streaming', [False, True])
+def test_subactors_unregister_on_cancel(
+    start_method,
+    use_signal,
+    arb_addr,
+    with_streaming,
+):
+    """Verify that cancelling a nursery results in all subactors
+    deregistering themselves with the arbiter.
+    """
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(
+            partial(
+                spawn_and_check_registry,
+                arb_addr,
+                use_signal,
+                remote_arbiter=False,
+                with_streaming=with_streaming,
+            ),
+        )
+
+
+@pytest.mark.parametrize('use_signal', [False, True])
+@pytest.mark.parametrize('with_streaming', [False, True])
+def test_subactors_unregister_on_cancel_remote_daemon(
+    daemon,
+    start_method,
+    use_signal,
+    arb_addr,
+    with_streaming,
+):
+    """Verify that cancelling a nursery results in all subactors
+    deregistering themselves with a **remote** (not in the local process
+    tree) arbiter.
+    """
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(
+            partial(
+                spawn_and_check_registry,
+                arb_addr,
+                use_signal,
+                remote_arbiter=True,
+                with_streaming=with_streaming,
+            ),
+        )
+
+
+async def streamer(agen):
+    async for item in agen:
+        print(item)
+
+
+async def close_chans_before_nursery(
+    arb_addr: tuple,
+    use_signal: bool,
+    remote_arbiter: bool = False,
+) -> None:
+
+    # logic for how many actors should still be
+    # in the registry at teardown.
+    if remote_arbiter:
+        entries_at_end = 2
+    else:
+        entries_at_end = 1
+
+    async with tractor.open_root_actor(
+        arbiter_addr=arb_addr,
+    ):
+        async with tractor.get_arbiter(*arb_addr) as aportal:
+            try:
+                get_reg = partial(unpack_reg, aportal)
+
+                async with tractor.open_nursery() as tn:
+                    portal1 = await tn.start_actor(
+                        name='consumer1', enable_modules=[__name__])
+                    portal2 = await tn.start_actor(
+                        'consumer2', enable_modules=[__name__])
+
+                    # TODO: compact this back as was in last commit once
+                    # 3.9+, see https://github.com/goodboy/tractor/issues/207
+                    async with portal1.open_stream_from(
+                        stream_forever
+                    ) as agen1:
+                        async with portal2.open_stream_from(
+                            stream_forever
+                        ) as agen2:
+                            async with trio.open_nursery() as n:
+                                n.start_soon(streamer, agen1)
+                                n.start_soon(cancel, use_signal, .5)
+                                try:
+                                    await streamer(agen2)
+                                finally:
+                                    # Kill the root nursery thus resulting in
+                                    # normal arbiter channel ops to fail during
+                                    # teardown. It doesn't seem like this is
+                                    # reliably triggered by an external SIGINT.
+                                    # tractor.current_actor()._root_nursery.cancel_scope.cancel()
+
+                                    # XXX: THIS IS THE KEY THING that
+                                    # happens **before** exiting the
+                                    # actor nursery block
+
+                                    # also kill off channels cuz why not
+                                    await agen1.aclose()
+                                    await agen2.aclose()
+            finally:
+                with trio.CancelScope(shield=True):
+                    await trio.sleep(1)
+
+                    # all subactors should have de-registered
+                    registry = await get_reg()
+                    assert portal1.channel.uid not in registry
+                    assert portal2.channel.uid not in registry
+                    assert len(registry) == entries_at_end
+
+
+@pytest.mark.parametrize('use_signal', [False, True])
+def test_close_channel_explicit(
+    start_method,
+    use_signal,
+    arb_addr,
+):
+    """Verify that closing a stream explicitly and killing the actor's
+    "root nursery" **before** the containing nursery tears down also
+    results in subactor(s) deregistering from the arbiter.
+    """
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(
+            partial(
+                close_chans_before_nursery,
+                arb_addr,
+                use_signal,
+                remote_arbiter=False,
+            ),
+        )
+
+
+@pytest.mark.parametrize('use_signal', [False, True])
+def test_close_channel_explicit_remote_arbiter(
+    daemon,
+    start_method,
+    use_signal,
+    arb_addr,
+):
+    """Verify that closing a stream explicitly and killing the actor's
+    "root nursery" **before** the containing nursery tears down also
+    results in subactor(s) deregistering from the arbiter.
+    """
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(
+            partial(
+                close_chans_before_nursery,
+                arb_addr,
+                use_signal,
+                remote_arbiter=True,
+            ),
+        )

tests/test_docs_examples.py

@@ -0,0 +1,135 @@
+'''
+Let's make sure them docs work yah?
+
+'''
+from contextlib import contextmanager
+import itertools
+import os
+import sys
+import subprocess
+import platform
+import shutil
+
+import pytest
+
+from conftest import (
+    examples_dir,
+)
+
+
+@pytest.fixture
+def run_example_in_subproc(
+    loglevel: str,
+    testdir,
+    arb_addr: tuple[str, int],
+):
+
+    @contextmanager
+    def run(script_code):
+        kwargs = dict()
+
+        if platform.system() == 'Windows':
+            # on windows we need to create a special __main__.py which will
+            # be executed with ``python -m <modulename>`` on windows..
+            shutil.copyfile(
+                examples_dir() / '__main__.py',
+                str(testdir / '__main__.py'),
+            )
+
+            # drop the ``if __name__ == '__main__'`` guard onwards from
+            # the *NIX version of each script
+            windows_script_lines = itertools.takewhile(
+                lambda line: "if __name__ ==" not in line,
+                script_code.splitlines()
+            )
+            script_code = '\n'.join(windows_script_lines)
+            script_file = testdir.makefile('.py', script_code)
+
+            # without this, tests hang on windows forever
+            kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
+
+            # run the testdir "libary module" as a script
+            cmdargs = [
+                sys.executable,
+                '-m',
+                # use the "module name" of this "package"
+                'test_example'
+            ]
+        else:
+            script_file = testdir.makefile('.py', script_code)
+            cmdargs = [
+                sys.executable,
+                str(script_file),
+            ]
+
+        # XXX: BE FOREVER WARNED: if you enable lots of tractor logging
+        # in the subprocess it may cause infinite blocking on the pipes
+        # due to backpressure!!!
+        proc = testdir.popen(
+            cmdargs,
+            **kwargs,
+        )
+        assert not proc.returncode
+        yield proc
+        proc.wait()
+        assert proc.returncode == 0
+
+    yield run
+
+
+@pytest.mark.parametrize(
+    'example_script',
+
+    # walk yields: (dirpath, dirnames, filenames)
+    [
+        (p[0], f) for p in os.walk(examples_dir()) for f in p[2]
+
+        if '__' not in f
+        and f[0] != '_'
+        and 'debugging' not in p[0]
+        and 'integration' not in p[0]
+        and 'advanced_faults' not in p[0]
+    ],
+
+    ids=lambda t: t[1],
+)
+def test_example(run_example_in_subproc, example_script):
+    """Load and run scripts from this repo's ``examples/`` dir as a user
+    would copy and pasing them into their editor.
+
+    On windows a little more "finessing" is done to make
+    ``multiprocessing`` play nice: we copy the ``__main__.py`` into the
+    test directory and invoke the script as a module with ``python -m
+    test_example``.
+    """
+    ex_file = os.path.join(*example_script)
+
+    if 'rpc_bidir_streaming' in ex_file and sys.version_info < (3, 9):
+        pytest.skip("2-way streaming example requires py3.9 async with syntax")
+
+    with open(ex_file, 'r') as ex:
+        code = ex.read()
+
+        with run_example_in_subproc(code) as proc:
+            proc.wait()
+            err, _ = proc.stderr.read(), proc.stdout.read()
+            # print(f'STDERR: {err}')
+            # print(f'STDOUT: {out}')
+
+            # if we get some gnarly output let's aggregate and raise
+            if err:
+                errmsg = err.decode()
+                errlines = errmsg.splitlines()
+                last_error = errlines[-1]
+                if (
+                    'Error' in last_error
+
+                    # XXX: currently we print this to console, but maybe
+                    # shouldn't eventually once we figure out what's
+                    # a better way to be explicit about aio side
+                    # cancels?
+                    and 'asyncio.exceptions.CancelledError' not in last_error
+                ):
+                    raise Exception(errmsg)
+
+            assert proc.returncode == 0

tests/test_infected_asyncio.py

@@ -0,0 +1,564 @@
+'''
+The hipster way to force SC onto the stdlib's "async": 'infection mode'.
+
+'''
+from typing import Optional, Iterable, Union
+import asyncio
+import builtins
+import itertools
+import importlib
+
+from exceptiongroup import BaseExceptionGroup
+import pytest
+import trio
+import tractor
+from tractor import (
+    to_asyncio,
+    RemoteActorError,
+)
+from tractor.trionics import BroadcastReceiver
+
+
+async def sleep_and_err(
+    sleep_for: float = 0.1,
+
+    # just signature placeholders for compat with
+    # ``to_asyncio.open_channel_from()``
+    to_trio: Optional[trio.MemorySendChannel] = None,
+    from_trio: Optional[asyncio.Queue] = None,
+
+):
+    if to_trio:
+        to_trio.send_nowait('start')
+
+    await asyncio.sleep(sleep_for)
+    assert 0
+
+
+async def sleep_forever():
+    await asyncio.sleep(float('inf'))
+
+
+async def trio_cancels_single_aio_task():
+
+    # spawn an ``asyncio`` task to run a func and return result
+    with trio.move_on_after(.2):
+        await tractor.to_asyncio.run_task(sleep_forever)
+
+
+def test_trio_cancels_aio_on_actor_side(arb_addr):
+    '''
+    Spawn an infected actor that is cancelled by the ``trio`` side
+    task using std cancel scope apis.
+
+    '''
+    async def main():
+        async with tractor.open_nursery(
+            arbiter_addr=arb_addr
+        ) as n:
+            await n.run_in_actor(
+                trio_cancels_single_aio_task,
+                infect_asyncio=True,
+            )
+
+    trio.run(main)
+
+
+async def asyncio_actor(
+
+    target: str,
+    expect_err: Optional[Exception] = None
+
+) -> None:
+
+    assert tractor.current_actor().is_infected_aio()
+    target = globals()[target]
+
+    if '.' in expect_err:
+        modpath, _, name = expect_err.rpartition('.')
+        mod = importlib.import_module(modpath)
+        error_type = getattr(mod, name)
+
+    else:  # toplevel builtin error type
+        error_type = builtins.__dict__.get(expect_err)
+
+    try:
+        # spawn an ``asyncio`` task to run a func and return result
+        await tractor.to_asyncio.run_task(target)
+
+    except BaseException as err:
+        if expect_err:
+            assert isinstance(err, error_type)
+
+        raise
+
+
+def test_aio_simple_error(arb_addr):
+    '''
+    Verify a simple remote asyncio error propagates back through trio
+    to the parent actor.
+
+
+    '''
+    async def main():
+        async with tractor.open_nursery(
+            arbiter_addr=arb_addr
+        ) as n:
+            await n.run_in_actor(
+                asyncio_actor,
+                target='sleep_and_err',
+                expect_err='AssertionError',
+                infect_asyncio=True,
+            )
+
+    with pytest.raises(RemoteActorError) as excinfo:
+        trio.run(main)
+
+    err = excinfo.value
+    assert isinstance(err, RemoteActorError)
+    assert err.type == AssertionError
+
+
+def test_tractor_cancels_aio(arb_addr):
+    '''
+    Verify we can cancel a spawned asyncio task gracefully.
+
+    '''
+    async def main():
+        async with tractor.open_nursery() as n:
+            portal = await n.run_in_actor(
+                asyncio_actor,
+                target='sleep_forever',
+                expect_err='trio.Cancelled',
+                infect_asyncio=True,
+            )
+            # cancel the entire remote runtime
+            await portal.cancel_actor()
+
+    trio.run(main)
+
+
+def test_trio_cancels_aio(arb_addr):
+    '''
+    Much like the above test with ``tractor.Portal.cancel_actor()``
+    except we just use a standard ``trio`` cancellation api.
+
+    '''
+    async def main():
+
+        with trio.move_on_after(1):
+            # cancel the nursery shortly after boot
+
+            async with tractor.open_nursery() as n:
+                await n.run_in_actor(
+                    asyncio_actor,
+                    target='sleep_forever',
+                    expect_err='trio.Cancelled',
+                    infect_asyncio=True,
+                )
+
+    trio.run(main)
+
+
+@tractor.context
+async def trio_ctx(
+    ctx: tractor.Context,
+):
+
+    await ctx.started('start')
+
+    # this will block until the ``asyncio`` task sends a "first"
+    # message.
+    with trio.fail_after(2):
+        async with (
+            trio.open_nursery() as n,
+
+            tractor.to_asyncio.open_channel_from(
+                sleep_and_err,
+            ) as (first, chan),
+        ):
+
+            assert first == 'start'
+
+            # spawn another asyncio task for the cuck of it.
+            n.start_soon(
+                tractor.to_asyncio.run_task,
+                sleep_forever,
+            )
+            await trio.sleep_forever()
+
+
+@pytest.mark.parametrize(
+    'parent_cancels', [False, True],
+    ids='parent_actor_cancels_child={}'.format
+)
+def test_context_spawns_aio_task_that_errors(
+    arb_addr,
+    parent_cancels: bool,
+):
+    '''
+    Verify that spawning a task via an intertask channel ctx mngr that
+    errors correctly propagates the error back from the `asyncio`-side
+    task.
+
+    '''
+    async def main():
+
+        with trio.fail_after(2):
+            async with tractor.open_nursery() as n:
+                p = await n.start_actor(
+                    'aio_daemon',
+                    enable_modules=[__name__],
+                    infect_asyncio=True,
+                    # debug_mode=True,
+                    loglevel='cancel',
+                )
+                async with p.open_context(
+                    trio_ctx,
+                ) as (ctx, first):
+
+                    assert first == 'start'
+
+                    if parent_cancels:
+                        await p.cancel_actor()
+
+                    await trio.sleep_forever()
+
+    with pytest.raises(RemoteActorError) as excinfo:
+        trio.run(main)
+
+    err = excinfo.value
+    assert isinstance(err, RemoteActorError)
+    if parent_cancels:
+        assert err.type == trio.Cancelled
+    else:
+        assert err.type == AssertionError
+
+
+async def aio_cancel():
+    ''''
+    Cancel urself boi.
+
+    '''
+    await asyncio.sleep(0.5)
+    task = asyncio.current_task()
+
+    # cancel and enter sleep
+    task.cancel()
+    await sleep_forever()
+
+
+def test_aio_cancelled_from_aio_causes_trio_cancelled(arb_addr):
+
+    async def main():
+        async with tractor.open_nursery() as n:
+            await n.run_in_actor(
+                asyncio_actor,
+                target='aio_cancel',
+                expect_err='tractor.to_asyncio.AsyncioCancelled',
+                infect_asyncio=True,
+            )
+
+    with pytest.raises(RemoteActorError) as excinfo:
+        trio.run(main)
+
+    # ensure boxed error is correct
+    assert excinfo.value.type == to_asyncio.AsyncioCancelled
+
+
+# TODO: verify open_channel_from will fail on this..
+async def no_to_trio_in_args():
+    pass
+
+
+async def push_from_aio_task(
+
+    sequence: Iterable,
+    to_trio: trio.abc.SendChannel,
+    expect_cancel: False,
+    fail_early: bool,
+
+) -> None:
+
+    try:
+        # sync caller ctx manager
+        to_trio.send_nowait(True)
+
+        for i in sequence:
+            print(f'asyncio sending {i}')
+            to_trio.send_nowait(i)
+            await asyncio.sleep(0.001)
+
+            if i == 50 and fail_early:
+                raise Exception
+
+        print('asyncio streamer complete!')
+
+    except asyncio.CancelledError:
+        if not expect_cancel:
+            pytest.fail("aio task was cancelled unexpectedly")
+        raise
+    else:
+        if expect_cancel:
+            pytest.fail("aio task wasn't cancelled as expected!?")
+
+
+async def stream_from_aio(
+
+    exit_early: bool = False,
+    raise_err: bool = False,
+    aio_raise_err: bool = False,
+    fan_out: bool = False,
+
+) -> None:
+    seq = range(100)
+    expect = list(seq)
+
+    try:
+        pulled = []
+
+        async with to_asyncio.open_channel_from(
+            push_from_aio_task,
+            sequence=seq,
+            expect_cancel=raise_err or exit_early,
+            fail_early=aio_raise_err,
+        ) as (first, chan):
+
+            assert first is True
+
+            async def consume(
+                chan: Union[
+                    to_asyncio.LinkedTaskChannel,
+                    BroadcastReceiver,
+                ],
+            ):
+                async for value in chan:
+                    print(f'trio received {value}')
+                    pulled.append(value)
+
+                    if value == 50:
+                        if raise_err:
+                            raise Exception
+                        elif exit_early:
+                            break
+
+            if fan_out:
+                # start second task that get's the same stream value set.
+                async with (
+
+                    # NOTE: this has to come first to avoid
+                    # the channel being closed before the nursery
+                    # tasks are joined..
+                    chan.subscribe() as br,
+
+                    trio.open_nursery() as n,
+                ):
+                    n.start_soon(consume, br)
+                    await consume(chan)
+
+            else:
+                await consume(chan)
+    finally:
+
+        if (
+            not raise_err and
+            not exit_early and
+            not aio_raise_err
+        ):
+            if fan_out:
+                # we get double the pulled values in the
+                # ``.subscribe()`` fan out case.
+                doubled = list(itertools.chain(*zip(expect, expect)))
+                expect = doubled[:len(pulled)]
+                assert list(sorted(pulled)) == expect
+
+            else:
+                assert pulled == expect
+        else:
+            assert not fan_out
+            assert pulled == expect[:51]
+
+        print('trio guest mode task completed!')
+
+
+@pytest.mark.parametrize(
+    'fan_out', [False, True],
+    ids='fan_out_w_chan_subscribe={}'.format
+)
+def test_basic_interloop_channel_stream(arb_addr, fan_out):
+    async def main():
+        async with tractor.open_nursery() as n:
+            portal = await n.run_in_actor(
+                stream_from_aio,
+                infect_asyncio=True,
+                fan_out=fan_out,
+            )
+            await portal.result()
+
+    trio.run(main)
+
+
+# TODO: parametrize the above test and avoid the duplication here?
+def test_trio_error_cancels_intertask_chan(arb_addr):
+    async def main():
+        async with tractor.open_nursery() as n:
+            portal = await n.run_in_actor(
+                stream_from_aio,
+                raise_err=True,
+                infect_asyncio=True,
+            )
+            # should trigger remote actor error
+            await portal.result()
+
+    with pytest.raises(BaseExceptionGroup) as excinfo:
+        trio.run(main)
+
+    # ensure boxed errors
+    for exc in excinfo.value.exceptions:
+        assert exc.type == Exception
+
+
+def test_trio_closes_early_and_channel_exits(arb_addr):
+    async def main():
+        async with tractor.open_nursery() as n:
+            portal = await n.run_in_actor(
+                stream_from_aio,
+                exit_early=True,
+                infect_asyncio=True,
+            )
+            # should trigger remote actor error
+            await portal.result()
+
+    # should be a quiet exit on a simple channel exit
+    trio.run(main)
+
+
+def test_aio_errors_and_channel_propagates_and_closes(arb_addr):
+    async def main():
+        async with tractor.open_nursery() as n:
+            portal = await n.run_in_actor(
+                stream_from_aio,
+                aio_raise_err=True,
+                infect_asyncio=True,
+            )
+            # should trigger remote actor error
+            await portal.result()
+
+    with pytest.raises(BaseExceptionGroup) as excinfo:
+        trio.run(main)
+
+    # ensure boxed errors
+    for exc in excinfo.value.exceptions:
+        assert exc.type == Exception
+
+
+@tractor.context
+async def trio_to_aio_echo_server(
+    ctx: tractor.Context,
+):
+
+    async def aio_echo_server(
+        to_trio: trio.MemorySendChannel,
+        from_trio: asyncio.Queue,
+    ) -> None:
+
+        to_trio.send_nowait('start')
+
+        while True:
+            msg = await from_trio.get()
+
+            # echo the msg back
+            to_trio.send_nowait(msg)
+
+            # if we get the terminate sentinel
+            # break the echo loop
+            if msg is None:
+                print('breaking aio echo loop')
+                break
+
+        print('exiting asyncio task')
+
+    async with to_asyncio.open_channel_from(
+        aio_echo_server,
+    ) as (first, chan):
+
+        assert first == 'start'
+        await ctx.started(first)
+
+        async with ctx.open_stream() as stream:
+
+            async for msg in stream:
+                print(f'asyncio echoing {msg}')
+                await chan.send(msg)
+
+                out = await chan.receive()
+                # echo back to parent actor-task
+                await stream.send(out)
+
+                if out is None:
+                    try:
+                        out = await chan.receive()
+                    except trio.EndOfChannel:
+                        break
+                    else:
+                        raise RuntimeError('aio channel never stopped?')
+
+
+@pytest.mark.parametrize(
+    'raise_error_mid_stream',
+    [False, Exception, KeyboardInterrupt],
+    ids='raise_error={}'.format,
+)
+def test_echoserver_detailed_mechanics(
+    arb_addr,
+    raise_error_mid_stream,
+):
+
+    async def main():
+        async with tractor.open_nursery() as n:
+            p = await n.start_actor(
+                'aio_server',
+                enable_modules=[__name__],
+                infect_asyncio=True,
+            )
+            async with p.open_context(
+                trio_to_aio_echo_server,
+            ) as (ctx, first):
+
+                assert first == 'start'
+
+                async with ctx.open_stream() as stream:
+                    for i in range(100):
+                        await stream.send(i)
+                        out = await stream.receive()
+                        assert i == out
+
+                        if raise_error_mid_stream and i == 50:
+                            raise raise_error_mid_stream
+
+                    # send terminate msg
+                    await stream.send(None)
+                    out = await stream.receive()
+                    assert out is None
+
+                    if out is None:
+                        # ensure the stream is stopped
+                        # with trio.fail_after(0.1):
+                        try:
+                            await stream.receive()
+                        except trio.EndOfChannel:
+                            pass
+                        else:
+                            pytest.fail(
+                                "stream wasn't stopped after sentinel?!")
+
+            # TODO: the case where this blocks and
+            # is cancelled by kbi or out of task cancellation
+            await p.cancel_actor()
+
+    if raise_error_mid_stream:
+        with pytest.raises(raise_error_mid_stream):
+            trio.run(main)
+
+    else:
+        trio.run(main)

tests/test_legacy_one_way_streaming.py

@@ -0,0 +1,352 @@
+"""
+Streaming via async gen api
+"""
+import time
+from functools import partial
+import platform
+
+import trio
+import tractor
+import pytest
+
+from conftest import tractor_test
+
+
+def test_must_define_ctx():
+
+    with pytest.raises(TypeError) as err:
+        @tractor.stream
+        async def no_ctx():
+            pass
+
+    assert "no_ctx must be `ctx: tractor.Context" in str(err.value)
+
+    @tractor.stream
+    async def has_ctx(ctx):
+        pass
+
+
+async def async_gen_stream(sequence):
+    for i in sequence:
+        yield i
+        await trio.sleep(0.1)
+
+    # block indefinitely waiting to be cancelled by ``aclose()`` call
+    with trio.CancelScope() as cs:
+        await trio.sleep_forever()
+        assert 0
+    assert cs.cancelled_caught
+
+
+@tractor.stream
+async def context_stream(
+    ctx: tractor.Context,
+    sequence
+):
+    for i in sequence:
+        await ctx.send_yield(i)
+        await trio.sleep(0.1)
+
+    # block indefinitely waiting to be cancelled by ``aclose()`` call
+    with trio.CancelScope() as cs:
+        await trio.sleep(float('inf'))
+        assert 0
+    assert cs.cancelled_caught
+
+
+async def stream_from_single_subactor(
+    arb_addr,
+    start_method,
+    stream_func,
+):
+    """Verify we can spawn a daemon actor and retrieve streamed data.
+    """
+    # only one per host address, spawns an actor if None
+
+    async with tractor.open_nursery(
+        arbiter_addr=arb_addr,
+        start_method=start_method,
+    ) as nursery:
+
+        async with tractor.find_actor('streamerd') as portals:
+
+            if not portals:
+
+                # no brokerd actor found
+                portal = await nursery.start_actor(
+                    'streamerd',
+                    enable_modules=[__name__],
+                )
+
+                seq = range(10)
+
+                with trio.fail_after(5):
+                    async with portal.open_stream_from(
+                        stream_func,
+                        sequence=list(seq),  # has to be msgpack serializable
+                    ) as stream:
+
+                        # it'd sure be nice to have an asyncitertools here...
+                        iseq = iter(seq)
+                        ival = next(iseq)
+
+                        async for val in stream:
+                            assert val == ival
+
+                            try:
+                                ival = next(iseq)
+                            except StopIteration:
+                                # should cancel far end task which will be
+                                # caught and no error is raised
+                                await stream.aclose()
+
+                        await trio.sleep(0.3)
+
+                        # ensure EOC signalled-state translates
+                        # XXX: not really sure this is correct,
+                        # shouldn't it be a `ClosedResourceError`?
+                        try:
+                            await stream.__anext__()
+                        except StopAsyncIteration:
+                            # stop all spawned subactors
+                            await portal.cancel_actor()
+
+
+@pytest.mark.parametrize(
+    'stream_func', [async_gen_stream, context_stream]
+)
+def test_stream_from_single_subactor(arb_addr, start_method, stream_func):
+    """Verify streaming from a spawned async generator.
+    """
+    trio.run(
+        partial(
+            stream_from_single_subactor,
+            arb_addr,
+            start_method,
+            stream_func=stream_func,
+        ),
+    )
+
+
+# this is the first 2 actors, streamer_1 and streamer_2
+async def stream_data(seed):
+
+    for i in range(seed):
+
+        yield i
+
+        # trigger scheduler to simulate practical usage
+        await trio.sleep(0.0001)
+
+
+# this is the third actor; the aggregator
+async def aggregate(seed):
+    """Ensure that the two streams we receive match but only stream
+    a single set of values to the parent.
+    """
+    async with tractor.open_nursery() as nursery:
+        portals = []
+        for i in range(1, 3):
+            # fork point
+            portal = await nursery.start_actor(
+                name=f'streamer_{i}',
+                enable_modules=[__name__],
+            )
+
+            portals.append(portal)
+
+        send_chan, recv_chan = trio.open_memory_channel(500)
+
+        async def push_to_chan(portal, send_chan):
+            async with send_chan:
+
+                async with portal.open_stream_from(
+                    stream_data, seed=seed,
+                ) as stream:
+
+                    async for value in stream:
+                        # leverage trio's built-in backpressure
+                        await send_chan.send(value)
+
+            print(f"FINISHED ITERATING {portal.channel.uid}")
+
+        # spawn 2 trio tasks to collect streams and push to a local queue
+        async with trio.open_nursery() as n:
+
+            for portal in portals:
+                n.start_soon(push_to_chan, portal, send_chan.clone())
+
+            # close this local task's reference to send side
+            await send_chan.aclose()
+
+            unique_vals = set()
+            async with recv_chan:
+                async for value in recv_chan:
+                    if value not in unique_vals:
+                        unique_vals.add(value)
+                        # yield upwards to the spawning parent actor
+                        yield value
+
+                assert value in unique_vals
+
+            print("FINISHED ITERATING in aggregator")
+
+        await nursery.cancel()
+        print("WAITING on `ActorNursery` to finish")
+    print("AGGREGATOR COMPLETE!")
+
+
+# this is the main actor and *arbiter*
+async def a_quadruple_example():
+    # a nursery which spawns "actors"
+    async with tractor.open_nursery() as nursery:
+
+        seed = int(1e3)
+        pre_start = time.time()
+
+        portal = await nursery.start_actor(
+            name='aggregator',
+            enable_modules=[__name__],
+        )
+
+        start = time.time()
+        # the portal call returns exactly what you'd expect
+        # as if the remote "aggregate" function was called locally
+        result_stream = []
+
+        async with portal.open_stream_from(aggregate, seed=seed) as stream:
+            async for value in stream:
+                result_stream.append(value)
+
+        print(f"STREAM TIME = {time.time() - start}")
+        print(f"STREAM + SPAWN TIME = {time.time() - pre_start}")
+        assert result_stream == list(range(seed))
+        await portal.cancel_actor()
+        return result_stream
+
+
+async def cancel_after(wait, arb_addr):
+    async with tractor.open_root_actor(arbiter_addr=arb_addr):
+        with trio.move_on_after(wait):
+            return await a_quadruple_example()
+
+
+@pytest.fixture(scope='module')
+def time_quad_ex(arb_addr, ci_env, spawn_backend):
+    if spawn_backend == 'mp':
+        """no idea but the  mp *nix runs are flaking out here often...
+        """
+        pytest.skip("Test is too flaky on mp in CI")
+
+    timeout = 7 if platform.system() in ('Windows', 'Darwin') else 4
+    start = time.time()
+    results = trio.run(cancel_after, timeout, arb_addr)
+    diff = time.time() - start
+    assert results
+    return results, diff
+
+
+def test_a_quadruple_example(time_quad_ex, ci_env, spawn_backend):
+    """This also serves as a kind of "we'd like to be this fast test"."""
+
+    results, diff = time_quad_ex
+    assert results
+    this_fast = 6 if platform.system() in ('Windows', 'Darwin') else 3
+    assert diff < this_fast
+
+
+@pytest.mark.parametrize(
+    'cancel_delay',
+    list(map(lambda i: i/10, range(3, 9)))
+)
+def test_not_fast_enough_quad(
+    arb_addr, time_quad_ex, cancel_delay, ci_env, spawn_backend
+):
+    """Verify we can cancel midway through the quad example and all actors
+    cancel gracefully.
+    """
+    results, diff = time_quad_ex
+    delay = max(diff - cancel_delay, 0)
+    results = trio.run(cancel_after, delay, arb_addr)
+    system = platform.system()
+    if system in ('Windows', 'Darwin') and results is not None:
+        # In CI envoirments it seems later runs are quicker then the first
+        # so just ignore these
+        print(f"Woa there {system} caught your breath eh?")
+    else:
+        # should be cancelled mid-streaming
+        assert results is None
+
+
+@tractor_test
+async def test_respawn_consumer_task(
+    arb_addr,
+    spawn_backend,
+    loglevel,
+):
+    """Verify that ``._portal.ReceiveStream.shield()``
+    sucessfully protects the underlying IPC channel from being closed
+    when cancelling and respawning a consumer task.
+
+    This also serves to verify that all values from the stream can be
+    received despite the respawns.
+
+    """
+    stream = None
+
+    async with tractor.open_nursery() as n:
+
+        portal = await n.start_actor(
+            name='streamer',
+            enable_modules=[__name__]
+        )
+        async with portal.open_stream_from(
+            stream_data,
+            seed=11,
+        ) as stream:
+
+            expect = set(range(11))
+            received = []
+
+            # this is the re-spawn task routine
+            async def consume(task_status=trio.TASK_STATUS_IGNORED):
+                print('starting consume task..')
+                nonlocal stream
+
+                with trio.CancelScope() as cs:
+                    task_status.started(cs)
+
+                    # shield stream's underlying channel from cancellation
+                    # with stream.shield():
+
+                    async for v in stream:
+                        print(f'from stream: {v}')
+                        expect.remove(v)
+                        received.append(v)
+
+                    print('exited consume')
+
+            async with trio.open_nursery() as ln:
+                cs = await ln.start(consume)
+
+                while True:
+
+                    await trio.sleep(0.1)
+
+                    if received[-1] % 2 == 0:
+
+                        print('cancelling consume task..')
+                        cs.cancel()
+
+                        # respawn
+                        cs = await ln.start(consume)
+
+                    if not expect:
+                        print("all values streamed, BREAKING")
+                        break
+
+                cs.cancel()
+
+        # TODO: this is justification for a
+        # ``ActorNursery.stream_from_actor()`` helper?
+        await portal.cancel_actor()

tests/test_local.py

@@ -1,5 +1,5 @@
 """
-Actor model API testing
+Arbiter and "local" actor api
 """
 import time
 
@@ -11,124 +11,62 @@ from conftest import tractor_test
 
 
 @pytest.mark.trio
-async def test_no_arbitter():
+async def test_no_runtime():
     """An arbitter must be established before any nurseries
     can be created.
 
-    (In other words ``tractor.run`` must be used instead of ``trio.run`` as is
-    done by the ``pytest-trio`` plugin.)
+    (In other words ``tractor.open_root_actor()`` must be engaged at
+    some point?)
     """
-    with pytest.raises(RuntimeError):
-        with tractor.open_nursery():
+    with pytest.raises(RuntimeError) :
+        async with tractor.find_actor('doggy'):
             pass
 
 
+@tractor_test
+async def test_self_is_registered(arb_addr):
+    "Verify waiting on the arbiter to register itself using the standard api."
+    actor = tractor.current_actor()
+    assert actor.is_arbiter
+    with trio.fail_after(0.2):
+        async with tractor.wait_for_actor('root') as portal:
+            assert portal.channel.uid[0] == 'root'
+
+
+@tractor_test
+async def test_self_is_registered_localportal(arb_addr):
+    "Verify waiting on the arbiter to register itself using a local portal."
+    actor = tractor.current_actor()
+    assert actor.is_arbiter
+    async with tractor.get_arbiter(*arb_addr) as portal:
+        assert isinstance(portal, tractor._portal.LocalPortal)
+
+        with trio.fail_after(0.2):
+            sockaddr = await portal.run_from_ns(
+                    'self', 'wait_for_actor', name='root')
+            assert sockaddr[0] == arb_addr
+
+
 def test_local_actor_async_func(arb_addr):
     """Verify a simple async function in-process.
     """
     nums = []
 
     async def print_loop():
-        # arbiter is started in-proc if dne
-        assert tractor.current_actor().is_arbiter
 
-        for i in range(10):
-            nums.append(i)
-            await trio.sleep(0.1)
+        async with tractor.open_root_actor(
+            arbiter_addr=arb_addr,
+        ):
+            # arbiter is started in-proc if dne
+            assert tractor.current_actor().is_arbiter
+
+            for i in range(10):
+                nums.append(i)
+                await trio.sleep(0.1)
 
     start = time.time()
-    tractor.run(print_loop, arbiter_addr=arb_addr)
+    trio.run(print_loop)
 
     # ensure the sleeps were actually awaited
     assert time.time() - start >= 1
     assert nums == list(range(10))
-
-
-statespace = {'doggy': 10, 'kitty': 4}
-
-
-async def spawn(is_arbiter):
-    namespaces = [__name__]
-
-    await trio.sleep(0.1)
-    actor = tractor.current_actor()
-    assert actor.is_arbiter == is_arbiter
-    assert actor.statespace == statespace
-
-    if actor.is_arbiter:
-        async with tractor.open_nursery() as nursery:
-            # forks here
-            portal = await nursery.run_in_actor(
-                'sub-actor',
-                spawn,
-                is_arbiter=False,
-                statespace=statespace,
-                rpc_module_paths=namespaces,
-            )
-
-            assert len(nursery._children) == 1
-            assert portal.channel.uid in tractor.current_actor()._peers
-            # be sure we can still get the result
-            result = await portal.result()
-            assert result == 10
-            return result
-    else:
-        return 10
-
-
-def test_local_arbiter_subactor_global_state(arb_addr):
-    result = tractor.run(
-        spawn,
-        True,
-        name='arbiter',
-        statespace=statespace,
-        arbiter_addr=arb_addr,
-    )
-    assert result == 10
-
-
-def movie_theatre_question():
-    """A question asked in a dark theatre, in a tangent
-    (errr, I mean different) process.
-    """
-    return 'have you ever seen a portal?'
-
-
-@tractor_test
-async def test_movie_theatre_convo():
-    """The main ``tractor`` routine.
-    """
-    async with tractor.open_nursery() as n:
-
-        portal = await n.start_actor(
-            'frank',
-            # enable the actor to run funcs from this current module
-            rpc_module_paths=[__name__],
-        )
-
-        print(await portal.run(__name__, 'movie_theatre_question'))
-        # call the subactor a 2nd time
-        print(await portal.run(__name__, 'movie_theatre_question'))
-
-        # the async with will block here indefinitely waiting
-        # for our actor "frank" to complete, we cancel 'frank'
-        # to avoid blocking indefinitely
-        await portal.cancel_actor()
-
-
-def cellar_door():
-    return "Dang that's beautiful"
-
-
-@tractor_test
-async def test_most_beautiful_word():
-    """The main ``tractor`` routine.
-    """
-    async with tractor.open_nursery() as n:
-
-        portal = await n.run_in_actor('some_linguist', cellar_door)
-
-    # The ``async with`` will unblock here since the 'some_linguist'
-    # actor has completed its main task ``cellar_door``.
-
-    print(await portal.result())

tests/test_multi_program.py

@@ -0,0 +1,66 @@
+"""
+Multiple python programs invoking the runtime.
+"""
+import platform
+import time
+
+import pytest
+import trio
+import tractor
+from conftest import (
+    tractor_test,
+    sig_prog,
+    _INT_SIGNAL,
+    _INT_RETURN_CODE,
+)
+
+
+def test_abort_on_sigint(daemon):
+    assert daemon.returncode is None
+    time.sleep(0.1)
+    sig_prog(daemon, _INT_SIGNAL)
+    assert daemon.returncode == _INT_RETURN_CODE
+
+    # XXX: oddly, couldn't get capfd.readouterr() to work here?
+    if platform.system() != 'Windows':
+        # don't check stderr on windows as its empty when sending CTRL_C_EVENT
+        assert "KeyboardInterrupt" in str(daemon.stderr.read())
+
+
+@tractor_test
+async def test_cancel_remote_arbiter(daemon, arb_addr):
+    assert not tractor.current_actor().is_arbiter
+    async with tractor.get_arbiter(*arb_addr) as portal:
+        await portal.cancel_actor()
+
+    time.sleep(0.1)
+    # the arbiter channel server is cancelled but not its main task
+    assert daemon.returncode is None
+
+    # no arbiter socket should exist
+    with pytest.raises(OSError):
+        async with tractor.get_arbiter(*arb_addr) as portal:
+            pass
+
+
+def test_register_duplicate_name(daemon, arb_addr):
+
+    async def main():
+
+        async with tractor.open_nursery(
+            arbiter_addr=arb_addr,
+        ) as n:
+
+            assert not tractor.current_actor().is_arbiter
+
+            p1 = await n.start_actor('doggy')
+            p2 = await n.start_actor('doggy')
+
+            async with tractor.wait_for_actor('doggy') as portal:
+                assert portal.channel.uid in (p2.channel.uid, p1.channel.uid)
+
+            await n.cancel()
+
+    # run it manually since we want to start **after**
+    # the other "daemon" program
+    trio.run(main)

tests/test_pubsub.py

@@ -0,0 +1,293 @@
+import time
+from itertools import cycle
+
+import pytest
+import trio
+import tractor
+from tractor.experimental import msgpub
+
+from conftest import tractor_test
+
+
+def test_type_checks():
+
+    with pytest.raises(TypeError) as err:
+        @msgpub
+        async def no_get_topics(yo):
+            yield
+
+    assert "must define a `get_topics`" in str(err.value)
+
+    with pytest.raises(TypeError) as err:
+        @msgpub
+        def not_async_gen(yo):
+            pass
+
+    assert "must be an async generator function" in str(err.value)
+
+
+def is_even(i):
+    return i % 2 == 0
+
+
+# placeholder for topics getter
+_get_topics = None
+
+
+@msgpub
+async def pubber(get_topics, seed=10):
+
+    # ensure topic subscriptions are as expected
+    global _get_topics
+    _get_topics = get_topics
+
+    for i in cycle(range(seed)):
+
+        yield {'even' if is_even(i) else 'odd': i}
+        await trio.sleep(0.1)
+
+
+async def subs(
+    which,
+    pub_actor_name,
+    seed=10,
+    task_status=trio.TASK_STATUS_IGNORED,
+):
+    if len(which) == 1:
+        if which[0] == 'even':
+            pred = is_even
+        else:
+            def pred(i):
+                return not is_even(i)
+    else:
+        def pred(i):
+            return isinstance(i, int)
+
+    # TODO: https://github.com/goodboy/tractor/issues/207
+    async with tractor.wait_for_actor(pub_actor_name) as portal:
+        assert portal
+
+        async with portal.open_stream_from(
+            pubber,
+            topics=which,
+            seed=seed,
+        ) as stream:
+            task_status.started(stream)
+            times = 10
+            count = 0
+            await stream.__anext__()
+            async for pkt in stream:
+                for topic, value in pkt.items():
+                    assert pred(value)
+                count += 1
+                if count >= times:
+                    break
+
+            await stream.aclose()
+
+        async with portal.open_stream_from(
+            pubber,
+            topics=['odd'],
+            seed=seed,
+        ) as stream:
+            await stream.__anext__()
+            count = 0
+            # async with aclosing(stream) as stream:
+            try:
+                async for pkt in stream:
+                    for topic, value in pkt.items():
+                        pass
+                        # assert pred(value)
+                    count += 1
+                    if count >= times:
+                        break
+            finally:
+                await stream.aclose()
+
+
+@msgpub(tasks=['one', 'two'])
+async def multilock_pubber(get_topics):
+    yield {'doggy': 10}
+
+
+@pytest.mark.parametrize(
+    'callwith_expecterror',
+    [
+        (pubber, {}, TypeError),
+        # missing a `topics`
+        (multilock_pubber, {'ctx': None}, TypeError),
+        # missing a `task_name`
+        (multilock_pubber, {'ctx': None, 'topics': ['topic1']}, TypeError),
+        # should work
+        (multilock_pubber,
+         {'ctx': None, 'topics': ['doggy'], 'task_name': 'one'},
+         None),
+    ],
+)
+@tractor_test
+async def test_required_args(callwith_expecterror):
+    func, kwargs, err = callwith_expecterror
+
+    if err is not None:
+        with pytest.raises(err):
+            await func(**kwargs)
+    else:
+        async with tractor.open_nursery() as n:
+
+            portal = await n.start_actor(
+                name='pubber',
+                enable_modules=[__name__],
+            )
+
+            async with tractor.wait_for_actor('pubber'):
+                pass
+
+            await trio.sleep(0.5)
+
+            async with portal.open_stream_from(
+                multilock_pubber,
+                **kwargs
+            ) as stream:
+                async for val in stream:
+                    assert val == {'doggy': 10}
+
+            await portal.cancel_actor()
+
+
+@pytest.mark.parametrize(
+    'pub_actor',
+    ['streamer', 'arbiter']
+)
+def test_multi_actor_subs_arbiter_pub(
+    loglevel,
+    arb_addr,
+    pub_actor,
+):
+    """Try out the neato @pub decorator system.
+    """
+    global _get_topics
+
+    async def main():
+
+        async with tractor.open_nursery(
+            arbiter_addr=arb_addr,
+            enable_modules=[__name__],
+        ) as n:
+
+            name = 'root'
+
+            if pub_actor == 'streamer':
+                # start the publisher as a daemon
+                master_portal = await n.start_actor(
+                    'streamer',
+                    enable_modules=[__name__],
+                )
+                name = 'streamer'
+
+            even_portal = await n.run_in_actor(
+                subs,
+                which=['even'],
+                name='evens',
+                pub_actor_name=name
+            )
+            odd_portal = await n.run_in_actor(
+                subs,
+                which=['odd'],
+                name='odds',
+                pub_actor_name=name
+            )
+
+            async with tractor.wait_for_actor('evens'):
+                # block until 2nd actor is initialized
+                pass
+
+            if pub_actor == 'arbiter':
+
+                # wait for publisher task to be spawned in a local RPC task
+                while _get_topics is None:
+                    await trio.sleep(0.1)
+
+                get_topics = _get_topics
+
+                assert 'even' in get_topics()
+
+            async with tractor.wait_for_actor('odds'):
+                # block until 2nd actor is initialized
+                pass
+
+            if pub_actor == 'arbiter':
+                start = time.time()
+                while 'odd' not in get_topics():
+                    await trio.sleep(0.1)
+                    if time.time() - start > 1:
+                        pytest.fail("odds subscription never arrived?")
+
+            # TODO: how to make this work when the arbiter gets
+            # a portal to itself? Currently this causes a hang
+            # when the channel server is torn down due to a lingering
+            # loopback channel
+            #     with trio.move_on_after(1):
+            #         await subs(['even', 'odd'])
+
+            # XXX: this would cause infinite
+            # blocking due to actor never terminating loop
+            # await even_portal.result()
+
+            await trio.sleep(0.5)
+            await even_portal.cancel_actor()
+            await trio.sleep(1)
+
+            if pub_actor == 'arbiter':
+                assert 'even' not in get_topics()
+
+            await odd_portal.cancel_actor()
+
+            if pub_actor == 'arbiter':
+                while get_topics():
+                    await trio.sleep(0.1)
+                    if time.time() - start > 2:
+                        pytest.fail("odds subscription never dropped?")
+            else:
+                await master_portal.cancel_actor()
+
+    trio.run(main)
+
+
+def test_single_subactor_pub_multitask_subs(
+    loglevel,
+    arb_addr,
+):
+    async def main():
+
+        async with tractor.open_nursery(
+            arbiter_addr=arb_addr,
+            enable_modules=[__name__],
+        ) as n:
+
+            portal = await n.start_actor(
+                'streamer',
+                enable_modules=[__name__],
+            )
+            async with tractor.wait_for_actor('streamer'):
+                # block until 2nd actor is initialized
+                pass
+
+            async with trio.open_nursery() as tn:
+                agen = await tn.start(subs, ['even'], 'streamer')
+
+                await trio.sleep(0.1)
+                tn.start_soon(subs, ['even'], 'streamer')
+
+                # XXX this will trigger the python bug:
+                # https://bugs.python.org/issue32526
+                # if using async generators to wrap tractor channels
+                await agen.aclose()
+
+                await trio.sleep(0.1)
+                tn.start_soon(subs, ['even'], 'streamer')
+                await trio.sleep(0.1)
+                tn.start_soon(subs, ['even'], 'streamer')
+
+            await portal.cancel_actor()
+
+    trio.run(main)

tests/test_resource_cache.py

@@ -0,0 +1,182 @@
+'''
+Async context manager cache api testing: ``trionics.maybe_open_context():``
+
+'''
+from contextlib import asynccontextmanager as acm
+import platform
+from typing import Awaitable
+
+import pytest
+import trio
+import tractor
+
+
+_resource: int = 0
+
+
+@acm
+async def maybe_increment_counter(task_name: str):
+    global _resource
+
+    _resource += 1
+    await trio.lowlevel.checkpoint()
+    yield _resource
+    await trio.lowlevel.checkpoint()
+    _resource -= 1
+
+
+@pytest.mark.parametrize(
+    'key_on',
+    ['key_value', 'kwargs'],
+    ids="key_on={}".format,
+)
+def test_resource_only_entered_once(key_on):
+    global _resource
+    _resource = 0
+
+    kwargs = {}
+    key = None
+    if key_on == 'key_value':
+        key = 'some_common_key'
+
+    async def main():
+        cache_active: bool = False
+
+        async def enter_cached_mngr(name: str):
+            nonlocal cache_active
+
+            if key_on == 'kwargs':
+                # make a common kwargs input to key on it
+                kwargs = {'task_name': 'same_task_name'}
+                assert key is None
+            else:
+                # different task names per task will be used
+                kwargs = {'task_name': name}
+
+            async with tractor.trionics.maybe_open_context(
+                maybe_increment_counter,
+                kwargs=kwargs,
+                key=key,
+
+            ) as (cache_hit, resource):
+                if cache_hit:
+                    try:
+                        cache_active = True
+                        assert resource == 1
+                        await trio.sleep_forever()
+                    finally:
+                        cache_active = False
+                else:
+                    assert resource == 1
+                    await trio.sleep_forever()
+
+        with trio.move_on_after(0.5):
+            async with (
+                tractor.open_root_actor(),
+                trio.open_nursery() as n,
+            ):
+
+                for i in range(10):
+                    n.start_soon(enter_cached_mngr, f'task_{i}')
+                    await trio.sleep(0.001)
+
+    trio.run(main)
+
+
+@tractor.context
+async def streamer(
+    ctx: tractor.Context,
+    seq: list[int] = list(range(1000)),
+) -> None:
+
+    await ctx.started()
+    async with ctx.open_stream() as stream:
+        for val in seq:
+            await stream.send(val)
+            await trio.sleep(0.001)
+
+    print('producer finished')
+
+
+@acm
+async def open_stream() -> Awaitable[tractor.MsgStream]:
+
+    async with tractor.open_nursery() as tn:
+        portal = await tn.start_actor('streamer', enable_modules=[__name__])
+        async with (
+            portal.open_context(streamer) as (ctx, first),
+            ctx.open_stream() as stream,
+        ):
+            yield stream
+
+        await portal.cancel_actor()
+    print('CANCELLED STREAMER')
+
+
+@acm
+async def maybe_open_stream(taskname: str):
+    async with tractor.trionics.maybe_open_context(
+        # NOTE: all secondary tasks should cache hit on the same key
+        acm_func=open_stream,
+    ) as (cache_hit, stream):
+
+        if cache_hit:
+            print(f'{taskname} loaded from cache')
+
+            # add a new broadcast subscription for the quote stream
+            # if this feed is already allocated by the first
+            # task that entereed
+            async with stream.subscribe() as bstream:
+                yield bstream
+        else:
+            # yield the actual stream
+            yield stream
+
+
+def test_open_local_sub_to_stream():
+    '''
+    Verify a single inter-actor stream can can be fanned-out shared to
+    N local tasks using ``trionics.maybe_open_context():``.
+
+    '''
+    timeout = 3 if platform.system() != "Windows" else 10
+
+    async def main():
+
+        full = list(range(1000))
+
+        async def get_sub_and_pull(taskname: str):
+            async with (
+                maybe_open_stream(taskname) as stream,
+            ):
+                if '0' in taskname:
+                    assert isinstance(stream, tractor.MsgStream)
+                else:
+                    assert isinstance(
+                        stream,
+                        tractor.trionics.BroadcastReceiver
+                    )
+
+                first = await stream.receive()
+                print(f'{taskname} started with value {first}')
+                seq = []
+                async for msg in stream:
+                    seq.append(msg)
+
+                assert set(seq).issubset(set(full))
+            print(f'{taskname} finished')
+
+        with trio.fail_after(timeout):
+            # TODO: turns out this isn't multi-task entrant XD
+            # We probably need an indepotent entry semantic?
+            async with tractor.open_root_actor():
+                async with (
+                    trio.open_nursery() as nurse,
+                ):
+                    for i in range(10):
+                        nurse.start_soon(get_sub_and_pull, f'task_{i}')
+                        await trio.sleep(0.001)
+
+                print('all consumer tasks finished')
+
+    trio.run(main)

tests/test_rpc.py

@@ -0,0 +1,122 @@
+"""
+RPC related
+"""
+import itertools
+
+import pytest
+import tractor
+import trio
+
+
+async def sleep_back_actor(
+    actor_name,
+    func_name,
+    func_defined,
+    exposed_mods,
+):
+    if actor_name:
+        async with tractor.find_actor(actor_name) as portal:
+            try:
+                await portal.run(__name__, func_name)
+            except tractor.RemoteActorError as err:
+                if not func_defined:
+                    expect = AttributeError
+                if not exposed_mods:
+                    expect = tractor.ModuleNotExposed
+
+                assert err.type is expect
+                raise
+    else:
+        await trio.sleep(float('inf'))
+
+
+async def short_sleep():
+    await trio.sleep(0)
+
+
+@pytest.mark.parametrize(
+    'to_call', [
+        ([], 'short_sleep', tractor.RemoteActorError),
+        ([__name__], 'short_sleep', tractor.RemoteActorError),
+        ([__name__], 'fake_func', tractor.RemoteActorError),
+        (['tmp_mod'], 'import doggy', ModuleNotFoundError),
+        (['tmp_mod'], '4doggy', SyntaxError),
+    ],
+    ids=['no_mods', 'this_mod', 'this_mod_bad_func', 'fail_to_import',
+         'fail_on_syntax'],
+)
+def test_rpc_errors(arb_addr, to_call, testdir):
+    """Test errors when making various RPC requests to an actor
+    that either doesn't have the requested module exposed or doesn't define
+    the named function.
+    """
+    exposed_mods, funcname, inside_err = to_call
+    subactor_exposed_mods = []
+    func_defined = globals().get(funcname, False)
+    subactor_requests_to = 'root'
+    remote_err = tractor.RemoteActorError
+
+    # remote module that fails at import time
+    if exposed_mods == ['tmp_mod']:
+        # create an importable module with a bad import
+        testdir.syspathinsert()
+        # module should raise a ModuleNotFoundError at import
+        testdir.makefile('.py', tmp_mod=funcname)
+
+        # no need to expose module to the subactor
+        subactor_exposed_mods = exposed_mods
+        exposed_mods = []
+        func_defined = False
+        # subactor should not try to invoke anything
+        subactor_requests_to = None
+        # the module will be attempted to be imported locally but will
+        # fail in the initial local instance of the actor
+        remote_err = inside_err
+
+    async def main():
+
+        # spawn a subactor which calls us back
+        async with tractor.open_nursery(
+            arbiter_addr=arb_addr,
+            enable_modules=exposed_mods.copy(),
+        ) as n:
+
+            actor = tractor.current_actor()
+            assert actor.is_arbiter
+            await n.run_in_actor(
+                sleep_back_actor,
+                actor_name=subactor_requests_to,
+
+                name='subactor',
+
+                # function from the local exposed module space
+                # the subactor will invoke when it RPCs back to this actor
+                func_name=funcname,
+                exposed_mods=exposed_mods,
+                func_defined=True if func_defined else False,
+                enable_modules=subactor_exposed_mods,
+            )
+
+    def run():
+        trio.run(main)
+
+    # handle both parameterized cases
+    if exposed_mods and func_defined:
+        run()
+    else:
+        # underlying errors aren't propagated upwards (yet)
+        with pytest.raises(remote_err) as err:
+            run()
+
+        # get raw instance from pytest wrapper
+        value = err.value
+
+        # might get multiple `trio.Cancelled`s as well inside an inception
+        if isinstance(value, trio.MultiError):
+            value = next(itertools.dropwhile(
+                lambda exc: not isinstance(exc, tractor.RemoteActorError),
+                value.exceptions
+            ))
+
+        if getattr(value, 'type', None):
+            assert value.type is inside_err

tests/test_runtime.py

@@ -0,0 +1,73 @@
+"""
+Verifying internal runtime state and undocumented extras.
+
+"""
+import os
+
+import pytest
+import trio
+import tractor
+
+from conftest import tractor_test
+
+
+_file_path: str = ''
+
+
+def unlink_file():
+    print('Removing tmp file!')
+    os.remove(_file_path)
+
+
+async def crash_and_clean_tmpdir(
+    tmp_file_path: str,
+    error: bool = True,
+):
+    global _file_path
+    _file_path = tmp_file_path
+
+    actor = tractor.current_actor()
+    actor.lifetime_stack.callback(unlink_file)
+
+    assert os.path.isfile(tmp_file_path)
+    await trio.sleep(0.1)
+    if error:
+        assert 0
+    else:
+        actor.cancel_soon()
+
+
+@pytest.mark.parametrize(
+    'error_in_child',
+    [True, False],
+)
+@tractor_test
+async def test_lifetime_stack_wipes_tmpfile(
+    tmp_path,
+    error_in_child: bool,
+):
+    child_tmp_file = tmp_path / "child.txt"
+    child_tmp_file.touch()
+    assert child_tmp_file.exists()
+    path = str(child_tmp_file)
+
+    try:
+        with trio.move_on_after(0.5):
+            async with tractor.open_nursery() as n:
+                    await (  # inlined portal
+                        await n.run_in_actor(
+                            crash_and_clean_tmpdir,
+                            tmp_file_path=path,
+                            error=error_in_child,
+                        )
+                    ).result()
+
+    except (
+        tractor.RemoteActorError,
+        tractor.BaseExceptionGroup,
+    ):
+        pass
+
+    # tmp file should have been wiped by
+    # teardown stack.
+    assert not child_tmp_file.exists()

tests/test_spawning.py

@@ -1,55 +1,71 @@
 """
 Spawning basics
+
 """
+from typing import Optional
+
+import pytest
 import trio
 import tractor
 
 from conftest import tractor_test
 
-statespace = {'doggy': 10, 'kitty': 4}
+data_to_pass_down = {'doggy': 10, 'kitty': 4}
 
 
-async def spawn(is_arbiter):
+async def spawn(
+    is_arbiter: bool,
+    data: dict,
+    arb_addr: tuple[str, int],
+):
     namespaces = [__name__]
 
     await trio.sleep(0.1)
-    actor = tractor.current_actor()
-    assert actor.is_arbiter == is_arbiter
-    assert actor.statespace == statespace
 
-    if actor.is_arbiter:
-        async with tractor.open_nursery() as nursery:
-            # forks here
-            portal = await nursery.run_in_actor(
-                'sub-actor',
-                spawn,
-                is_arbiter=False,
-                statespace=statespace,
-                rpc_module_paths=namespaces,
-            )
+    async with tractor.open_root_actor(
+        arbiter_addr=arb_addr,
+    ):
 
-            assert len(nursery._children) == 1
-            assert portal.channel.uid in tractor.current_actor()._peers
-            # be sure we can still get the result
-            result = await portal.result()
-            assert result == 10
-            return result
-    else:
-        return 10
+        actor = tractor.current_actor()
+        assert actor.is_arbiter == is_arbiter
+        data = data_to_pass_down
+
+        if actor.is_arbiter:
+
+            async with tractor.open_nursery(
+            ) as nursery:
+
+                # forks here
+                portal = await nursery.run_in_actor(
+                    spawn,
+                    is_arbiter=False,
+                    name='sub-actor',
+                    data=data,
+                    arb_addr=arb_addr,
+                    enable_modules=namespaces,
+                )
+
+                assert len(nursery._children) == 1
+                assert portal.channel.uid in tractor.current_actor()._peers
+                # be sure we can still get the result
+                result = await portal.result()
+                assert result == 10
+                return result
+        else:
+            return 10
 
 
 def test_local_arbiter_subactor_global_state(arb_addr):
-    result = tractor.run(
+    result = trio.run(
         spawn,
         True,
-        name='arbiter',
-        statespace=statespace,
-        arbiter_addr=arb_addr,
+        data_to_pass_down,
+        arb_addr,
     )
     assert result == 10
 
 
-def movie_theatre_question():
+async def movie_theatre_question():
     """A question asked in a dark theatre, in a tangent
     (errr, I mean different) process.
     """
@@ -57,7 +73,7 @@ def movie_theatre_question():
 
 
 @tractor_test
-async def test_movie_theatre_convo():
+async def test_movie_theatre_convo(start_method):
     """The main ``tractor`` routine.
     """
     async with tractor.open_nursery() as n:
@@ -65,12 +81,12 @@ async def test_movie_theatre_convo():
         portal = await n.start_actor(
             'frank',
             # enable the actor to run funcs from this current module
-            rpc_module_paths=[__name__],
+            enable_modules=[__name__],
         )
 
-        print(await portal.run(__name__, 'movie_theatre_question'))
+        print(await portal.run(movie_theatre_question))
         # call the subactor a 2nd time
-        print(await portal.run(__name__, 'movie_theatre_question'))
+        print(await portal.run(movie_theatre_question))
 
         # the async with will block here indefinitely waiting
         # for our actor "frank" to complete, we cancel 'frank'
@@ -78,19 +94,75 @@ async def test_movie_theatre_convo():
         await portal.cancel_actor()
 
 
-def cellar_door():
-    return "Dang that's beautiful"
+async def cellar_door(return_value: Optional[str]):
+    return return_value
 
 
+@pytest.mark.parametrize(
+    'return_value', ["Dang that's beautiful", None],
+    ids=['return_str', 'return_None'],
+)
 @tractor_test
-async def test_most_beautiful_word():
-    """The main ``tractor`` routine.
-    """
-    async with tractor.open_nursery() as n:
+async def test_most_beautiful_word(
+    start_method,
+    return_value
+):
+    '''
+    The main ``tractor`` routine.
 
-        portal = await n.run_in_actor('some_linguist', cellar_door)
+    '''
+    with trio.fail_after(1):
+        async with tractor.open_nursery() as n:
 
+            portal = await n.run_in_actor(
+                cellar_door,
+                return_value=return_value,
+                name='some_linguist',
+            )
+
+            print(await portal.result())
     # The ``async with`` will unblock here since the 'some_linguist'
     # actor has completed its main task ``cellar_door``.
 
+    # this should pull the cached final result already captured during
+    # the nursery block exit.
     print(await portal.result())
+
+
+async def check_loglevel(level):
+    assert tractor.current_actor().loglevel == level
+    log = tractor.log.get_logger()
+    # XXX using a level actually used inside tractor seems to trigger
+    # some kind of `logging` module bug FYI.
+    log.critical('yoyoyo')
+
+
+def test_loglevel_propagated_to_subactor(
+    start_method,
+    capfd,
+    arb_addr,
+):
+    if start_method == 'mp_forkserver':
+        pytest.skip(
+            "a bug with `capfd` seems to make forkserver capture not work?")
+
+    level = 'critical'
+
+    async def main():
+        async with tractor.open_nursery(
+            name='arbiter',
+            start_method=start_method,
+            arbiter_addr=arb_addr,
+
+        ) as tn:
+            await tn.run_in_actor(
+                check_loglevel,
+                loglevel=level,
+                level=level,
+            )
+
+    trio.run(main)
+
+    # ensure subactor spits log message on stderr
+    captured = capfd.readouterr()
+    assert 'yoyoyo' in captured.err

tests/test_streaming.py

@@ -1,178 +0,0 @@
-"""
-Streaming via async gen api
-"""
-import time
-
-import trio
-import tractor
-import pytest
-
-
-async def stream_seq(sequence):
-    for i in sequence:
-        yield i
-        await trio.sleep(0.1)
-
-
-async def stream_from_single_subactor():
-    """Verify we can spawn a daemon actor and retrieve streamed data.
-    """
-    async with tractor.find_actor('brokerd') as portals:
-        if not portals:
-            # only one per host address, spawns an actor if None
-            async with tractor.open_nursery() as nursery:
-                # no brokerd actor found
-                portal = await nursery.start_actor(
-                    'streamerd',
-                    rpc_module_paths=[__name__],
-                    statespace={'global_dict': {}},
-                )
-
-                seq = range(10)
-
-                agen = await portal.run(
-                    __name__,
-                    'stream_seq',  # the func above
-                    sequence=list(seq),  # has to be msgpack serializable
-                )
-                # it'd sure be nice to have an asyncitertools here...
-                iseq = iter(seq)
-                async for val in agen:
-                    assert val == next(iseq)
-                    # TODO: test breaking the loop (should it kill the
-                    # far end?)
-                    # break
-                    # terminate far-end async-gen
-                    # await gen.asend(None)
-                    # break
-
-                # stop all spawned subactors
-                await portal.cancel_actor()
-                # await nursery.cancel()
-
-
-def test_stream_from_single_subactor(arb_addr):
-    """Verify streaming from a spawned async generator.
-    """
-    tractor.run(stream_from_single_subactor, arbiter_addr=arb_addr)
-
-
-# this is the first 2 actors, streamer_1 and streamer_2
-async def stream_data(seed):
-    for i in range(seed):
-        yield i
-        await trio.sleep(0)  # trigger scheduler
-
-
-# this is the third actor; the aggregator
-async def aggregate(seed):
-    """Ensure that the two streams we receive match but only stream
-    a single set of values to the parent.
-    """
-    async with tractor.open_nursery() as nursery:
-        portals = []
-        for i in range(1, 3):
-            # fork point
-            portal = await nursery.start_actor(
-                name=f'streamer_{i}',
-                rpc_module_paths=[__name__],
-            )
-
-            portals.append(portal)
-
-        q = trio.Queue(500)
-
-        async def push_to_q(portal):
-            async for value in await portal.run(
-                __name__, 'stream_data', seed=seed
-            ):
-                # leverage trio's built-in backpressure
-                await q.put(value)
-
-            await q.put(None)
-            print(f"FINISHED ITERATING {portal.channel.uid}")
-
-        # spawn 2 trio tasks to collect streams and push to a local queue
-        async with trio.open_nursery() as n:
-            for portal in portals:
-                n.start_soon(push_to_q, portal)
-
-            unique_vals = set()
-            async for value in q:
-                if value not in unique_vals:
-                    unique_vals.add(value)
-                    # yield upwards to the spawning parent actor
-                    yield value
-
-                    if value is None:
-                        break
-
-                assert value in unique_vals
-
-            print("FINISHED ITERATING in aggregator")
-
-        await nursery.cancel()
-        print("WAITING on `ActorNursery` to finish")
-    print("AGGREGATOR COMPLETE!")
-
-
-# this is the main actor and *arbiter*
-async def a_quadruple_example():
-    # a nursery which spawns "actors"
-    async with tractor.open_nursery() as nursery:
-
-        seed = int(1e3)
-        pre_start = time.time()
-
-        portal = await nursery.run_in_actor(
-            'aggregator',
-            aggregate,
-            seed=seed,
-        )
-
-        start = time.time()
-        # the portal call returns exactly what you'd expect
-        # as if the remote "aggregate" function was called locally
-        result_stream = []
-        async for value in await portal.result():
-            result_stream.append(value)
-
-        print(f"STREAM TIME = {time.time() - start}")
-        print(f"STREAM + SPAWN TIME = {time.time() - pre_start}")
-        assert result_stream == list(range(seed)) + [None]
-        return result_stream
-
-
-async def cancel_after(wait):
-    with trio.move_on_after(wait):
-        return await a_quadruple_example()
-
-
-@pytest.fixture(scope='module')
-def time_quad_ex(arb_addr):
-    start = time.time()
-    results = tractor.run(cancel_after, 3, arbiter_addr=arb_addr)
-    diff = time.time() - start
-    assert results
-    return results, diff
-
-
-def test_a_quadruple_example(time_quad_ex):
-    """This also serves as a kind of "we'd like to be this fast test"."""
-    results, diff = time_quad_ex
-    assert results
-    assert diff < 2.5
-
-
-@pytest.mark.parametrize(
-    'cancel_delay',
-    list(map(lambda i: i/10, range(2, 8)))
-)
-def test_not_fast_enough_quad(arb_addr, time_quad_ex, cancel_delay):
-    """Verify we can cancel midway through the quad example and all actors
-    cancel gracefully.
-    """
-    results, diff = time_quad_ex
-    delay = max(diff - cancel_delay, 0)
-    results = tractor.run(cancel_after, delay, arbiter_addr=arb_addr)
-    assert results is None

tests/test_task_broadcasting.py

@@ -0,0 +1,514 @@
+"""
+Broadcast channels for fan-out to local tasks.
+
+"""
+from contextlib import asynccontextmanager
+from functools import partial
+from itertools import cycle
+import time
+from typing import Optional
+
+import pytest
+import trio
+from trio.lowlevel import current_task
+import tractor
+from tractor.trionics import (
+    broadcast_receiver,
+    Lagged,
+)
+
+
+@tractor.context
+async def echo_sequences(
+
+    ctx:  tractor.Context,
+
+) -> None:
+    '''Bidir streaming endpoint which will stream
+    back any sequence it is sent item-wise.
+
+    '''
+    await ctx.started()
+
+    async with ctx.open_stream() as stream:
+        async for sequence in stream:
+            seq = list(sequence)
+            for value in seq:
+                await stream.send(value)
+                print(f'producer sent {value}')
+
+
+async def ensure_sequence(
+
+    stream: tractor.MsgStream,
+    sequence: list,
+    delay: Optional[float] = None,
+
+) -> None:
+
+    name = current_task().name
+    async with stream.subscribe() as bcaster:
+        assert not isinstance(bcaster, type(stream))
+        async for value in bcaster:
+            print(f'{name} rx: {value}')
+            assert value == sequence[0]
+            sequence.remove(value)
+
+            if delay:
+                await trio.sleep(delay)
+
+            if not sequence:
+                # fully consumed
+                break
+
+
+@asynccontextmanager
+async def open_sequence_streamer(
+
+    sequence: list[int],
+    arb_addr: tuple[str, int],
+    start_method: str,
+
+) -> tractor.MsgStream:
+
+    async with tractor.open_nursery(
+        arbiter_addr=arb_addr,
+        start_method=start_method,
+    ) as tn:
+
+        portal = await tn.start_actor(
+            'sequence_echoer',
+            enable_modules=[__name__],
+        )
+
+        async with portal.open_context(
+            echo_sequences,
+        ) as (ctx, first):
+
+            assert first is None
+            async with ctx.open_stream(backpressure=True) as stream:
+                yield stream
+
+        await portal.cancel_actor()
+
+
+def test_stream_fan_out_to_local_subscriptions(
+    arb_addr,
+    start_method,
+):
+
+    sequence = list(range(1000))
+
+    async def main():
+
+        async with open_sequence_streamer(
+            sequence,
+            arb_addr,
+            start_method,
+        ) as stream:
+
+            async with trio.open_nursery() as n:
+                for i in range(10):
+                    n.start_soon(
+                        ensure_sequence,
+                        stream,
+                        sequence.copy(),
+                        name=f'consumer_{i}',
+                    )
+
+                await stream.send(tuple(sequence))
+
+                async for value in stream:
+                    print(f'source stream rx: {value}')
+                    assert value == sequence[0]
+                    sequence.remove(value)
+
+                    if not sequence:
+                        # fully consumed
+                        break
+
+    trio.run(main)
+
+
+@pytest.mark.parametrize(
+    'task_delays',
+    [
+        (0.01, 0.001),
+        (0.001, 0.01),
+    ]
+)
+def test_consumer_and_parent_maybe_lag(
+    arb_addr,
+    start_method,
+    task_delays,
+):
+
+    async def main():
+
+        sequence = list(range(300))
+        parent_delay, sub_delay = task_delays
+
+        async with open_sequence_streamer(
+            sequence,
+            arb_addr,
+            start_method,
+        ) as stream:
+
+            try:
+                async with trio.open_nursery() as n:
+
+                    n.start_soon(
+                        ensure_sequence,
+                        stream,
+                        sequence.copy(),
+                        sub_delay,
+                        name='consumer_task',
+                    )
+
+                    await stream.send(tuple(sequence))
+
+                    # async for value in stream:
+                    lagged = False
+                    lag_count = 0
+
+                    while True:
+                        try:
+                            value = await stream.receive()
+                            print(f'source stream rx: {value}')
+
+                            if lagged:
+                                # re set the sequence starting at our last
+                                # value
+                                sequence = sequence[sequence.index(value) + 1:]
+                            else:
+                                assert value == sequence[0]
+                                sequence.remove(value)
+
+                            lagged = False
+
+                        except Lagged:
+                            lagged = True
+                            print(f'source stream lagged after {value}')
+                            lag_count += 1
+                            continue
+
+                        # lag the parent
+                        await trio.sleep(parent_delay)
+
+                        if not sequence:
+                            # fully consumed
+                            break
+                    print(f'parent + source stream lagged: {lag_count}')
+
+                    if parent_delay > sub_delay:
+                        assert lag_count > 0
+
+            except Lagged:
+                # child was lagged
+                assert parent_delay < sub_delay
+
+    trio.run(main)
+
+
+def test_faster_task_to_recv_is_cancelled_by_slower(
+    arb_addr,
+    start_method,
+):
+    '''
+    Ensure that if a faster task consuming from a stream is cancelled
+    the slower task can continue to receive all expected values.
+
+    '''
+    async def main():
+
+        sequence = list(range(1000))
+
+        async with open_sequence_streamer(
+            sequence,
+            arb_addr,
+            start_method,
+
+        ) as stream:
+
+            async with trio.open_nursery() as n:
+                n.start_soon(
+                    ensure_sequence,
+                    stream,
+                    sequence.copy(),
+                    0,
+                    name='consumer_task',
+                )
+
+                await stream.send(tuple(sequence))
+
+                # pull 3 values, cancel the subtask, then
+                # expect to be able to pull all values still
+                for i in range(20):
+                    try:
+                        value = await stream.receive()
+                        print(f'source stream rx: {value}')
+                        await trio.sleep(0.01)
+                    except Lagged:
+                        print(f'parent overrun after {value}')
+                        continue
+
+                print('cancelling faster subtask')
+                n.cancel_scope.cancel()
+
+            try:
+                value = await stream.receive()
+                print(f'source stream after cancel: {value}')
+            except Lagged:
+                print(f'parent overrun after {value}')
+
+            # expect to see all remaining values
+            with trio.fail_after(0.5):
+                async for value in stream:
+                    assert stream._broadcaster._state.recv_ready is None
+                    print(f'source stream rx: {value}')
+                    if value == 999:
+                        # fully consumed and we missed no values once
+                        # the faster subtask was cancelled
+                        break
+
+                # await tractor.breakpoint()
+                # await stream.receive()
+                print(f'final value: {value}')
+
+    trio.run(main)
+
+
+def test_subscribe_errors_after_close():
+
+    async def main():
+
+        size = 1
+        tx, rx = trio.open_memory_channel(size)
+        async with broadcast_receiver(rx, size) as brx:
+            pass
+
+        try:
+            # open and close
+            async with brx.subscribe():
+                pass
+
+        except trio.ClosedResourceError:
+            assert brx.key not in brx._state.subs
+
+        else:
+            assert 0
+
+    trio.run(main)
+
+
+def test_ensure_slow_consumers_lag_out(
+    arb_addr,
+    start_method,
+):
+    '''This is a pure local task test; no tractor
+    machinery is really required.
+
+    '''
+    async def main():
+
+        # make sure it all works within the runtime
+        async with tractor.open_root_actor():
+
+            num_laggers = 4
+            laggers: dict[str, int] = {}
+            retries = 3
+            size = 100
+            tx, rx = trio.open_memory_channel(size)
+            brx = broadcast_receiver(rx, size)
+
+            async def sub_and_print(
+                delay: float,
+            ) -> None:
+
+                task = current_task()
+                start = time.time()
+
+                async with brx.subscribe() as lbrx:
+                    while True:
+                        print(f'{task.name}: starting consume loop')
+                        try:
+                            async for value in lbrx:
+                                print(f'{task.name}: {value}')
+                                await trio.sleep(delay)
+
+                            if task.name == 'sub_1':
+                                # trigger checkpoint to clean out other subs
+                                await trio.sleep(0.01)
+
+                                # the non-lagger got
+                                # a ``trio.EndOfChannel``
+                                # because the ``tx`` below was closed
+                                assert len(lbrx._state.subs) == 1
+
+                                await lbrx.aclose()
+
+                                assert len(lbrx._state.subs) == 0
+
+                        except trio.ClosedResourceError:
+                            # only the fast sub will try to re-enter
+                            # iteration on the now closed bcaster
+                            assert task.name == 'sub_1'
+                            return
+
+                        except Lagged:
+                            lag_time = time.time() - start
+                            lags = laggers[task.name]
+                            print(
+                                f'restarting slow task {task.name} '
+                                f'that bailed out on {lags}:{value} '
+                                f'after {lag_time:.3f}')
+                            if lags <= retries:
+                                laggers[task.name] += 1
+                                continue
+                            else:
+                                print(
+                                    f'{task.name} was too slow and terminated '
+                                    f'on {lags}:{value}')
+                                return
+
+            async with trio.open_nursery() as nursery:
+
+                for i in range(1, num_laggers):
+
+                    task_name = f'sub_{i}'
+                    laggers[task_name] = 0
+                    nursery.start_soon(
+                        partial(
+                            sub_and_print,
+                            delay=i*0.001,
+                        ),
+                        name=task_name,
+                    )
+
+                # allow subs to sched
+                await trio.sleep(0.1)
+
+                async with tx:
+                    for i in cycle(range(size)):
+                        await tx.send(i)
+                        if len(brx._state.subs) == 2:
+                            # only one, the non lagger, sub is left
+                            break
+
+                # the non-lagger
+                assert laggers.pop('sub_1') == 0
+
+                for n, v in laggers.items():
+                    assert v == 4
+
+                assert tx._closed
+                assert not tx._state.open_send_channels
+
+                # check that "first" bcaster that we created
+                # above, never was iterated and is thus overrun
+                try:
+                    await brx.receive()
+                except Lagged:
+                    # expect tokio style index truncation
+                    seq = brx._state.subs[brx.key]
+                    assert seq == len(brx._state.queue) - 1
+
+                # all backpressured entries in the underlying
+                # channel should have been copied into the caster
+                # queue trailing-window
+                async for i in rx:
+                    print(f'bped: {i}')
+                    assert i in brx._state.queue
+
+                # should be noop
+                await brx.aclose()
+
+    trio.run(main)
+
+
+def test_first_recver_is_cancelled():
+
+    async def main():
+
+        # make sure it all works within the runtime
+        async with tractor.open_root_actor():
+
+            tx, rx = trio.open_memory_channel(1)
+            brx = broadcast_receiver(rx, 1)
+            cs = trio.CancelScope()
+
+            async def sub_and_recv():
+                with cs:
+                    async with brx.subscribe() as bc:
+                        async for value in bc:
+                            print(value)
+
+            async def cancel_and_send():
+                await trio.sleep(0.2)
+                cs.cancel()
+                await tx.send(1)
+
+            async with trio.open_nursery() as n:
+
+                n.start_soon(sub_and_recv)
+                await trio.sleep(0.1)
+                assert brx._state.recv_ready
+
+                n.start_soon(cancel_and_send)
+
+                # ensure that we don't hang because no-task is now
+                # waiting on the underlying receive..
+                with trio.fail_after(0.5):
+                    value = await brx.receive()
+                    print(f'parent: {value}')
+                    assert value == 1
+
+    trio.run(main)
+
+
+def test_no_raise_on_lag():
+    '''
+    Run a simple 2-task broadcast where one task is slow but configured
+    so that it does not raise `Lagged` on overruns using
+    `raise_on_lasg=False` and verify that the task does not raise.
+
+    '''
+    size = 100
+    tx, rx = trio.open_memory_channel(size)
+    brx = broadcast_receiver(rx, size)
+
+    async def slow():
+        async with brx.subscribe(
+            raise_on_lag=False,
+        ) as br:
+            async for msg in br:
+                print(f'slow task got: {msg}')
+                await trio.sleep(0.1)
+
+    async def fast():
+        async with brx.subscribe() as br:
+            async for msg in br:
+                print(f'fast task got: {msg}')
+
+    async def main():
+        async with (
+            tractor.open_root_actor(
+                # NOTE: so we see the warning msg emitted by the bcaster
+                # internals when the no raise flag is set.
+                loglevel='warning',
+            ),
+            trio.open_nursery() as n,
+        ):
+            n.start_soon(slow)
+            n.start_soon(fast)
+
+            for i in range(1000):
+                await tx.send(i)
+
+            # simulate user nailing ctl-c after realizing
+            # there's a lag in the slow task.
+            await trio.sleep(1)
+            raise KeyboardInterrupt
+
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(main)

tests/test_trioisms.py

@@ -0,0 +1,82 @@
+'''
+Reminders for oddities in `trio` that we need to stay aware of and/or
+want to see changed.
+
+'''
+import pytest
+import trio
+from trio_typing import TaskStatus
+
+
+@pytest.mark.parametrize(
+    'use_start_soon', [
+        pytest.param(
+            True,
+            marks=pytest.mark.xfail(reason="see python-trio/trio#2258")
+        ),
+        False,
+    ]
+)
+def test_stashed_child_nursery(use_start_soon):
+
+    _child_nursery = None
+
+    async def waits_on_signal(
+        ev: trio.Event(),
+        task_status: TaskStatus[trio.Nursery] = trio.TASK_STATUS_IGNORED,
+    ):
+        '''
+        Do some stuf, then signal other tasks, then yield back to "starter".
+
+        '''
+        await ev.wait()
+        task_status.started()
+
+    async def mk_child_nursery(
+        task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
+    ):
+        '''
+        Allocate a child sub-nursery and stash it as a global.
+
+        '''
+        nonlocal _child_nursery
+
+        async with trio.open_nursery() as cn:
+            _child_nursery = cn
+            task_status.started(cn)
+
+            # block until cancelled by parent.
+            await trio.sleep_forever()
+
+    async def sleep_and_err(
+        ev: trio.Event,
+        task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
+    ):
+        await trio.sleep(0.5)
+        doggy()  # noqa
+        ev.set()
+        task_status.started()
+
+    async def main():
+
+        async with (
+            trio.open_nursery() as pn,
+        ):
+            cn = await pn.start(mk_child_nursery)
+            assert cn
+
+            ev = trio.Event()
+
+            if use_start_soon:
+                # this causes inf hang
+                cn.start_soon(sleep_and_err, ev)
+
+            else:
+                # this does not.
+                await cn.start(sleep_and_err, ev)
+
+            with trio.fail_after(1):
+                await cn.start(waits_on_signal, ev)
+
+    with pytest.raises(NameError):
+        trio.run(main)

tractor/__init__.py

@@ -1,93 +1,86 @@
-"""
-tractor: An actor model micro-framework built on
-         ``trio`` and ``multiprocessing``.
-"""
-from functools import partial
-import typing
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
 
-import trio  # type: ignore
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
 
-from .log import get_console_log, get_logger, get_loglevel
-from ._ipc import _connect_chan, Channel
-from ._actor import (
-    Actor, _start_actor, Arbiter, get_arbiter, find_actor, wait_for_actor
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+tractor: structured concurrent "actors".
+
+"""
+from exceptiongroup import BaseExceptionGroup
+
+from ._clustering import open_actor_cluster
+from ._ipc import Channel
+from ._streaming import (
+    Context,
+    MsgStream,
+    stream,
+    context,
 )
-from ._trionics import open_nursery
-from ._state import current_actor
-from ._portal import RemoteActorError
+from ._discovery import (
+    get_arbiter,
+    find_actor,
+    wait_for_actor,
+    query_actor,
+)
+from ._supervise import open_nursery
+from ._state import (
+    current_actor,
+    is_root_process,
+)
+from ._exceptions import (
+    RemoteActorError,
+    ModuleNotExposed,
+    ContextCancelled,
+)
+from ._debug import (
+    breakpoint,
+    post_mortem,
+)
+from . import msg
+from ._root import (
+    run_daemon,
+    open_root_actor,
+)
+from ._portal import Portal
+from ._runtime import Actor
 
 
 __all__ = [
+    'Actor',
+    'Channel',
+    'Context',
+    'ContextCancelled',
+    'ModuleNotExposed',
+    'MsgStream',
+    'BaseExceptionGroup',
+    'Portal',
+    'RemoteActorError',
+    'breakpoint',
+    'context',
     'current_actor',
     'find_actor',
     'get_arbiter',
-    'wait_for_actor',
+    'is_root_process',
+    'msg',
+    'open_actor_cluster',
     'open_nursery',
-    'RemoteActorError',
-    'Channel',
+    'open_root_actor',
+    'post_mortem',
+    'query_actor',
+    'run_daemon',
+    'stream',
+    'to_asyncio',
+    'wait_for_actor',
 ]
-
-
-# set at startup and after forks
-_default_arbiter_host = '127.0.0.1'
-_default_arbiter_port = 1616
-
-
-async def _main(
-    async_fn: typing.Callable[..., typing.Awaitable],
-    args: typing.Tuple,
-    kwargs: typing.Dict[str, typing.Any],
-    name: str,
-    arbiter_addr: typing.Tuple[str, int]
-) -> typing.Any:
-    """Async entry point for ``tractor``.
-    """
-    log = get_logger('tractor')
-    main = partial(async_fn, *args) if async_fn else None
-    arbiter_addr = (host, port) = arbiter_addr or (
-            _default_arbiter_host, _default_arbiter_port)
-    get_console_log(kwargs.get('loglevel', get_loglevel()))
-
-    # make a temporary connection to see if an arbiter exists
-    arbiter_found = False
-    try:
-        async with _connect_chan(host, port):
-            arbiter_found = True
-    except OSError:
-        log.warn(f"No actor could be found @ {host}:{port}")
-
-    # create a local actor and start up its main routine/task
-    if arbiter_found:  # we were able to connect to an arbiter
-        log.info(f"Arbiter seems to exist @ {host}:{port}")
-        actor = Actor(
-            name or 'anonymous',
-            arbiter_addr=arbiter_addr,
-            **kwargs
-        )
-        host, port = (host, 0)
-    else:
-        # start this local actor as the arbiter
-        actor = Arbiter(
-            name or 'arbiter', arbiter_addr=arbiter_addr, **kwargs)
-
-    # ``Actor._async_main()`` creates an internal nursery if one is not
-    # provided and thus blocks here until it's main task completes.
-    # Note that if the current actor is the arbiter it is desirable
-    # for it to stay up indefinitely until a re-election process has
-    # taken place - which is not implemented yet FYI).
-    return await _start_actor(
-        actor, main, host, port, arbiter_addr=arbiter_addr)
-
-
-def run(
-    async_fn: typing.Callable[..., typing.Awaitable],
-    *args: typing.Tuple,
-    name: str = None,
-    arbiter_addr: typing.Tuple[str, int] = (_default_arbiter_host, _default_arbiter_port),
-    **kwargs: typing.Dict[str, typing.Any],
-):
-    """Run a trio-actor async function in process.
-
-    This is tractor's main entry and the start point for any async actor.
-    """
-    return trio.run(_main, async_fn, args, kwargs, name, arbiter_addr)

tractor/_actor.py

@@ -1,780 +0,0 @@
-"""
-Actor primitives and helpers
-"""
-from collections import defaultdict
-from functools import partial
-from itertools import chain
-import importlib
-import inspect
-import traceback
-import uuid
-import typing
-from typing import Dict, List, Tuple, Any, Optional, Union
-
-import trio  # type: ignore
-from async_generator import asynccontextmanager, aclosing
-
-from ._ipc import Channel, _connect_chan
-from .log import get_console_log, get_logger
-from ._portal import (
-    Portal,
-    open_portal,
-    _do_handshake,
-    LocalPortal,
-)
-from . import _state
-from ._state import current_actor
-
-
-log = get_logger('tractor')
-
-
-class ActorFailure(Exception):
-    "General actor failure"
-
-
-class InternalActorError(RuntimeError):
-    "Actor primitive internals failure"
-
-
-async def _invoke(
-    actor: 'Actor',
-    cid: str,
-    chan: Channel,
-    func: typing.Callable,
-    kwargs: Dict[str, Any],
-    task_status=trio.TASK_STATUS_IGNORED
-):
-    """Invoke local func and return results over provided channel.
-    """
-    sig = inspect.signature(func)
-    treat_as_gen = False
-    if 'chan' in sig.parameters:
-        assert 'cid' in sig.parameters, \
-            f"{func} must accept a `cid` (caller id) kwarg"
-        kwargs['chan'] = chan
-        kwargs['cid'] = cid
-        # TODO: eventually we want to be more stringent
-        # about what is considered a far-end async-generator.
-        # Right now both actual async gens and any async
-        # function which declares a `chan` kwarg in its
-        # signature will be treated as one.
-        treat_as_gen = True
-    try:
-        is_async_partial = False
-        is_async_gen_partial = False
-        if isinstance(func, partial):
-            is_async_partial = inspect.iscoroutinefunction(func.func)
-            is_async_gen_partial = inspect.isasyncgenfunction(func.func)
-
-        if (
-            not inspect.iscoroutinefunction(func) and
-            not inspect.isasyncgenfunction(func) and
-            not is_async_partial and
-            not is_async_gen_partial
-        ):
-            await chan.send({'functype': 'function', 'cid': cid})
-            with trio.open_cancel_scope() as cs:
-                task_status.started(cs)
-                await chan.send({'return': func(**kwargs), 'cid': cid})
-        else:
-            coro = func(**kwargs)
-
-            if inspect.isasyncgen(coro):
-                await chan.send({'functype': 'asyncgen', 'cid': cid})
-                # XXX: massive gotcha! If the containing scope
-                # is cancelled and we execute the below line,
-                # any ``ActorNursery.__aexit__()`` WON'T be
-                # triggered in the underlying async gen! So we
-                # have to properly handle the closing (aclosing)
-                # of the async gen in order to be sure the cancel
-                # is propagated!
-                with trio.open_cancel_scope() as cs:
-                    task_status.started(cs)
-                    async with aclosing(coro) as agen:
-                        async for item in agen:
-                            # TODO: can we send values back in here?
-                            # it's gonna require a `while True:` and
-                            # some non-blocking way to retrieve new `asend()`
-                            # values from the channel:
-                            # to_send = await chan.recv_nowait()
-                            # if to_send is not None:
-                            #     to_yield = await coro.asend(to_send)
-                            await chan.send({'yield': item, 'cid': cid})
-
-                log.debug(f"Finished iterating {coro}")
-                # TODO: we should really support a proper
-                # `StopAsyncIteration` system here for returning a final
-                # value if desired
-                await chan.send({'stop': None, 'cid': cid})
-            else:
-                if treat_as_gen:
-                    await chan.send({'functype': 'asyncgen', 'cid': cid})
-                    # XXX: the async-func may spawn further tasks which push
-                    # back values like an async-generator would but must
-                    # manualy construct the response dict-packet-responses as
-                    # above
-                    with trio.open_cancel_scope() as cs:
-                        task_status.started(cs)
-                        await coro
-                else:
-                    await chan.send({'functype': 'asyncfunction', 'cid': cid})
-                    with trio.open_cancel_scope() as cs:
-                        task_status.started(cs)
-                        await chan.send({'return': await coro, 'cid': cid})
-    except Exception:
-        # always ship errors back to caller
-        log.exception("Actor errored:")
-        await chan.send({'error': traceback.format_exc(), 'cid': cid})
-    finally:
-        # RPC task bookeeping
-        tasks = actor._rpc_tasks.get(chan, None)
-        if tasks:
-            tasks.remove((cs, func))
-
-        if not tasks:
-            actor._rpc_tasks.pop(chan, None)
-
-        if not actor._rpc_tasks:
-            log.info(f"All RPC tasks have completed")
-            actor._no_more_rpc_tasks.set()
-
-
-class Actor:
-    """The fundamental concurrency primitive.
-
-    An *actor* is the combination of a regular Python or
-    ``multiprocessing.Process`` executing a ``trio`` task tree, communicating
-    with other actors through "portals" which provide a native async API
-    around "channels".
-    """
-    is_arbiter = False
-
-    def __init__(
-        self,
-        name: str,
-        rpc_module_paths: List[str] = [],
-        statespace: Optional[Dict[str, Any]] = None,
-        uid: str = None,
-        loglevel: str = None,
-        arbiter_addr: Optional[Tuple[str, int]] = None,
-    ) -> None:
-        self.name = name
-        self.uid = (name, uid or str(uuid.uuid1()))
-        self.rpc_module_paths = rpc_module_paths
-        self._mods: dict = {}
-        # TODO: consider making this a dynamically defined
-        # @dataclass once we get py3.7
-        self.statespace = statespace or {}
-        self.loglevel = loglevel
-        self._arb_addr = arbiter_addr
-
-        # filled in by `_async_main` after fork
-        self._root_nursery: trio._core._run.Nursery = None
-        self._server_nursery: trio._core._run.Nursery = None
-        self._peers: defaultdict = defaultdict(list)
-        self._peer_connected: dict = {}
-        self._no_more_peers = trio.Event()
-        self._no_more_peers.set()
-
-        self._no_more_rpc_tasks = trio.Event()
-        self._no_more_rpc_tasks.set()
-        self._rpc_tasks: Dict[
-            Channel,
-            List[Tuple[trio._core._run.CancelScope, typing.Callable]]
-        ] = {}
-        # map {uids -> {callids -> waiter queues}}
-        self._actors2calls: Dict[Tuple[str, str], Dict[str, trio.Queue]] = {}
-        self._listeners: List[trio.abc.Listener] = []
-        self._parent_chan: Optional[Channel] = None
-        self._forkserver_info: Optional[Tuple[Any, Any, Any, Any, Any]] = None
-
-    async def wait_for_peer(
-        self, uid: Tuple[str, str]
-    ) -> Tuple[trio.Event, Channel]:
-        """Wait for a connection back from a spawned actor with a given
-        ``uid``.
-        """
-        log.debug(f"Waiting for peer {uid} to connect")
-        event = self._peer_connected.setdefault(uid, trio.Event())
-        await event.wait()
-        log.debug(f"{uid} successfully connected back to us")
-        return event, self._peers[uid][-1]
-
-    def load_modules(self) -> None:
-        """Load allowed RPC modules locally (after fork).
-
-        Since this actor may be spawned on a different machine from
-        the original nursery we need to try and load the local module
-        code (if it exists).
-        """
-        for path in self.rpc_module_paths:
-            self._mods[path] = importlib.import_module(path)
-
-            # XXX: triggers an internal error which can cause a hanging
-            # problem (without the recently added .throw()) on teardown
-            # (root nursery tears down thus killing all channels before
-            # sending cancels to subactors during actor nursery teardown
-            # - has to do with await main() in MainProcess)
-            # if self.name == 'gretchen':
-            #     self._mods.pop('test_discovery')
-
-    async def _stream_handler(
-        self,
-        stream: trio.SocketStream,
-    ) -> None:
-        """Entry point for new inbound connections to the channel server.
-        """
-        self._no_more_peers.clear()
-        chan = Channel(stream=stream)
-        log.info(f"New connection to us {chan}")
-
-        # send/receive initial handshake response
-        try:
-            uid = await _do_handshake(self, chan)
-        except StopAsyncIteration:
-            log.warn(f"Channel {chan} failed to handshake")
-            return
-
-        # channel tracking
-        event = self._peer_connected.pop(uid, None)
-        if event:
-            # Instructing connection: this is likely a new channel to
-            # a recently spawned actor which we'd like to control via
-            # async-rpc calls.
-            log.debug(f"Waking channel waiters {event.statistics()}")
-            # Alert any task waiting on this connection to come up
-            event.set()
-
-        chans = self._peers[uid]
-        if chans:
-            log.warn(
-                f"already have channel(s) for {uid}:{chans}?"
-            )
-        log.debug(f"Registered {chan} for {uid}")
-        # append new channel
-        self._peers[uid].append(chan)
-
-        # Begin channel management - respond to remote requests and
-        # process received reponses.
-        try:
-            await self._process_messages(chan)
-        finally:
-            # Drop ref to channel so it can be gc-ed and disconnected
-            log.debug(f"Releasing channel {chan} from {chan.uid}")
-            chans = self._peers.get(chan.uid)
-            chans.remove(chan)
-            if not chans:
-                log.debug(f"No more channels for {chan.uid}")
-                self._peers.pop(chan.uid, None)
-
-            log.debug(f"Peers is {self._peers}")
-
-            if not self._peers:  # no more channels connected
-                self._no_more_peers.set()
-                log.debug(f"Signalling no more peer channels")
-
-            # # XXX: is this necessary (GC should do it?)
-            if chan.connected():
-                log.debug(f"Disconnecting channel {chan}")
-                await chan.send(None)
-                await chan.aclose()
-
-    async def _push_result(self, actorid, cid: str, msg: dict) -> None:
-        """Push an RPC result to the local consumer's queue.
-        """
-        assert actorid, f"`actorid` can't be {actorid}"
-        q = self.get_waitq(actorid, cid)
-        log.debug(f"Delivering {msg} from {actorid} to caller {cid}")
-        # maintain backpressure
-        await q.put(msg)
-
-    def get_waitq(
-        self,
-        actorid: Tuple[str, str],
-        cid: str
-    ) -> trio.Queue:
-        log.debug(f"Getting result queue for {actorid} cid {cid}")
-        cids2qs = self._actors2calls.setdefault(actorid, {})
-        return cids2qs.setdefault(cid, trio.Queue(1000))
-
-    async def send_cmd(
-        self, chan: Channel, ns: str, func: str, kwargs: dict
-    ) -> Tuple[str, trio.Queue]:
-        """Send a ``'cmd'`` message to a remote actor and return a
-        caller id and a ``trio.Queue`` that can be used to wait for
-        responses delivered by the local message processing loop.
-        """
-        cid = str(uuid.uuid1())
-        assert chan.uid
-        q = self.get_waitq(chan.uid, cid)
-        log.debug(f"Sending cmd to {chan.uid}: {ns}.{func}({kwargs})")
-        await chan.send({'cmd': (ns, func, kwargs, self.uid, cid)})
-        return cid, q
-
-    async def _process_messages(
-        self, chan: Channel, treat_as_gen: bool = False
-    ) -> None:
-        """Process messages for the channel async-RPC style.
-
-        Receive multiplexed RPC requests and deliver responses over ``chan``.
-        """
-        # TODO: once https://github.com/python-trio/trio/issues/467 gets
-        # worked out we'll likely want to use that!
-        log.debug(f"Entering msg loop for {chan} from {chan.uid}")
-        try:
-            async for msg in chan:
-                if msg is None:  # terminate sentinel
-                    log.debug(
-                        f"Cancelling all tasks for {chan} from {chan.uid}")
-                    for scope, func in self._rpc_tasks.pop(chan, ()):
-                        scope.cancel()
-
-                    log.debug(
-                            f"Msg loop signalled to terminate for"
-                            f" {chan} from {chan.uid}")
-                    break
-                log.debug(f"Received msg {msg} from {chan.uid}")
-                cid = msg.get('cid')
-                if cid:
-                    # deliver response to local caller/waiter
-                    await self._push_result(chan.uid, cid, msg)
-                    log.debug(
-                        f"Waiting on next msg for {chan} from {chan.uid}")
-                    continue
-
-                # process command request
-                try:
-                    ns, funcname, kwargs, actorid, cid = msg['cmd']
-                except KeyError:
-                    # push any non-rpc-response error to all local consumers
-                    # and mark the channel as errored
-                    chan._exc = err = msg['error']
-                    assert chan.uid
-                    for cid in self._actors2calls[chan.uid]:
-                        await self._push_result(chan.uid, cid, msg)
-                    raise InternalActorError(f"{chan.uid}\n" + err)
-
-                log.debug(
-                    f"Processing request from {actorid}\n"
-                    f"{ns}.{funcname}({kwargs})")
-                if ns == 'self':
-                    func = getattr(self, funcname)
-                else:
-                    func = getattr(self._mods[ns], funcname)
-
-                # spin up a task for the requested function
-                log.debug(f"Spawning task for {func}")
-                cs = await self._root_nursery.start(
-                    _invoke, self, cid, chan, func, kwargs,
-                    name=funcname
-                )
-                # never allow cancelling cancel requests (results in
-                # deadlock and other weird behaviour)
-                if func != self.cancel:
-                    self._no_more_rpc_tasks.clear()
-                    log.info(f"RPC func is {func}")
-                    self._rpc_tasks.setdefault(chan, []).append((cs, func))
-                log.debug(
-                    f"Waiting on next msg for {chan} from {chan.uid}")
-            else:  # channel disconnect
-                log.debug(f"{chan} from {chan.uid} disconnected")
-
-        except trio.ClosedResourceError:
-            log.error(f"{chan} form {chan.uid} broke")
-        except Exception:
-            # ship exception (from above code) to parent
-            log.exception("Actor errored:")
-            if self._parent_chan:
-                await self._parent_chan.send({'error': traceback.format_exc()})
-                raise
-            # if this is the `MainProcess` we expect the error broadcasting
-            # above to trigger an error at consuming portal "checkpoints"
-        finally:
-            log.debug(f"Exiting msg loop for {chan} from {chan.uid}")
-
-    def _fork_main(
-        self,
-        accept_addr: Tuple[str, int],
-        forkserver_info: Tuple[Any, Any, Any, Any, Any],
-        parent_addr: Tuple[str, int] = None
-    ) -> None:
-        # after fork routine which invokes a fresh ``trio.run``
-        # log.warn("Log level after fork is {self.loglevel}")
-        self._forkserver_info = forkserver_info
-        from ._trionics import ctx
-        if self.loglevel is not None:
-            get_console_log(self.loglevel)
-        log.info(
-            f"Started new {ctx.current_process()} for {self.uid}")
-        _state._current_actor = self
-        log.debug(f"parent_addr is {parent_addr}")
-        try:
-            trio.run(partial(
-                self._async_main, accept_addr, parent_addr=parent_addr))
-        except KeyboardInterrupt:
-            pass  # handle it the same way trio does?
-        log.info(f"Actor {self.uid} terminated")
-
-    async def _async_main(
-        self,
-        accept_addr: Tuple[str, int],
-        arbiter_addr: Optional[Tuple[str, int]] = None,
-        parent_addr: Optional[Tuple[str, int]] = None,
-        task_status: trio._core._run._TaskStatus = trio.TASK_STATUS_IGNORED,
-    ) -> None:
-        """Start the channel server, maybe connect back to the parent, and
-        start the main task.
-
-        A "root-most" (or "top-level") nursery for this actor is opened here
-        and when cancelled effectively cancels the actor.
-        """
-        arbiter_addr = arbiter_addr or self._arb_addr
-        registered_with_arbiter = False
-        try:
-            async with trio.open_nursery() as nursery:
-                self._root_nursery = nursery
-
-                # load allowed RPC module
-                self.load_modules()
-
-                # Startup up channel server
-                host, port = accept_addr
-                await nursery.start(partial(
-                    self._serve_forever, accept_host=host, accept_port=port)
-                )
-
-                if parent_addr is not None:
-                    try:
-                        # Connect back to the parent actor and conduct initial
-                        # handshake (From this point on if we error ship the
-                        # exception back to the parent actor)
-                        chan = self._parent_chan = Channel(
-                            destaddr=parent_addr,
-                        )
-                        await chan.connect()
-                        # initial handshake, report who we are, who they are
-                        await _do_handshake(self, chan)
-                    except OSError:  # failed to connect
-                        log.warn(
-                            f"Failed to connect to parent @ {parent_addr},"
-                            " closing server")
-                        await self.cancel()
-                        self._parent_chan = None
-
-                    # handle new connection back to parent
-                    nursery.start_soon(
-                        self._process_messages, self._parent_chan)
-
-                # register with the arbiter if we're told its addr
-                log.debug(f"Registering {self} for role `{self.name}`")
-                async with get_arbiter(*arbiter_addr) as arb_portal:
-                    await arb_portal.run(
-                        'self', 'register_actor',
-                        uid=self.uid, sockaddr=self.accept_addr)
-                    registered_with_arbiter = True
-
-                task_status.started()
-                log.debug("Waiting on root nursery to complete")
-            # blocks here as expected if no nursery was provided until
-            # the channel server is killed (i.e. this actor is
-            # cancelled or signalled by the parent actor)
-        except Exception:
-            if self._parent_chan:
-                try:
-                    await self._parent_chan.send(
-                        # {'error': traceback.format_exc(), 'cid': 'internal'})
-                        {'error': traceback.format_exc()})
-                except trio.ClosedResourceError:
-                    log.error(
-                        f"Failed to ship error to parent "
-                        f"{self._parent_chan.uid}, channel was closed")
-                    log.exception("Actor errored:")
-
-            if not registered_with_arbiter:
-                log.exception(
-                    f"Actor errored and failed to register with arbiter "
-                    f"@ {arbiter_addr}")
-            else:
-                raise
-        finally:
-            await self._do_unreg(arbiter_addr)
-            # terminate actor once all it's peers (actors that connected
-            # to it as clients) have disappeared
-            if not self._no_more_peers.is_set():
-                if any(
-                    chan.connected() for chan in chain(*self._peers.values())
-                ):
-                    log.debug(
-                        f"Waiting for remaining peers {self._peers} to clear")
-                    await self._no_more_peers.wait()
-            log.debug(f"All peer channels are complete")
-
-            # tear down channel server no matter what since we errored
-            # or completed
-            self.cancel_server()
-
-    async def _serve_forever(
-        self,
-        *,
-        # (host, port) to bind for channel server
-        accept_host: Tuple[str, int] = None,
-        accept_port: int = 0,
-        task_status: trio._core._run._TaskStatus = trio.TASK_STATUS_IGNORED,
-    ) -> None:
-        """Start the channel server, begin listening for new connections.
-
-        This will cause an actor to continue living (blocking) until
-        ``cancel_server()`` is called.
-        """
-        async with trio.open_nursery() as nursery:
-            self._server_nursery = nursery
-            # TODO: might want to consider having a separate nursery
-            # for the stream handler such that the server can be cancelled
-            # whilst leaving existing channels up
-            listeners = await nursery.start(
-                partial(
-                    trio.serve_tcp,
-                    self._stream_handler,
-                    # new connections will stay alive even if this server
-                    # is cancelled
-                    handler_nursery=self._root_nursery,
-                    port=accept_port, host=accept_host,
-                )
-            )
-            log.debug(
-                f"Started tcp server(s) on {[l.socket for l in listeners]}")
-            self._listeners.extend(listeners)
-            task_status.started()
-
-    async def _do_unreg(self, arbiter_addr: Optional[Tuple[str, int]]) -> None:
-        # UNregister actor from the arbiter
-        try:
-            if arbiter_addr is not None:
-                async with get_arbiter(*arbiter_addr) as arb_portal:
-                    await arb_portal.run(
-                        'self', 'unregister_actor', uid=self.uid)
-        except OSError:
-            log.warn(f"Unable to unregister {self.name} from arbiter")
-
-    async def cancel(self) -> None:
-        """Cancel this actor.
-
-        The sequence in order is:
-            - cancelling all rpc tasks
-            - cancelling the channel server
-            - cancel the "root" nursery
-        """
-        # cancel all ongoing rpc tasks
-        await self.cancel_rpc_tasks()
-        self.cancel_server()
-        self._root_nursery.cancel_scope.cancel()
-
-    async def cancel_rpc_tasks(self) -> None:
-        """Cancel all existing RPC responder tasks using the cancel scope
-        registered for each.
-        """
-        tasks = self._rpc_tasks
-        log.info(f"Cancelling all {len(tasks)} rpc tasks:\n{tasks}")
-        for chan, scopes in tasks.items():
-            log.debug(f"Cancelling all tasks for {chan.uid}")
-            for scope, func in scopes:
-                log.debug(f"Cancelling task for {func}")
-                scope.cancel()
-        if tasks:
-            log.info(
-                f"Waiting for remaining rpc tasks to complete {tasks}")
-            await self._no_more_rpc_tasks.wait()
-
-    def cancel_server(self) -> None:
-        """Cancel the internal channel server nursery thereby
-        preventing any new inbound connections from being established.
-        """
-        log.debug("Shutting down channel server")
-        self._server_nursery.cancel_scope.cancel()
-
-    @property
-    def accept_addr(self) -> Optional[Tuple[str, int]]:
-        """Primary address to which the channel server is bound.
-        """
-        try:
-            return self._listeners[0].socket.getsockname()
-        except OSError:
-            return None
-
-    def get_parent(self) -> Portal:
-        """Return a portal to our parent actor."""
-        assert self._parent_chan, "No parent channel for this actor?"
-        return Portal(self._parent_chan)
-
-    def get_chans(self, uid: Tuple[str, str]) -> List[Channel]:
-        """Return all channels to the actor with provided uid."""
-        return self._peers[uid]
-
-
-class Arbiter(Actor):
-    """A special actor who knows all the other actors and always has
-    access to a top level nursery.
-
-    The arbiter is by default the first actor spawned on each host
-    and is responsible for keeping track of all other actors for
-    coordination purposes. If a new main process is launched and an
-    arbiter is already running that arbiter will be used.
-    """
-    is_arbiter = True
-
-    def __init__(self, *args, **kwargs):
-        self._registry = defaultdict(list)
-        self._waiters = {}
-        super().__init__(*args, **kwargs)
-
-    def find_actor(self, name: str) -> Optional[Tuple[str, int]]:
-        for uid, sockaddr in self._registry.items():
-            if name in uid:
-                return sockaddr
-
-        return None
-
-    async def wait_for_actor(
-        self, name: str
-    ) -> List[Tuple[str, int]]:
-        """Wait for a particular actor to register.
-
-        This is a blocking call if no actor by the provided name is currently
-        registered.
-        """
-        sockaddrs = []
-
-        for (aname, _), sockaddr in self._registry.items():
-            if name == aname:
-                sockaddrs.append(sockaddr)
-
-        if not sockaddrs:
-            waiter = trio.Event()
-            self._waiters.setdefault(name, []).append(waiter)
-            await waiter.wait()
-            for uid in self._waiters[name]:
-                sockaddrs.append(self._registry[uid])
-
-        return sockaddrs
-
-    def register_actor(
-        self, uid: Tuple[str, str], sockaddr: Tuple[str, int]
-    ) -> None:
-        name, uuid = uid
-        self._registry[uid] = sockaddr
-
-        # pop and signal all waiter events
-        events = self._waiters.pop(name, ())
-        self._waiters.setdefault(name, []).append(uid)
-        for event in events:
-            event.set()
-
-    def unregister_actor(self, uid: Tuple[str, str]) -> None:
-        self._registry.pop(uid, None)
-
-
-async def _start_actor(
-    actor: Actor,
-    main: typing.Callable[..., typing.Awaitable],
-    host: str,
-    port: int,
-    arbiter_addr: Tuple[str, int],
-    nursery: trio._core._run.Nursery = None
-):
-    """Spawn a local actor by starting a task to execute it's main async
-    function.
-
-    Blocks if no nursery is provided, in which case it is expected the nursery
-    provider is responsible for waiting on the task to complete.
-    """
-    # assign process-local actor
-    _state._current_actor = actor
-
-    # start local channel-server and fake the portal API
-    # NOTE: this won't block since we provide the nursery
-    log.info(f"Starting local {actor} @ {host}:{port}")
-
-    async with trio.open_nursery() as nursery:
-        await nursery.start(
-            partial(
-                actor._async_main,
-                accept_addr=(host, port),
-                parent_addr=None,
-                arbiter_addr=arbiter_addr,
-            )
-        )
-        if main is not None:
-            result = await main()
-
-        # XXX: If spawned with a dedicated "main function",
-        # the actor is cancelled when this context is complete
-        # given that there are no more active peer channels connected
-        actor.cancel_server()
-
-    # unset module state
-    _state._current_actor = None
-    log.info("Completed async main")
-
-    return result
-
-
-@asynccontextmanager
-async def get_arbiter(
-    host: str, port: int
-) -> typing.AsyncGenerator[Union[Portal, LocalPortal], None]:
-    """Return a portal instance connected to a local or remote
-    arbiter.
-    """
-    actor = current_actor()
-    if not actor:
-        raise RuntimeError("No actor instance has been defined yet?")
-
-    if actor.is_arbiter:
-        # we're already the arbiter
-        # (likely a re-entrant call from the arbiter actor)
-        yield LocalPortal(actor)
-    else:
-        async with _connect_chan(host, port) as chan:
-            async with open_portal(chan) as arb_portal:
-                yield arb_portal
-
-
-@asynccontextmanager
-async def find_actor(
-    name: str, arbiter_sockaddr: Tuple[str, int] = None
-) -> typing.AsyncGenerator[Optional[Portal], None]:
-    """Ask the arbiter to find actor(s) by name.
-
-    Returns a connected portal to the last registered matching actor
-    known to the arbiter.
-    """
-    actor = current_actor()
-    async with get_arbiter(*arbiter_sockaddr or actor._arb_addr) as arb_portal:
-        sockaddr = await arb_portal.run('self', 'find_actor', name=name)
-        # TODO: return portals to all available actors - for now just
-        # the last one that registered
-        if sockaddr:
-            async with _connect_chan(*sockaddr) as chan:
-                async with open_portal(chan) as portal:
-                    yield portal
-        else:
-            yield None
-
-
-@asynccontextmanager
-async def wait_for_actor(
-    name: str,
-    arbiter_sockaddr: Tuple[str, int] = None
-) -> typing.AsyncGenerator[Portal, None]:
-    """Wait on an actor to register with the arbiter.
-
-    A portal to the first actor which registered is be returned.
-    """
-    actor = current_actor()
-    async with get_arbiter(*arbiter_sockaddr or actor._arb_addr) as arb_portal:
-        sockaddrs = await arb_portal.run('self', 'wait_for_actor', name=name)
-        sockaddr = sockaddrs[-1]
-        async with _connect_chan(*sockaddr) as chan:
-            async with open_portal(chan) as portal:
-                yield portal

tractor/_child.py

@@ -0,0 +1,62 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+This is the "bootloader" for actors started using the native trio backend.
+
+"""
+import sys
+import trio
+import argparse
+
+from ast import literal_eval
+
+from ._runtime import Actor
+from ._entry import _trio_main
+
+
+def parse_uid(arg):
+    name, uuid = literal_eval(arg)  # ensure 2 elements
+    return str(name), str(uuid)  # ensures str encoding
+
+def parse_ipaddr(arg):
+    host, port = literal_eval(arg)
+    return (str(host), int(port))
+
+
+from ._entry import _trio_main
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--uid", type=parse_uid)
+    parser.add_argument("--loglevel", type=str)
+    parser.add_argument("--parent_addr", type=parse_ipaddr)
+    parser.add_argument("--asyncio", action='store_true')
+    args = parser.parse_args()
+
+    subactor = Actor(
+        args.uid[0],
+        uid=args.uid[1],
+        loglevel=args.loglevel,
+        spawn_method="trio"
+    )
+
+    _trio_main(
+        subactor,
+        parent_addr=args.parent_addr,
+        infect_asyncio=args.asyncio,
+    )

tractor/_clustering.py

@@ -0,0 +1,74 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Actor cluster helpers.
+
+'''
+from __future__ import annotations
+
+from contextlib import asynccontextmanager as acm
+from multiprocessing import cpu_count
+from typing import AsyncGenerator, Optional
+
+import trio
+import tractor
+
+
+@acm
+async def open_actor_cluster(
+    modules: list[str],
+    count: int = cpu_count(),
+    names: list[str] | None = None,
+    hard_kill: bool = False,
+
+    # passed through verbatim to ``open_root_actor()``
+    **runtime_kwargs,
+
+) -> AsyncGenerator[
+    dict[str, tractor.Portal],
+    None,
+]:
+
+    portals: dict[str, tractor.Portal] = {}
+
+    if not names:
+        names = [f'worker_{i}' for i in range(count)]
+
+    if not len(names) == count:
+        raise ValueError(
+            'Number of names is {len(names)} but count it {count}')
+
+    async with tractor.open_nursery(
+        **runtime_kwargs,
+    ) as an:
+        async with trio.open_nursery() as n:
+            uid = tractor.current_actor().uid
+
+            async def _start(name: str) -> None:
+                name = f'{uid[0]}.{name}'
+                portals[name] = await an.start_actor(
+                    enable_modules=modules,
+                    name=name,
+                )
+
+            for name in names:
+                n.start_soon(_start, name)
+
+        assert len(portals) == count
+        yield portals
+
+        await an.cancel(hard_kill=hard_kill)

tractor/_debug.py

@@ -0,0 +1,922 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+Multi-core debugging for da peeps!
+
+"""
+from __future__ import annotations
+import bdb
+import os
+import sys
+import signal
+from functools import (
+    partial,
+    cached_property,
+)
+from contextlib import asynccontextmanager as acm
+from typing import (
+    Any,
+    Optional,
+    Callable,
+    AsyncIterator,
+    AsyncGenerator,
+)
+from types import FrameType
+
+import pdbp
+import tractor
+import trio
+from trio_typing import TaskStatus
+
+from .log import get_logger
+from ._discovery import get_root
+from ._state import (
+    is_root_process,
+    debug_mode,
+)
+from ._exceptions import (
+    is_multi_cancelled,
+    ContextCancelled,
+)
+from ._ipc import Channel
+
+log = get_logger(__name__)
+
+
+__all__ = ['breakpoint', 'post_mortem']
+
+
+class Lock:
+    '''
+    Actor global debug lock state.
+
+    Mostly to avoid a lot of ``global`` declarations for now XD.
+
+    '''
+    repl: MultiActorPdb | None = None
+    # placeholder for function to set a ``trio.Event`` on debugger exit
+    # pdb_release_hook: Optional[Callable] = None
+
+    _trio_handler: Callable[
+        [int, Optional[FrameType]], Any
+    ] | int | None = None
+
+    # actor-wide variable pointing to current task name using debugger
+    local_task_in_debug: str | None = None
+
+    # NOTE: set by the current task waiting on the root tty lock from
+    # the CALLER side of the `lock_tty_for_child()` context entry-call
+    # and must be cancelled if this actor is cancelled via IPC
+    # request-message otherwise deadlocks with the parent actor may
+    # ensure
+    _debugger_request_cs: Optional[trio.CancelScope] = None
+
+    # NOTE: set only in the root actor for the **local** root spawned task
+    # which has acquired the lock (i.e. this is on the callee side of
+    # the `lock_tty_for_child()` context entry).
+    _root_local_task_cs_in_debug: Optional[trio.CancelScope] = None
+
+    # actor tree-wide actor uid that supposedly has the tty lock
+    global_actor_in_debug: Optional[tuple[str, str]] = None
+
+    local_pdb_complete: Optional[trio.Event] = None
+    no_remote_has_tty: Optional[trio.Event] = None
+
+    # lock in root actor preventing multi-access to local tty
+    _debug_lock: trio.StrictFIFOLock = trio.StrictFIFOLock()
+
+    _orig_sigint_handler: Optional[Callable] = None
+    _blocked: set[tuple[str, str]] = set()
+
+    @classmethod
+    def shield_sigint(cls):
+        cls._orig_sigint_handler = signal.signal(
+            signal.SIGINT,
+            shield_sigint_handler,
+        )
+
+    @classmethod
+    def unshield_sigint(cls):
+        # always restore ``trio``'s sigint handler. see notes below in
+        # the pdb factory about the nightmare that is that code swapping
+        # out the handler when the repl activates...
+        signal.signal(signal.SIGINT, cls._trio_handler)
+        cls._orig_sigint_handler = None
+
+    @classmethod
+    def release(cls):
+        try:
+            cls._debug_lock.release()
+        except RuntimeError:
+            # uhhh makes no sense but been seeing the non-owner
+            # release error even though this is definitely the task
+            # that locked?
+            owner = cls._debug_lock.statistics().owner
+            if owner:
+                raise
+
+        # actor-local state, irrelevant for non-root.
+        cls.global_actor_in_debug = None
+        cls.local_task_in_debug = None
+
+        try:
+            # sometimes the ``trio`` might already be terminated in
+            # which case this call will raise.
+            if cls.local_pdb_complete is not None:
+                cls.local_pdb_complete.set()
+        finally:
+            # restore original sigint handler
+            cls.unshield_sigint()
+            cls.repl = None
+
+
+class TractorConfig(pdbp.DefaultConfig):
+    '''
+    Custom ``pdbp`` goodness :surfer:
+
+    '''
+    use_pygments: bool = True
+    sticky_by_default: bool = False
+    enable_hidden_frames: bool = False
+
+    # much thanks @mdmintz for the hot tip!
+    # fixes line spacing issue when resizing terminal B)
+    truncate_long_lines: bool = False
+
+
+class MultiActorPdb(pdbp.Pdb):
+    '''
+    Add teardown hooks to the regular ``pdbp.Pdb``.
+
+    '''
+    # override the pdbp config with our coolio one
+    DefaultConfig = TractorConfig
+
+    # def preloop(self):
+    #     print('IN PRELOOP')
+    #     super().preloop()
+
+    # TODO: figure out how to disallow recursive .set_trace() entry
+    # since that'll cause deadlock for us.
+    def set_continue(self):
+        try:
+            super().set_continue()
+        finally:
+            Lock.release()
+
+    def set_quit(self):
+        try:
+            super().set_quit()
+        finally:
+            Lock.release()
+
+    # XXX NOTE: we only override this because apparently the stdlib pdb
+    # bois likes to touch the SIGINT handler as much as i like to touch
+    # my d$%&.
+    def _cmdloop(self):
+        self.cmdloop()
+
+    @cached_property
+    def shname(self) -> str | None:
+        '''
+        Attempt to return the login shell name with a special check for
+        the infamous `xonsh` since it seems to have some issues much
+        different from std shells when it comes to flushing the prompt?
+
+        '''
+        # SUPER HACKY and only really works if `xonsh` is not used
+        # before spawning further sub-shells..
+        shpath = os.getenv('SHELL', None)
+
+        if shpath:
+            if (
+                os.getenv('XONSH_LOGIN', default=False)
+                or 'xonsh' in shpath
+            ):
+                return 'xonsh'
+
+            return os.path.basename(shpath)
+
+        return None
+
+
+@acm
+async def _acquire_debug_lock_from_root_task(
+    uid: tuple[str, str]
+
+) -> AsyncIterator[trio.StrictFIFOLock]:
+    '''
+    Acquire a root-actor local FIFO lock which tracks mutex access of
+    the process tree's global debugger breakpoint.
+
+    This lock avoids tty clobbering (by preventing multiple processes
+    reading from stdstreams) and ensures multi-actor, sequential access
+    to the ``pdb`` repl.
+
+    '''
+    task_name = trio.lowlevel.current_task().name
+
+    log.runtime(
+        f"Attempting to acquire TTY lock, remote task: {task_name}:{uid}"
+    )
+
+    we_acquired = False
+
+    try:
+        log.runtime(
+            f"entering lock checkpoint, remote task: {task_name}:{uid}"
+        )
+        we_acquired = True
+
+        # NOTE: if the surrounding cancel scope from the
+        # `lock_tty_for_child()` caller is cancelled, this line should
+        # unblock and NOT leave us in some kind of
+        # a "child-locked-TTY-but-child-is-uncontactable-over-IPC"
+        # condition.
+        await Lock._debug_lock.acquire()
+
+        if Lock.no_remote_has_tty is None:
+            # mark the tty lock as being in use so that the runtime
+            # can try to avoid clobbering any connection from a child
+            # that's currently relying on it.
+            Lock.no_remote_has_tty = trio.Event()
+
+        Lock.global_actor_in_debug = uid
+        log.runtime(f"TTY lock acquired, remote task: {task_name}:{uid}")
+
+        # NOTE: critical section: this yield is unshielded!
+
+        # IF we received a cancel during the shielded lock entry of some
+        # next-in-queue requesting task, then the resumption here will
+        # result in that ``trio.Cancelled`` being raised to our caller
+        # (likely from ``lock_tty_for_child()`` below)!  In
+        # this case the ``finally:`` below should trigger and the
+        # surrounding caller side context should cancel normally
+        # relaying back to the caller.
+
+        yield Lock._debug_lock
+
+    finally:
+        if (
+            we_acquired
+            and Lock._debug_lock.locked()
+        ):
+            Lock._debug_lock.release()
+
+        # IFF there are no more requesting tasks queued up fire, the
+        # "tty-unlocked" event thereby alerting any monitors of the lock that
+        # we are now back in the "tty unlocked" state. This is basically
+        # and edge triggered signal around an empty queue of sub-actor
+        # tasks that may have tried to acquire the lock.
+        stats = Lock._debug_lock.statistics()
+        if (
+            not stats.owner
+        ):
+            log.runtime(f"No more tasks waiting on tty lock! says {uid}")
+            if Lock.no_remote_has_tty is not None:
+                Lock.no_remote_has_tty.set()
+                Lock.no_remote_has_tty = None
+
+        Lock.global_actor_in_debug = None
+
+        log.runtime(
+            f"TTY lock released, remote task: {task_name}:{uid}"
+        )
+
+
+@tractor.context
+async def lock_tty_for_child(
+
+    ctx: tractor.Context,
+    subactor_uid: tuple[str, str]
+
+) -> str:
+    '''
+    Lock the TTY in the root process of an actor tree in a new
+    inter-actor-context-task such that the ``pdbp`` debugger console
+    can be mutex-allocated to the calling sub-actor for REPL control
+    without interference by other processes / threads.
+
+    NOTE: this task must be invoked in the root process of the actor
+    tree. It is meant to be invoked as an rpc-task and should be
+    highly reliable at releasing the mutex complete!
+
+    '''
+    task_name = trio.lowlevel.current_task().name
+
+    if tuple(subactor_uid) in Lock._blocked:
+        log.warning(
+            f'Actor {subactor_uid} is blocked from acquiring debug lock\n'
+            f"remote task: {task_name}:{subactor_uid}"
+        )
+        ctx._enter_debugger_on_cancel = False
+        await ctx.cancel(f'Debug lock blocked for {subactor_uid}')
+        return 'pdb_lock_blocked'
+
+    # TODO: when we get to true remote debugging
+    # this will deliver stdin data?
+
+    log.debug(
+        "Attempting to acquire TTY lock\n"
+        f"remote task: {task_name}:{subactor_uid}"
+    )
+
+    log.debug(f"Actor {subactor_uid} is WAITING on stdin hijack lock")
+    Lock.shield_sigint()
+
+    try:
+        with (
+            trio.CancelScope(shield=True) as debug_lock_cs,
+        ):
+            Lock._root_local_task_cs_in_debug = debug_lock_cs
+            async with _acquire_debug_lock_from_root_task(subactor_uid):
+
+                # indicate to child that we've locked stdio
+                await ctx.started('Locked')
+                log.debug(
+                    f"Actor {subactor_uid} acquired stdin hijack lock"
+                )
+
+                # wait for unlock pdb by child
+                async with ctx.open_stream() as stream:
+                    assert await stream.receive() == 'pdb_unlock'
+
+        return "pdb_unlock_complete"
+
+    finally:
+        Lock._root_local_task_cs_in_debug = None
+        Lock.unshield_sigint()
+
+
+async def wait_for_parent_stdin_hijack(
+    actor_uid: tuple[str, str],
+    task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED
+):
+    '''
+    Connect to the root actor via a ``Context`` and invoke a task which
+    locks a root-local TTY lock: ``lock_tty_for_child()``; this func
+    should be called in a new task from a child actor **and never the
+    root*.
+
+    This function is used by any sub-actor to acquire mutex access to
+    the ``pdb`` REPL and thus the root's TTY for interactive debugging
+    (see below inside ``_breakpoint()``). It can be used to ensure that
+    an intermediate nursery-owning actor does not clobber its children
+    if they are in debug (see below inside
+    ``maybe_wait_for_debugger()``).
+
+    '''
+    with trio.CancelScope(shield=True) as cs:
+        Lock._debugger_request_cs = cs
+
+        try:
+            async with get_root() as portal:
+
+                # this syncs to child's ``Context.started()`` call.
+                async with portal.open_context(
+
+                    tractor._debug.lock_tty_for_child,
+                    subactor_uid=actor_uid,
+
+                ) as (ctx, val):
+
+                    log.debug('locked context')
+                    assert val == 'Locked'
+
+                    async with ctx.open_stream() as stream:
+                        # unblock local caller
+
+                        try:
+                            assert Lock.local_pdb_complete
+                            task_status.started(cs)
+                            await Lock.local_pdb_complete.wait()
+
+                        finally:
+                            # TODO: shielding currently can cause hangs...
+                            # with trio.CancelScope(shield=True):
+                            await stream.send('pdb_unlock')
+
+                        # sync with callee termination
+                        assert await ctx.result() == "pdb_unlock_complete"
+
+                log.debug('exitting child side locking task context')
+
+        except ContextCancelled:
+            log.warning('Root actor cancelled debug lock')
+            raise
+
+        finally:
+            Lock.local_task_in_debug = None
+            log.debug('Exiting debugger from child')
+
+
+def mk_mpdb() -> tuple[MultiActorPdb, Callable]:
+
+    pdb = MultiActorPdb()
+    # signal.signal = pdbp.hideframe(signal.signal)
+
+    Lock.shield_sigint()
+
+    # XXX: These are the important flags mentioned in
+    # https://github.com/python-trio/trio/issues/1155
+    # which resolve the traceback spews to console.
+    pdb.allow_kbdint = True
+    pdb.nosigint = True
+
+    return pdb, Lock.unshield_sigint
+
+
+async def _breakpoint(
+
+    debug_func,
+
+    # TODO:
+    # shield: bool = False
+
+) -> None:
+    '''
+    Breakpoint entry for engaging debugger instance sync-interaction,
+    from async code, executing in actor runtime (task).
+
+    '''
+    __tracebackhide__ = True
+    actor = tractor.current_actor()
+    pdb, undo_sigint = mk_mpdb()
+    task_name = trio.lowlevel.current_task().name
+
+    # TODO: is it possible to debug a trio.Cancelled except block?
+    # right now it seems like we can kinda do with by shielding
+    # around ``tractor.breakpoint()`` but not if we move the shielded
+    # scope here???
+    # with trio.CancelScope(shield=shield):
+    #     await trio.lowlevel.checkpoint()
+
+    if (
+        not Lock.local_pdb_complete
+        or Lock.local_pdb_complete.is_set()
+    ):
+        Lock.local_pdb_complete = trio.Event()
+
+    # TODO: need a more robust check for the "root" actor
+    if (
+        not is_root_process()
+        and actor._parent_chan  # a connected child
+    ):
+
+        if Lock.local_task_in_debug:
+
+            # Recurrence entry case: this task already has the lock and
+            # is likely recurrently entering a breakpoint
+            if Lock.local_task_in_debug == task_name:
+                # noop on recurrent entry case but we want to trigger
+                # a checkpoint to allow other actors error-propagate and
+                # potetially avoid infinite re-entries in some subactor.
+                await trio.lowlevel.checkpoint()
+                return
+
+            # if **this** actor is already in debug mode block here
+            # waiting for the control to be released - this allows
+            # support for recursive entries to `tractor.breakpoint()`
+            log.warning(f"{actor.uid} already has a debug lock, waiting...")
+
+            await Lock.local_pdb_complete.wait()
+            await trio.sleep(0.1)
+
+        # mark local actor as "in debug mode" to avoid recurrent
+        # entries/requests to the root process
+        Lock.local_task_in_debug = task_name
+
+        # this **must** be awaited by the caller and is done using the
+        # root nursery so that the debugger can continue to run without
+        # being restricted by the scope of a new task nursery.
+
+        # TODO: if we want to debug a trio.Cancelled triggered exception
+        # we have to figure out how to avoid having the service nursery
+        # cancel on this task start? I *think* this works below:
+        # ```python
+        #   actor._service_n.cancel_scope.shield = shield
+        # ```
+        # but not entirely sure if that's a sane way to implement it?
+        try:
+            with trio.CancelScope(shield=True):
+                await actor._service_n.start(
+                    wait_for_parent_stdin_hijack,
+                    actor.uid,
+                )
+                Lock.repl = pdb
+        except RuntimeError:
+            Lock.release()
+
+            if actor._cancel_called:
+                # service nursery won't be usable and we
+                # don't want to lock up the root either way since
+                # we're in (the midst of) cancellation.
+                return
+
+            raise
+
+    elif is_root_process():
+
+        # we also wait in the root-parent for any child that
+        # may have the tty locked prior
+        # TODO: wait, what about multiple root tasks acquiring it though?
+        if Lock.global_actor_in_debug == actor.uid:
+            # re-entrant root process already has it: noop.
+            return
+
+        # XXX: since we need to enter pdb synchronously below,
+        # we have to release the lock manually from pdb completion
+        # callbacks. Can't think of a nicer way then this atm.
+        if Lock._debug_lock.locked():
+            log.warning(
+                'Root actor attempting to shield-acquire active tty lock'
+                f' owned by {Lock.global_actor_in_debug}')
+
+            # must shield here to avoid hitting a ``Cancelled`` and
+            # a child getting stuck bc we clobbered the tty
+            with trio.CancelScope(shield=True):
+                await Lock._debug_lock.acquire()
+        else:
+            # may be cancelled
+            await Lock._debug_lock.acquire()
+
+        Lock.global_actor_in_debug = actor.uid
+        Lock.local_task_in_debug = task_name
+        Lock.repl = pdb
+
+    try:
+        # block here one (at the appropriate frame *up*) where
+        # ``breakpoint()`` was awaited and begin handling stdio.
+        log.debug("Entering the synchronous world of pdb")
+        debug_func(actor, pdb)
+
+    except bdb.BdbQuit:
+        Lock.release()
+        raise
+
+    # XXX: apparently we can't do this without showing this frame
+    # in the backtrace on first entry to the REPL? Seems like an odd
+    # behaviour that should have been fixed by now. This is also why
+    # we scrapped all the @cm approaches that were tried previously.
+    # finally:
+    #     __tracebackhide__ = True
+    #     # frame = sys._getframe()
+    #     # last_f = frame.f_back
+    #     # last_f.f_globals['__tracebackhide__'] = True
+    #     # signal.signal = pdbp.hideframe(signal.signal)
+
+
+def shield_sigint_handler(
+    signum: int,
+    frame: 'frame',  # type: ignore # noqa
+    # pdb_obj: Optional[MultiActorPdb] = None,
+    *args,
+
+) -> None:
+    '''
+    Specialized, debugger-aware SIGINT handler.
+
+    In childred we always ignore to avoid deadlocks since cancellation
+    should always be managed by the parent supervising actor. The root
+    is always cancelled on ctrl-c.
+
+    '''
+    __tracebackhide__ = True
+
+    uid_in_debug = Lock.global_actor_in_debug
+
+    actor = tractor.current_actor()
+    # print(f'{actor.uid} in HANDLER with ')
+
+    def do_cancel():
+        # If we haven't tried to cancel the runtime then do that instead
+        # of raising a KBI (which may non-gracefully destroy
+        # a ``trio.run()``).
+        if not actor._cancel_called:
+            actor.cancel_soon()
+
+        # If the runtime is already cancelled it likely means the user
+        # hit ctrl-c again because teardown didn't full take place in
+        # which case we do the "hard" raising of a local KBI.
+        else:
+            raise KeyboardInterrupt
+
+    any_connected = False
+
+    if uid_in_debug is not None:
+        # try to see if the supposed (sub)actor in debug still
+        # has an active connection to *this* actor, and if not
+        # it's likely they aren't using the TTY lock / debugger
+        # and we should propagate SIGINT normally.
+        chans = actor._peers.get(tuple(uid_in_debug))
+        if chans:
+            any_connected = any(chan.connected() for chan in chans)
+            if not any_connected:
+                log.warning(
+                    'A global actor reported to be in debug '
+                    'but no connection exists for this child:\n'
+                    f'{uid_in_debug}\n'
+                    'Allowing SIGINT propagation..'
+                )
+                return do_cancel()
+
+    # only set in the actor actually running the REPL
+    pdb_obj = Lock.repl
+
+    # root actor branch that reports whether or not a child
+    # has locked debugger.
+    if (
+        is_root_process()
+        and uid_in_debug is not None
+
+        # XXX: only if there is an existing connection to the
+        # (sub-)actor in debug do we ignore SIGINT in this
+        # parent! Otherwise we may hang waiting for an actor
+        # which has already terminated to unlock.
+        and any_connected
+    ):
+        # we are root and some actor is in debug mode
+        # if uid_in_debug is not None:
+
+        if pdb_obj:
+            name = uid_in_debug[0]
+            if name != 'root':
+                log.pdb(
+                    f"Ignoring SIGINT, child in debug mode: `{uid_in_debug}`"
+                )
+
+            else:
+                log.pdb(
+                    "Ignoring SIGINT while in debug mode"
+                )
+    elif (
+        is_root_process()
+    ):
+        if pdb_obj:
+            log.pdb(
+                "Ignoring SIGINT since debug mode is enabled"
+            )
+
+        if (
+            Lock._root_local_task_cs_in_debug
+            and not Lock._root_local_task_cs_in_debug.cancel_called
+        ):
+            Lock._root_local_task_cs_in_debug.cancel()
+
+            # revert back to ``trio`` handler asap!
+            Lock.unshield_sigint()
+
+    # child actor that has locked the debugger
+    elif not is_root_process():
+
+        chan: Channel = actor._parent_chan
+        if not chan or not chan.connected():
+            log.warning(
+                'A global actor reported to be in debug '
+                'but no connection exists for its parent:\n'
+                f'{uid_in_debug}\n'
+                'Allowing SIGINT propagation..'
+            )
+            return do_cancel()
+
+        task = Lock.local_task_in_debug
+        if (
+            task
+            and pdb_obj
+        ):
+            log.pdb(
+                f"Ignoring SIGINT while task in debug mode: `{task}`"
+            )
+
+        # TODO: how to handle the case of an intermediary-child actor
+        # that **is not** marked in debug mode? See oustanding issue:
+        # https://github.com/goodboy/tractor/issues/320
+        # elif debug_mode():
+
+    else:  # XXX: shouldn't ever get here?
+        print("WTFWTFWTF")
+        raise KeyboardInterrupt
+
+    # NOTE: currently (at least on ``fancycompleter`` 0.9.2)
+    # it looks to be that the last command that was run (eg. ll)
+    # will be repeated by default.
+
+    # maybe redraw/print last REPL output to console since
+    # we want to alert the user that more input is expect since
+    # nothing has been done dur to ignoring sigint.
+    if (
+        pdb_obj  # only when this actor has a REPL engaged
+    ):
+        # XXX: yah, mega hack, but how else do we catch this madness XD
+        if pdb_obj.shname == 'xonsh':
+            pdb_obj.stdout.write(pdb_obj.prompt)
+
+        pdb_obj.stdout.flush()
+
+        # TODO: make this work like sticky mode where if there is output
+        # detected as written to the tty we redraw this part underneath
+        # and erase the past draw of this same bit above?
+        # pdb_obj.sticky = True
+        # pdb_obj._print_if_sticky()
+
+        # also see these links for an approach from ``ptk``:
+        # https://github.com/goodboy/tractor/issues/130#issuecomment-663752040
+        # https://github.com/prompt-toolkit/python-prompt-toolkit/blob/c2c6af8a0308f9e5d7c0e28cb8a02963fe0ce07a/prompt_toolkit/patch_stdout.py
+
+        # XXX LEGACY: lol, see ``pdbpp`` issue:
+        # https://github.com/pdbpp/pdbpp/issues/496
+
+
+def _set_trace(
+    actor: tractor.Actor | None = None,
+    pdb: MultiActorPdb | None = None,
+):
+    __tracebackhide__ = True
+    actor = actor or tractor.current_actor()
+
+    # start 2 levels up in user code
+    frame: Optional[FrameType] = sys._getframe()
+    if frame:
+        frame = frame.f_back  # type: ignore
+
+    if (
+        frame
+        and pdb
+        and actor is not None
+    ):
+        log.pdb(f"\nAttaching pdb to actor: {actor.uid}\n")
+        # no f!#$&* idea, but when we're in async land
+        # we need 2x frames up?
+        frame = frame.f_back
+
+    else:
+        pdb, undo_sigint = mk_mpdb()
+
+        # we entered the global ``breakpoint()`` built-in from sync
+        # code?
+        Lock.local_task_in_debug = 'sync'
+
+    pdb.set_trace(frame=frame)
+
+
+breakpoint = partial(
+    _breakpoint,
+    _set_trace,
+)
+
+
+def _post_mortem(
+    actor: tractor.Actor,
+    pdb: MultiActorPdb,
+
+) -> None:
+    '''
+    Enter the ``pdbpp`` port mortem entrypoint using our custom
+    debugger instance.
+
+    '''
+    log.pdb(f"\nAttaching to pdb in crashed actor: {actor.uid}\n")
+
+    # TODO: you need ``pdbpp`` master (at least this commit
+    # https://github.com/pdbpp/pdbpp/commit/b757794857f98d53e3ebbe70879663d7d843a6c2)
+    # to fix this and avoid the hang it causes. See issue:
+    # https://github.com/pdbpp/pdbpp/issues/480
+    # TODO: help with a 3.10+ major release if/when it arrives.
+
+    pdbp.xpm(Pdb=lambda: pdb)
+
+
+post_mortem = partial(
+    _breakpoint,
+    _post_mortem,
+)
+
+
+async def _maybe_enter_pm(err):
+    if (
+        debug_mode()
+
+        # NOTE: don't enter debug mode recursively after quitting pdb
+        # Iow, don't re-enter the repl if the `quit` command was issued
+        # by the user.
+        and not isinstance(err, bdb.BdbQuit)
+
+        # XXX: if the error is the likely result of runtime-wide
+        # cancellation, we don't want to enter the debugger since
+        # there's races between when the parent actor has killed all
+        # comms and when the child tries to contact said parent to
+        # acquire the tty lock.
+
+        # Really we just want to mostly avoid catching KBIs here so there
+        # might be a simpler check we can do?
+        and not is_multi_cancelled(err)
+    ):
+        log.debug("Actor crashed, entering debug mode")
+        try:
+            await post_mortem()
+        finally:
+            Lock.release()
+            return True
+
+    else:
+        return False
+
+
+@acm
+async def acquire_debug_lock(
+    subactor_uid: tuple[str, str],
+) -> AsyncGenerator[None, tuple]:
+    '''
+    Grab root's debug lock on entry, release on exit.
+
+    This helper is for actor's who don't actually need
+    to acquired the debugger but want to wait until the
+    lock is free in the process-tree root.
+
+    '''
+    if not debug_mode():
+        yield None
+        return
+
+    async with trio.open_nursery() as n:
+        cs = await n.start(
+            wait_for_parent_stdin_hijack,
+            subactor_uid,
+        )
+        yield None
+        cs.cancel()
+
+
+async def maybe_wait_for_debugger(
+    poll_steps: int = 2,
+    poll_delay: float = 0.1,
+    child_in_debug: bool = False,
+
+) -> None:
+
+    if (
+        not debug_mode()
+        and not child_in_debug
+    ):
+        return
+
+    if (
+        is_root_process()
+    ):
+        # If we error in the root but the debugger is
+        # engaged we don't want to prematurely kill (and
+        # thus clobber access to) the local tty since it
+        # will make the pdb repl unusable.
+        # Instead try to wait for pdb to be released before
+        # tearing down.
+
+        sub_in_debug = None
+
+        for _ in range(poll_steps):
+
+            if Lock.global_actor_in_debug:
+                sub_in_debug = tuple(Lock.global_actor_in_debug)
+
+            log.debug('Root polling for debug')
+
+            with trio.CancelScope(shield=True):
+                await trio.sleep(poll_delay)
+
+                # TODO: could this make things more deterministic?  wait
+                # to see if a sub-actor task will be scheduled and grab
+                # the tty lock on the next tick?
+                # XXX: doesn't seem to work
+                # await trio.testing.wait_all_tasks_blocked(cushion=0)
+
+                debug_complete = Lock.no_remote_has_tty
+                if (
+                    (debug_complete and
+                     not debug_complete.is_set())
+                ):
+                    log.debug(
+                        'Root has errored but pdb is in use by '
+                        f'child {sub_in_debug}\n'
+                        'Waiting on tty lock to release..')
+
+                    await debug_complete.wait()
+
+                await trio.sleep(poll_delay)
+                continue
+        else:
+            log.debug(
+                    'Root acquired TTY LOCK'
+            )

tractor/_discovery.py

@@ -0,0 +1,157 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+Actor discovery API.
+
+"""
+from typing import (
+    Optional,
+    Union,
+    AsyncGenerator,
+)
+from contextlib import asynccontextmanager as acm
+
+from ._ipc import _connect_chan, Channel
+from ._portal import (
+    Portal,
+    open_portal,
+    LocalPortal,
+)
+from ._state import current_actor, _runtime_vars
+
+
+@acm
+async def get_arbiter(
+
+    host: str,
+    port: int,
+
+) -> AsyncGenerator[Union[Portal, LocalPortal], None]:
+    '''Return a portal instance connected to a local or remote
+    arbiter.
+    '''
+    actor = current_actor()
+
+    if not actor:
+        raise RuntimeError("No actor instance has been defined yet?")
+
+    if actor.is_arbiter:
+        # we're already the arbiter
+        # (likely a re-entrant call from the arbiter actor)
+        yield LocalPortal(actor, Channel((host, port)))
+    else:
+        async with _connect_chan(host, port) as chan:
+
+            async with open_portal(chan) as arb_portal:
+
+                yield arb_portal
+
+
+@acm
+async def get_root(
+    **kwargs,
+) -> AsyncGenerator[Portal, None]:
+
+    host, port = _runtime_vars['_root_mailbox']
+    assert host is not None
+
+    async with _connect_chan(host, port) as chan:
+        async with open_portal(chan, **kwargs) as portal:
+            yield portal
+
+
+@acm
+async def query_actor(
+    name: str,
+    arbiter_sockaddr: Optional[tuple[str, int]] = None,
+
+) -> AsyncGenerator[tuple[str, int], None]:
+    '''
+    Simple address lookup for a given actor name.
+
+    Returns the (socket) address or ``None``.
+
+    '''
+    actor = current_actor()
+    async with get_arbiter(
+        *arbiter_sockaddr or actor._arb_addr
+    ) as arb_portal:
+
+        sockaddr = await arb_portal.run_from_ns(
+            'self',
+            'find_actor',
+            name=name,
+        )
+
+        # TODO: return portals to all available actors - for now just
+        # the last one that registered
+        if name == 'arbiter' and actor.is_arbiter:
+            raise RuntimeError("The current actor is the arbiter")
+
+        yield sockaddr if sockaddr else None
+
+
+@acm
+async def find_actor(
+    name: str,
+    arbiter_sockaddr: tuple[str, int] | None = None
+
+) -> AsyncGenerator[Optional[Portal], None]:
+    '''
+    Ask the arbiter to find actor(s) by name.
+
+    Returns a connected portal to the last registered matching actor
+    known to the arbiter.
+
+    '''
+    async with query_actor(
+        name=name,
+        arbiter_sockaddr=arbiter_sockaddr,
+    ) as sockaddr:
+
+        if sockaddr:
+            async with _connect_chan(*sockaddr) as chan:
+                async with open_portal(chan) as portal:
+                    yield portal
+        else:
+            yield None
+
+
+@acm
+async def wait_for_actor(
+    name: str,
+    arbiter_sockaddr: tuple[str, int] | None = None
+) -> AsyncGenerator[Portal, None]:
+    """Wait on an actor to register with the arbiter.
+
+    A portal to the first registered actor is returned.
+    """
+    actor = current_actor()
+
+    async with get_arbiter(
+        *arbiter_sockaddr or actor._arb_addr,
+    ) as arb_portal:
+        sockaddrs = await arb_portal.run_from_ns(
+            'self',
+            'wait_for_actor',
+            name=name,
+        )
+        sockaddr = sockaddrs[-1]
+
+        async with _connect_chan(*sockaddr) as chan:
+            async with open_portal(chan) as portal:
+                yield portal

tractor/_entry.py

@@ -0,0 +1,138 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+Sub-process entry points.
+
+"""
+from __future__ import annotations
+from functools import partial
+from typing import (
+    Any,
+    TYPE_CHECKING,
+)
+
+import trio  # type: ignore
+
+from .log import (
+    get_console_log,
+    get_logger,
+)
+from . import _state
+from .to_asyncio import run_as_asyncio_guest
+from ._runtime import (
+    async_main,
+    Actor,
+)
+
+if TYPE_CHECKING:
+    from ._spawn import SpawnMethodKey
+
+
+log = get_logger(__name__)
+
+
+def _mp_main(
+
+    actor: Actor,  # type: ignore
+    accept_addr: tuple[str, int],
+    forkserver_info: tuple[Any, Any, Any, Any, Any],
+    start_method: SpawnMethodKey,
+    parent_addr: tuple[str, int] | None = None,
+    infect_asyncio: bool = False,
+
+) -> None:
+    '''
+    The routine called *after fork* which invokes a fresh ``trio.run``
+
+    '''
+    actor._forkserver_info = forkserver_info
+    from ._spawn import try_set_start_method
+    spawn_ctx = try_set_start_method(start_method)
+
+    if actor.loglevel is not None:
+        log.info(
+            f"Setting loglevel for {actor.uid} to {actor.loglevel}")
+        get_console_log(actor.loglevel)
+
+    assert spawn_ctx
+    log.info(
+        f"Started new {spawn_ctx.current_process()} for {actor.uid}")
+
+    _state._current_actor = actor
+
+    log.debug(f"parent_addr is {parent_addr}")
+    trio_main = partial(
+        async_main,
+        actor,
+        accept_addr,
+        parent_addr=parent_addr
+    )
+    try:
+        if infect_asyncio:
+            actor._infected_aio = True
+            run_as_asyncio_guest(trio_main)
+        else:
+            trio.run(trio_main)
+    except KeyboardInterrupt:
+        pass  # handle it the same way trio does?
+
+    finally:
+        log.info(f"Actor {actor.uid} terminated")
+
+
+def _trio_main(
+
+    actor: Actor,  # type: ignore
+    *,
+    parent_addr: tuple[str, int] | None = None,
+    infect_asyncio: bool = False,
+
+) -> None:
+    '''
+    Entry point for a `trio_run_in_process` subactor.
+
+    '''
+    log.info(f"Started new trio process for {actor.uid}")
+
+    if actor.loglevel is not None:
+        log.info(
+            f"Setting loglevel for {actor.uid} to {actor.loglevel}")
+        get_console_log(actor.loglevel)
+
+    log.info(
+        f"Started {actor.uid}")
+
+    _state._current_actor = actor
+
+    log.debug(f"parent_addr is {parent_addr}")
+    trio_main = partial(
+        async_main,
+        actor,
+        parent_addr=parent_addr
+    )
+
+    try:
+        if infect_asyncio:
+            actor._infected_aio = True
+            run_as_asyncio_guest(trio_main)
+        else:
+            trio.run(trio_main)
+    except KeyboardInterrupt:
+        log.warning(f"Actor {actor.uid} received KBI")
+
+    finally:
+        log.info(f"Actor {actor.uid} terminated")