Re-route errors from spawn tasks and mngr task to handler

tractor/_supervise.py

@@ -34,14 +34,12 @@ class ActorNursery:
     def __init__(
         self,
         actor: Actor,
-        ria_nursery: trio.Nursery,
+        spawn_nursery: trio.Nursery,
-        da_nursery: trio.Nursery,
         errors: Dict[Tuple[str, str], Exception],
     ) -> None:
         # self.supervisor = supervisor  # TODO
         self._actor: Actor = actor
-        self._ria_nursery = ria_nursery
+        self._spawn_n = spawn_nursery
-        self._da_nursery = da_nursery
         self._children: Dict[
             Tuple[str, str],
             Tuple[Actor, mp.Process, Optional[Portal]]
@@ -99,7 +97,7 @@ class ActorNursery:
 
         # start a task to spawn a process
         # blocks until process has been started and a portal setup
-        nursery = nursery or self._da_nursery
+        nursery = nursery or self._spawn_n
 
         # XXX: the type ignore is actually due to a `mypy` bug
         return await nursery.start(  # type: ignore
@@ -149,7 +147,7 @@ class ActorNursery:
             bind_addr=bind_addr,
             loglevel=loglevel,
             # use the run_in_actor nursery
-            nursery=self._ria_nursery,
+            nursery=self._spawn_n,
             infect_asyncio=infect_asyncio,
         )
 
@@ -182,77 +180,35 @@ class ActorNursery:
         """
         self.cancelled = True
 
-        childs = tuple(self._children.keys())
+        # entries may be poppsed by the spawning backend as
+        # actors cancel individually
+        childs = self._children.copy()
+
         log.cancel(
-            f"Cancelling nursery in {self._actor.uid} with children\n{childs}"
+            f'Cancelling nursery in {self._actor.uid} with children\n'
+            f'{childs.keys()}'
         )
 
+        # wake up all spawn tasks to move on as those nursery
+        # has ``__aexit__()``-ed
+        self._join_procs.set()
+
         await maybe_wait_for_debugger()
 
-        # wake up all spawn tasks
+        # one-cancels-all strat
-        self._join_procs.set()
+        async with trio.open_nursery() as cancel_sender:
+            for subactor, proc, portal in childs.values():
+                cancel_sender.start_soon(portal.cancel_actor)
 
-        # cancel all spawner nurseries
+        # cancel all spawner tasks
-        self._ria_nursery.cancel_scope.cancel()
+        # self._spawn_n.cancel_scope.cancel()
-        self._da_nursery.cancel_scope.cancel()
 
+    async def _handle_err(
+        self,
+        err: BaseException,
+        portal: Optional[Portal] = None,
 
-@asynccontextmanager
+    ) -> None:
-async def _open_and_supervise_one_cancels_all_nursery(
-    actor: Actor,
-) -> typing.AsyncGenerator[ActorNursery, None]:
-
-    # the collection of errors retreived from spawned sub-actors
-    errors: Dict[Tuple[str, str], Exception] = {}
-
-    # This is the outermost level "deamon actor" nursery. It is awaited
-    # **after** the below inner "run in actor nursery". This allows for
-    # handling errors that are generated by the inner nursery in
-    # a supervisor strategy **before** blocking indefinitely to wait for
-    # actors spawned in "daemon mode" (aka started using
-    # ``ActorNursery.start_actor()``).
-    original_err = None
-
-    # errors from this daemon actor nursery bubble up to caller
-    try:
-        async with trio.open_nursery() as da_nursery:
-            # try:
-
-            # This is the inner level "run in actor" nursery. It is
-            # awaited first since actors spawned in this way (using
-            # ``ActorNusery.run_in_actor()``) are expected to only
-            # return a single result and then complete (i.e. be canclled
-            # gracefully). Errors collected from these actors are
-            # immediately raised for handling by a supervisor strategy.
-            # As such if the strategy propagates any error(s) upwards
-            # the above "daemon actor" nursery will be notified.
-            try:
-                async with trio.open_nursery() as ria_nursery:
-
-                    anursery = ActorNursery(
-                        actor,
-                        ria_nursery,
-                        da_nursery,
-                        errors
-                    )
-                    # spawning of actors happens in the caller's scope
-                    # after we yield upwards
-                    yield anursery
-
-                    log.runtime(
-                        f"Waiting on subactors {anursery._children} "
-                        "to complete"
-                    )
-
-                    # signal all process monitor tasks to conduct
-                    # hard join phase.
-                    # await maybe_wait_for_debugger()
-                    # log.error('joing trigger NORMAL')
-                    anursery._join_procs.set()
-
-            except BaseException as err:
-                original_err = err
-
         # XXX: hypothetically an error could be
         # raised and then a cancel signal shows up
         # slightly after in which case the `else:`
@@ -276,21 +232,108 @@ async def _open_and_supervise_one_cancels_all_nursery(
                     f"errored with {err}, ")
 
             # cancel all subactors
-                    await anursery.cancel()
+            await self.cancel()
 
-            # ria_nursery scope end - nursery checkpoint
 
-    # after daemon nursery exit
+@asynccontextmanager
+async def _open_and_supervise_one_cancels_all_nursery(
+    actor: Actor,
+) -> typing.AsyncGenerator[ActorNursery, None]:
+
+    # the collection of errors retreived from spawned sub-actors
+    errors: Dict[Tuple[str, str], Exception] = {}
+
+    # This is the outermost level "deamon actor" nursery. It is awaited
+    # **after** the below inner "run in actor nursery". This allows for
+    # handling errors that are generated by the inner nursery in
+    # a supervisor strategy **before** blocking indefinitely to wait for
+    # actors spawned in "daemon mode" (aka started using
+    # ``ActorNursery.start_actor()``).
+    src_err: Optional[BaseException] = None
+
+    # errors from this daemon actor nursery bubble up to caller
+    try:
+        async with trio.open_nursery() as spawn_n:
+            # try:
+
+            # This is the inner level "run in actor" nursery. It is
+            # awaited first since actors spawned in this way (using
+            # ``ActorNusery.run_in_actor()``) are expected to only
+            # return a single result and then complete (i.e. be canclled
+            # gracefully). Errors collected from these actors are
+            # immediately raised for handling by a supervisor strategy.
+            # As such if the strategy propagates any error(s) upwards
+            # the above "daemon actor" nursery will be notified.
+
+            anursery = ActorNursery(
+                actor,
+                spawn_n,
+                errors
+            )
+            # spawning of actors happens in the caller's scope
+            # after we yield upwards
+            try:
+                yield anursery
+
+                log.runtime(
+                    f"Waiting on subactors {anursery._children} "
+                    "to complete"
+                )
+
+                # signal all process monitor tasks to conduct
+                # hard join phase.
+                # await maybe_wait_for_debugger()
+                # log.error('joing trigger NORMAL')
+                anursery._join_procs.set()
+
+            # NOTE: there are 2 cases for error propagation:
+            # - an actor which is ``.run_in_actor()`` invoked
+            #   runs a single task and reports the error upwards
+            # - the top level task which opened this nursery (in the
+            #   parent actor) raises. In this case the raise can come
+            #   from a variety of places:
+            #   - user task code unrelated to the nursery/child actors
+            #   - a ``RemoteActorError`` propagated up through the
+            #   portal api from a child actor which will look the exact
+            #   same as a user code failure.
+
+            except BaseException as err:
+                print('ERROR')
+                # anursery._join_procs.set()
+                src_err = err
+
+                # with trio.CancelScope(shield=True):
+                await anursery._handle_err(err)
+                raise
+
+    except BaseException as err:
+        # nursery bubble up
+        nurse_err = err
+
+        # do not double cancel subactors
+        if not anursery.cancelled:
+            await anursery._handle_err(err)
+
+        raise
+
     finally:
+        if anursery._children:
             log.cancel(f'Waiting on remaining children {anursery._children}')
             with trio.CancelScope(shield=True):
                 await anursery._all_children_reaped.wait()
+
+        log.cancel(f'All children complete for {anursery}')
+
         # No errors were raised while awaiting ".run_in_actor()"
         # actors but those actors may have returned remote errors as
         # results (meaning they errored remotely and have relayed
         # those errors back to this parent actor). The errors are
         # collected in ``errors`` so cancel all actors, summarize
         # all errors and re-raise.
+
+        if src_err and src_err not in errors.values():
+            errors[actor.uid] = src_err
+
         if errors:
             if anursery._children:
                 raise RuntimeError("WHERE TF IS THE ZOMBIE LORD!?!?!")
@@ -306,8 +349,8 @@ async def _open_and_supervise_one_cancels_all_nursery(
         log.cancel(f'{anursery} terminated gracefully')
 
         # XXX" honestly no idea why this is needed but sure..
-        if isinstance(original_err, KeyboardInterrupt) and anursery.cancelled:
+        if isinstance(src_err, KeyboardInterrupt) and anursery.cancelled:
-            raise original_err
+            raise src_err
 
 
 @asynccontextmanager