From 6d9ac53bd5b0219827c84c39a786311474561b5f Mon Sep 17 00:00:00 2001
From: Tyler Goodlet <jgbt@protonmail.com>
Date: Wed, 30 Oct 2019 00:16:39 -0400
Subject: [PATCH] Add nested multierror testing

Add a test to verify that `trio.MultiError`s are properly propagated up
a simple actor nursery tree. We don't have any exception marshalling
between processes (yet) so we can't validate much more then a simple
2-depth tree. This satisfies the final bullet in #43.

Note I've limited the number of subactors per layer to around 5 since
any more then this seems to break the `multiprocessing` forkserver;
zombie subprocesses seem to be blocking teardown somehow...

Also add a single depth fast fail test just to verify that it's the
nested spawning that triggers this forkserver bug.
---
 tests/test_cancellation.py | 77 ++++++++++++++++++++++++++------------
 1 file changed, 53 insertions(+), 24 deletions(-)

diff --git a/tests/test_cancellation.py b/tests/test_cancellation.py
index 89aa8d0..7bbdd08 100644
--- a/tests/test_cancellation.py
+++ b/tests/test_cancellation.py
@@ -87,6 +87,33 @@ def test_multierror(arb_addr):
         tractor.run(main, arbiter_addr=arb_addr)
 
 
+
+@pytest.mark.parametrize('delay', (0, 0.5))
+@pytest.mark.parametrize(
+    'num_subactors', range(25, 26),
+)
+def test_multierror_fast_nursery(arb_addr, start_method, num_subactors, delay):
+    """Verify we raise a ``trio.MultiError`` out of a nursery where
+    more then one actor errors and also with a delay before failure
+    to test failure during an ongoing spawning.
+    """
+    async def main():
+        async with tractor.open_nursery() as nursery:
+            for i in range(num_subactors):
+                await nursery.run_in_actor(
+                    f'errorer{i}', assert_err, delay=delay)
+
+    with pytest.raises(trio.MultiError) as exc_info:
+        tractor.run(main, arbiter_addr=arb_addr)
+
+    assert exc_info.type == tractor.MultiError
+    err = exc_info.value
+    assert len(err.exceptions) == num_subactors
+    for exc in err.exceptions:
+        assert isinstance(exc, tractor.RemoteActorError)
+        assert exc.type == AssertionError
+
+
 def do_nothing():
     pass
 
@@ -236,35 +263,37 @@ async def test_some_cancels_all(num_actors_and_errs, start_method):
 
 async def spawn_and_error(num) -> None:
     name = tractor.current_actor().name
-    try:
-        async with tractor.open_nursery() as nursery:
-            for i in range(num):
-                await nursery.run_in_actor(
-                    f'{name}_errorer_{i}', assert_err
-                )
-    except tractor.MultiError as err:
-        assert len(err.exceptions) == num
-        raise
-    else:
-        pytest.fail("Did not raise `MultiError`?")
+    async with tractor.open_nursery() as nursery:
+        for i in range(num):
+            await nursery.run_in_actor(
+                f'{name}_errorer_{i}', assert_err
+            )
 
 
 @pytest.mark.parametrize(
     'num_subactors',
-    range(1, 5),
+    # NOTE: any more then this and the forkserver will
+    # start bailing hard...gotta look into it
+    range(4, 5),
     ids='{}_subactors'.format,
 )
 @tractor_test
-async def test_nested_multierrors_propogate(start_method, num_subactors):
+async def test_nested_multierrors(loglevel, num_subactors, start_method):
+    """Test that failed actor sets are wrapped in `trio.MultiError`s.
+    This test goes only 2 nurseries deep but we should eventually have tests
+    for arbitrary n-depth actor trees.
+    """
+    try:
+        async with tractor.open_nursery() as nursery:
 
-    async with tractor.open_nursery() as nursery:
-
-        for i in range(num_subactors):
-            await nursery.run_in_actor(
-                f'spawner_{i}',
-                spawn_and_error,
-                num=num_subactors,
-            )
-
-        # would hang otherwise
-        await nursery.cancel()
+            for i in range(num_subactors):
+                await nursery.run_in_actor(
+                    f'spawner_{i}',
+                    spawn_and_error,
+                    num=num_subactors,
+                )
+    except trio.MultiError as err:
+        assert len(err.exceptions) == num_subactors
+        for subexc in err.exceptions:
+            assert isinstance(subexc, tractor.RemoteActorError)
+            assert subexc.type is trio.MultiError