forked from goodboy/tractor
Add more explicit `asyncio` task error logging
When an `asyncio` side task errors or is cancelled we now explicitly report the traceback and task name if possible as well as the source reason for the error (some come from the `trio` side). Further, properly set any `trio` side exception (after unwrapping it from the `outcome.Error`) on the future that runs the `trio` guest run.aio_explicit_task_cancels
parent
13c8300226
commit
9b77b8c9ee
|
@ -23,6 +23,7 @@ from asyncio.exceptions import CancelledError
|
||||||
from contextlib import asynccontextmanager as acm
|
from contextlib import asynccontextmanager as acm
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
import inspect
|
import inspect
|
||||||
|
import traceback
|
||||||
from typing import (
|
from typing import (
|
||||||
Any,
|
Any,
|
||||||
Callable,
|
Callable,
|
||||||
|
@ -32,6 +33,7 @@ from typing import (
|
||||||
)
|
)
|
||||||
|
|
||||||
import trio
|
import trio
|
||||||
|
from outcome import Error
|
||||||
|
|
||||||
from .log import get_logger
|
from .log import get_logger
|
||||||
from ._state import current_actor
|
from ._state import current_actor
|
||||||
|
@ -67,6 +69,14 @@ class LinkedTaskChannel(trio.abc.Channel):
|
||||||
|
|
||||||
async def receive(self) -> Any:
|
async def receive(self) -> Any:
|
||||||
async with translate_aio_errors(self):
|
async with translate_aio_errors(self):
|
||||||
|
|
||||||
|
# TODO: do we need this to guarantee asyncio code get's
|
||||||
|
# cancelled in the case where the trio side somehow creates
|
||||||
|
# a state where the asyncio cycle-task isn't getting the
|
||||||
|
# cancel request sent by (in theory) the last checkpoint
|
||||||
|
# cycle on the trio side?
|
||||||
|
# await trio.lowlevel.checkpoint()
|
||||||
|
|
||||||
return await self._from_aio.receive()
|
return await self._from_aio.receive()
|
||||||
|
|
||||||
async def wait_ayncio_complete(self) -> None:
|
async def wait_ayncio_complete(self) -> None:
|
||||||
|
@ -202,20 +212,20 @@ def _run_asyncio_task(
|
||||||
'''
|
'''
|
||||||
nonlocal chan
|
nonlocal chan
|
||||||
aio_err = chan._aio_err
|
aio_err = chan._aio_err
|
||||||
|
task_err: Optional[BaseException] = None
|
||||||
|
|
||||||
# only to avoid ``asyncio`` complaining about uncaptured
|
# only to avoid ``asyncio`` complaining about uncaptured
|
||||||
# task exceptions
|
# task exceptions
|
||||||
try:
|
try:
|
||||||
task.exception()
|
task.exception()
|
||||||
except BaseException as terr:
|
except BaseException as terr:
|
||||||
|
task_err = terr
|
||||||
|
log.exception(f'`asyncio` task: {task.get_name()} errored')
|
||||||
assert type(terr) is type(aio_err), 'Asyncio task error mismatch?'
|
assert type(terr) is type(aio_err), 'Asyncio task error mismatch?'
|
||||||
|
|
||||||
if aio_err is not None:
|
if aio_err is not None:
|
||||||
if type(aio_err) is CancelledError:
|
# XXX: uhh is this true?
|
||||||
log.cancel("infected task was cancelled")
|
# assert task_err, f'Asyncio task {task.get_name()} discrepancy!?'
|
||||||
else:
|
|
||||||
aio_err.with_traceback(aio_err.__traceback__)
|
|
||||||
log.exception("infected task errorred:")
|
|
||||||
|
|
||||||
# NOTE: currently mem chan closure may act as a form
|
# NOTE: currently mem chan closure may act as a form
|
||||||
# of error relay (at least in the ``asyncio.CancelledError``
|
# of error relay (at least in the ``asyncio.CancelledError``
|
||||||
|
@ -224,8 +234,25 @@ def _run_asyncio_task(
|
||||||
# We might want to change this in the future though.
|
# We might want to change this in the future though.
|
||||||
from_aio.close()
|
from_aio.close()
|
||||||
|
|
||||||
task.add_done_callback(cancel_trio)
|
if type(aio_err) is CancelledError:
|
||||||
|
log.cancel("infected task was cancelled")
|
||||||
|
|
||||||
|
# TODO: show that the cancellation originated
|
||||||
|
# from the ``trio`` side? right?
|
||||||
|
# if cancel_scope.cancelled:
|
||||||
|
# raise aio_err from err
|
||||||
|
|
||||||
|
elif task_err is None:
|
||||||
|
aio_err.with_traceback(aio_err.__traceback__)
|
||||||
|
msg = ''.join(traceback.format_exception(aio_err))
|
||||||
|
log.error(
|
||||||
|
f'infected task errorred:\n{msg}'
|
||||||
|
)
|
||||||
|
|
||||||
|
# raise any ``asyncio`` side error.
|
||||||
|
raise aio_err
|
||||||
|
|
||||||
|
task.add_done_callback(cancel_trio)
|
||||||
return chan
|
return chan
|
||||||
|
|
||||||
|
|
||||||
|
@ -240,6 +267,8 @@ async def translate_aio_errors(
|
||||||
appropriately translates errors and cancels into ``trio`` land.
|
appropriately translates errors and cancels into ``trio`` land.
|
||||||
|
|
||||||
'''
|
'''
|
||||||
|
trio_task = trio.lowlevel.current_task()
|
||||||
|
|
||||||
aio_err: Optional[BaseException] = None
|
aio_err: Optional[BaseException] = None
|
||||||
|
|
||||||
def maybe_raise_aio_err(
|
def maybe_raise_aio_err(
|
||||||
|
@ -260,10 +289,21 @@ async def translate_aio_errors(
|
||||||
assert task
|
assert task
|
||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
|
|
||||||
|
except (
|
||||||
|
trio.Cancelled,
|
||||||
|
):
|
||||||
|
# relay cancel through to called ``asyncio`` task
|
||||||
|
chan._aio_task.cancel(
|
||||||
|
msg=f'the `trio` caller task was cancelled:\n{trio_task.name}'
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
except (
|
except (
|
||||||
# NOTE: see the note in the ``cancel_trio()`` asyncio task
|
# NOTE: see the note in the ``cancel_trio()`` asyncio task
|
||||||
# termination callback
|
# termination callback
|
||||||
trio.ClosedResourceError,
|
trio.ClosedResourceError,
|
||||||
|
# trio.BrokenResourceError,
|
||||||
):
|
):
|
||||||
aio_err = chan._aio_err
|
aio_err = chan._aio_err
|
||||||
if (
|
if (
|
||||||
|
@ -277,6 +317,7 @@ async def translate_aio_errors(
|
||||||
|
|
||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
# always cancel the ``asyncio`` task if we've made it this far
|
# always cancel the ``asyncio`` task if we've made it this far
|
||||||
# and it's not done.
|
# and it's not done.
|
||||||
|
@ -289,6 +330,7 @@ async def translate_aio_errors(
|
||||||
maybe_raise_aio_err()
|
maybe_raise_aio_err()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
async def run_task(
|
async def run_task(
|
||||||
func: Callable,
|
func: Callable,
|
||||||
*,
|
*,
|
||||||
|
@ -309,7 +351,6 @@ async def run_task(
|
||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
with chan._from_aio:
|
with chan._from_aio:
|
||||||
# try:
|
|
||||||
async with translate_aio_errors(chan):
|
async with translate_aio_errors(chan):
|
||||||
# return single value that is the output from the
|
# return single value that is the output from the
|
||||||
# ``asyncio`` function-as-task. Expect the mem chan api to
|
# ``asyncio`` function-as-task. Expect the mem chan api to
|
||||||
|
@ -343,7 +384,7 @@ async def open_channel_from(
|
||||||
# ``asyncio`` task.
|
# ``asyncio`` task.
|
||||||
first = await chan.receive()
|
first = await chan.receive()
|
||||||
|
|
||||||
# stream values upward
|
# deliver stream handle upward
|
||||||
yield first, chan
|
yield first, chan
|
||||||
|
|
||||||
|
|
||||||
|
@ -380,9 +421,22 @@ def run_as_asyncio_guest(
|
||||||
trio_done_fut = asyncio.Future()
|
trio_done_fut = asyncio.Future()
|
||||||
|
|
||||||
def trio_done_callback(main_outcome):
|
def trio_done_callback(main_outcome):
|
||||||
|
actor = current_actor()
|
||||||
|
|
||||||
print(f"trio_main finished: {main_outcome!r}")
|
if isinstance(main_outcome, Error):
|
||||||
trio_done_fut.set_result(main_outcome)
|
error = main_outcome.error
|
||||||
|
trio_done_fut.set_exception(error)
|
||||||
|
|
||||||
|
# TODO: explicit asyncio tb?
|
||||||
|
# traceback.print_exception(error)
|
||||||
|
|
||||||
|
# XXX: do we need this?
|
||||||
|
# actor.cancel_soon()
|
||||||
|
|
||||||
|
main_outcome.unwrap()
|
||||||
|
else:
|
||||||
|
trio_done_fut.set_result(main_outcome)
|
||||||
|
print(f"trio_main finished: {main_outcome!r}")
|
||||||
|
|
||||||
# start the infection: run trio on the asyncio loop in "guest mode"
|
# start the infection: run trio on the asyncio loop in "guest mode"
|
||||||
log.info(f"Infecting asyncio process with {trio_main}")
|
log.info(f"Infecting asyncio process with {trio_main}")
|
||||||
|
@ -392,6 +446,7 @@ def run_as_asyncio_guest(
|
||||||
run_sync_soon_threadsafe=loop.call_soon_threadsafe,
|
run_sync_soon_threadsafe=loop.call_soon_threadsafe,
|
||||||
done_callback=trio_done_callback,
|
done_callback=trio_done_callback,
|
||||||
)
|
)
|
||||||
|
# ``.unwrap()`` will raise here on error
|
||||||
return (await trio_done_fut).unwrap()
|
return (await trio_done_fut).unwrap()
|
||||||
|
|
||||||
# might as well if it's installed.
|
# might as well if it's installed.
|
||||||
|
|
Loading…
Reference in New Issue