Always `Cancelled`-unmask ctx endpoint excs

To resolve the recently added and failing
`test_remote_exc_relay::test_unmasked_remote_exc`: never allow
`trio.Cancelled` to mask an underlying user-code exception, ever.

Our first real-world (runtime internal) use case for the new
`.trionics.maybe_raise_from_masking_exc()` such that the failing
test now passes with an properly relayed remote RTE unmasking B)

Details,
- flip the `Context._scope_nursery` to the default strict-eg behaviour
  and instead stack its outer scope with a `.trionics.collapse_eg()`.
- wrap the inner-most scope (after `msgops.maybe_limit_plds()`) with
  a `maybe_raise_from_masking_exc()` to ensure user-code errors are
  never masked by `trio.Cancelled`s.

Some err-reporting refinement,
- always capture any `scope_err` from the entire block for debug
  purposes; report it in the `finally` block's log.
- always capture any suppressed `maybe_re`, output from
  `ctx.maybe_raise()`, and `log.cancel()` report it.
moar_eg_smoothing
Tyler Goodlet 2025-06-15 21:22:08 -04:00
parent bad42734db
commit 4bc443ccae
1 changed files with 50 additions and 24 deletions

View File

@ -37,6 +37,7 @@ import warnings
import trio import trio
from trio import ( from trio import (
Cancelled,
CancelScope, CancelScope,
Nursery, Nursery,
TaskStatus, TaskStatus,
@ -52,10 +53,14 @@ from ._exceptions import (
ModuleNotExposed, ModuleNotExposed,
MsgTypeError, MsgTypeError,
TransportClosed, TransportClosed,
is_multi_cancelled,
pack_error, pack_error,
unpack_error, unpack_error,
) )
from .trionics import (
collapse_eg,
is_multi_cancelled,
maybe_raise_from_masking_exc,
)
from .devx import ( from .devx import (
debug, debug,
add_div, add_div,
@ -616,32 +621,40 @@ async def _invoke(
# -> the below scope is never exposed to the # -> the below scope is never exposed to the
# `@context` marked RPC function. # `@context` marked RPC function.
# - `._portal` is never set. # - `._portal` is never set.
scope_err: BaseException|None = None
try: try:
tn: trio.Nursery # TODO: better `trionics` primitive/tooling usage here!
rpc_ctx_cs: CancelScope
async with (
trio.open_nursery(
strict_exception_groups=False,
# ^XXX^ TODO? instead unpack any RAE as per "loose" style?
) as tn,
msgops.maybe_limit_plds(
ctx=ctx,
spec=ctx_meta.get('pld_spec'),
dec_hook=ctx_meta.get('dec_hook'),
),
):
ctx._scope_nursery = tn
rpc_ctx_cs = ctx._scope = tn.cancel_scope
task_status.started(ctx)
# TODO: better `trionics` tooling:
# -[ ] should would be nice to have our `TaskMngr` # -[ ] should would be nice to have our `TaskMngr`
# nursery here! # nursery here!
# -[ ] payload value checking like we do with # -[ ] payload value checking like we do with
# `.started()` such that the debbuger can engage # `.started()` such that the debbuger can engage
# here in the child task instead of waiting for the # here in the child task instead of waiting for the
# parent to crash with it's own MTE.. # parent to crash with it's own MTE..
#
tn: Nursery
rpc_ctx_cs: CancelScope
async with (
collapse_eg(),
trio.open_nursery() as tn,
msgops.maybe_limit_plds(
ctx=ctx,
spec=ctx_meta.get('pld_spec'),
dec_hook=ctx_meta.get('dec_hook'),
),
# XXX NOTE, this being the "most embedded"
# scope ensures unasking of the `await coro` below
# *should* never be interfered with!!
maybe_raise_from_masking_exc(
tn=tn,
unmask_from=Cancelled,
) as _mbme, # maybe boxed masked exc
):
ctx._scope_nursery = tn
rpc_ctx_cs = ctx._scope = tn.cancel_scope
task_status.started(ctx)
# invoke user endpoint fn.
res: Any|PayloadT = await coro res: Any|PayloadT = await coro
return_msg: Return|CancelAck = return_msg_type( return_msg: Return|CancelAck = return_msg_type(
cid=cid, cid=cid,
@ -744,38 +757,48 @@ async def _invoke(
BaseException, BaseException,
trio.Cancelled, trio.Cancelled,
) as scope_error: ) as _scope_err:
scope_err = _scope_err
if ( if (
isinstance(scope_error, RuntimeError) isinstance(scope_err, RuntimeError)
and scope_error.args and
and 'Cancel scope stack corrupted' in scope_error.args[0] scope_err.args
and
'Cancel scope stack corrupted' in scope_err.args[0]
): ):
log.exception('Cancel scope stack corrupted!?\n') log.exception('Cancel scope stack corrupted!?\n')
# debug.mk_pdb().set_trace() # debug.mk_pdb().set_trace()
# always set this (child) side's exception as the # always set this (child) side's exception as the
# local error on the context # local error on the context
ctx._local_error: BaseException = scope_error ctx._local_error: BaseException = scope_err
# ^-TODO-^ question, # ^-TODO-^ question,
# does this matter other then for # does this matter other then for
# consistentcy/testing? # consistentcy/testing?
# |_ no user code should be in this scope at this point # |_ no user code should be in this scope at this point
# AND we already set this in the block below? # AND we already set this in the block below?
# if a remote error was set then likely the # XXX if a remote error was set then likely the
# exception group was raised due to that, so # exc group was raised due to that, so
# and we instead raise that error immediately! # and we instead raise that error immediately!
ctx.maybe_raise() maybe_re: (
ContextCancelled|RemoteActorError
) = ctx.maybe_raise()
if maybe_re:
log.cancel(
f'Suppressing remote-exc from peer,\n'
f'{maybe_re!r}\n'
)
# maybe TODO: pack in come kinda # maybe TODO: pack in come kinda
# `trio.Cancelled.__traceback__` here so they can be # `trio.Cancelled.__traceback__` here so they can be
# unwrapped and displayed on the caller side? no se.. # unwrapped and displayed on the caller side? no se..
raise raise scope_err
# `@context` entrypoint task bookeeping. # `@context` entrypoint task bookeeping.
# i.e. only pop the context tracking if used ;) # i.e. only pop the context tracking if used ;)
finally: finally:
assert chan.uid assert chan.aid
# don't pop the local context until we know the # don't pop the local context until we know the
# associated child isn't in debug any more # associated child isn't in debug any more
@ -802,6 +825,9 @@ async def _invoke(
descr_str += ( descr_str += (
f'\n{merr!r}\n' # needed? f'\n{merr!r}\n' # needed?
f'{tb_str}\n' f'{tb_str}\n'
f'\n'
f'scope_error:\n'
f'{scope_err!r}\n'
) )
else: else:
descr_str += f'\n{merr!r}\n' descr_str += f'\n{merr!r}\n'