From 34ca7429c714e04136ed5208e3cf69b9a8420adc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 15 Aug 2025 13:36:19 -0400 Subject: [PATCH 01/17] Add a "real-world" example of cancelled-masking with `.aclose()` --- examples/trio/send_chan_aclose_masks_beg.py | 145 ++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 examples/trio/send_chan_aclose_masks_beg.py diff --git a/examples/trio/send_chan_aclose_masks_beg.py b/examples/trio/send_chan_aclose_masks_beg.py new file mode 100644 index 00000000..74611682 --- /dev/null +++ b/examples/trio/send_chan_aclose_masks_beg.py @@ -0,0 +1,145 @@ +from contextlib import ( + contextmanager as cm, + # TODO, any diff in async case(s)?? + # asynccontextmanager as acm, +) +from functools import partial + +import tractor +import trio + + +log = tractor.log.get_logger(__name__) +tractor.log.get_console_log('info') + +@cm +def teardown_on_exc( + raise_from_handler: bool = False, +): + ''' + You could also have a teardown handler which catches any exc and + does some required teardown. In this case the problem is + compounded UNLESS you ensure the handler's scope is OUTSIDE the + `ux.aclose()`.. that is in the caller's enclosing scope. + + ''' + try: + yield + except BaseException as _berr: + berr = _berr + log.exception( + f'Handling termination teardown in child due to,\n' + f'{berr!r}\n' + ) + if raise_from_handler: + # XXX teardown ops XXX + # on termination these steps say need to be run to + # ensure wider system consistency (like the state of + # remote connections/services). + # + # HOWEVER, any bug in this teardown code is also + # masked by the `tx.aclose()`! + # this is also true if `_tn.cancel_scope` is + # `.cancel_called` by the parent in a graceful + # request case.. + + # simulate a bug in teardown handler. + raise RuntimeError( + 'woopsie teardown bug!' + ) + + raise # no teardown bug. + + +async def finite_stream_to_rent( + tx: trio.abc.SendChannel, + child_errors_mid_stream: bool, + + task_status: trio.TaskStatus[ + trio.CancelScope, + ] = trio.TASK_STATUS_IGNORED, +): + async with ( + # XXX without this unmasker the mid-streaming RTE is never + # reported since it is masked by the `tx.aclose()` + # call which in turn raises `Cancelled`! + # + # NOTE, this is WITHOUT doing any exception handling + # inside the child task! + # + # TODO, uncomment next LoC to see the supprsessed beg[RTE]! + # tractor.trionics.maybe_raise_from_masking_exc(), + + tx as tx, # .aclose() is the guilty masker chkpt! + trio.open_nursery() as _tn, + ): + # pass our scope back to parent for supervision\ + # control. + task_status.started(_tn.cancel_scope) + + with teardown_on_exc( + raise_from_handler=not child_errors_mid_stream, + ): + for i in range(100): + log.info( + f'Child tx {i!r}\n' + ) + if ( + child_errors_mid_stream + and + i == 66 + ): + # oh wait but WOOPS there's a bug + # in that teardown code!? + raise RuntimeError( + 'woopsie, a mid-streaming bug!?' + ) + + await tx.send(i) + + +async def main( + # TODO! toggle this for the 2 cases! + # 1. child errors mid-stream while parent is also requesting + # (graceful) cancel of that child streamer. + # + # 2. child contains a teardown handler which contains a + # bug and raises. + # + child_errors_mid_stream: bool, +): + tx, rx = trio.open_memory_channel(1) + + async with ( + trio.open_nursery() as tn, + rx as rx, + ): + + _child_cs = await tn.start( + partial( + finite_stream_to_rent, + child_errors_mid_stream=child_errors_mid_stream, + tx=tx, + ) + ) + async for msg in rx: + log.info( + f'Rent rx {msg!r}\n' + ) + + # simulate some external cancellation + # request **JUST BEFORE** the child errors. + if msg == 65: + log.cancel( + f'Cancelling parent on,\n' + f'msg={msg}\n' + f'\n' + f'Simulates OOB cancel request!\n' + ) + tn.cancel_scope.cancel() + + +if __name__ == '__main__': + + for case in [True, False]: + trio.run(main, case) From 6c361a9564c32307f806f84a9eaaf88cc8a5b13e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 25 Jul 2025 10:25:33 -0400 Subject: [PATCH 02/17] Drop `except*` usage from `._taskc` unmasker That is from `maybe_raise_from_masking_exc()` thus minimizing us to a single `except BaseException` block with logic branching for the beg vs. `unmask_from` exc cases. Also, - raise val-err when `unmask_from` is not a `tuple`. - tweak the exc-note warning format. - drop all pausing from dev work. --- tractor/trionics/_taskc.py | 72 ++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/tractor/trionics/_taskc.py b/tractor/trionics/_taskc.py index 8809524b..fb743bbc 100644 --- a/tractor/trionics/_taskc.py +++ b/tractor/trionics/_taskc.py @@ -74,8 +74,9 @@ async def maybe_raise_from_masking_exc( raise_unmasked: bool = True, extra_note: str = ( 'This can occurr when,\n' - ' - a `trio.Nursery` scope embeds a `finally:`-block ' - 'which executes a checkpoint!' + '\n' + ' - a `trio.Nursery/CancelScope` embeds a `finally/except:`-block ' + 'which execs an un-shielded checkpoint!' # # ^TODO? other cases? ), @@ -104,62 +105,52 @@ async def maybe_raise_from_masking_exc( individual sub-excs but maintain the eg-parent's form right? ''' + if not isinstance(unmask_from, tuple): + raise ValueError( + f'Invalid unmask_from = {unmask_from!r}\n' + f'Must be a `tuple[Type[BaseException]]`.\n' + ) + from tractor.devx.debug import ( BoxedMaybeException, - pause, ) boxed_maybe_exc = BoxedMaybeException( raise_on_exit=raise_unmasked, ) matching: list[BaseException]|None = None - maybe_eg: ExceptionGroup|None - - if tn: - try: # handle egs - yield boxed_maybe_exc - return - except* unmask_from as _maybe_eg: - maybe_eg = _maybe_eg + try: + yield boxed_maybe_exc + return + except BaseException as _bexc: + bexc = _bexc + if isinstance(bexc, BaseExceptionGroup): matches: ExceptionGroup - matches, _ = maybe_eg.split( - unmask_from - ) - if not matches: - raise + matches, _ = bexc.split(unmask_from) + if matches: + matching = matches.exceptions - matching: list[BaseException] = matches.exceptions - else: - try: # handle non-egs - yield boxed_maybe_exc - return - except unmask_from as _maybe_exc: - maybe_exc = _maybe_exc - matching: list[BaseException] = [ - maybe_exc - ] - - # XXX, only unmask-ed for debuggin! - # TODO, remove eventually.. - except BaseException as _berr: - berr = _berr - await pause(shield=True) - raise berr + elif ( + unmask_from + and + type(bexc) in unmask_from + ): + matching = [bexc] if matching is None: raise masked: list[tuple[BaseException, BaseException]] = [] for exc_match in matching: - if exc_ctx := find_masked_excs( maybe_masker=exc_match, - unmask_from={unmask_from}, + unmask_from=set(unmask_from), ): masked.append((exc_ctx, exc_match)) boxed_maybe_exc.value = exc_match note: str = ( f'\n' - f'^^WARNING^^ the above {exc_ctx!r} was masked by a {unmask_from!r}\n' + f'^^WARNING^^\n' + f'the above {type(exc_ctx)!r} was masked by a {type(exc_match)!r}\n' ) if extra_note: note += ( @@ -171,14 +162,13 @@ async def maybe_raise_from_masking_exc( if type(exc_match) in always_warn_on: log.warning(note) - # await tractor.pause(shield=True) if raise_unmasked: if len(masked) < 2: raise exc_ctx from exc_match - else: - # ?TODO, see above but, possibly unmasking sub-exc - # entries if there are > 1 - await pause(shield=True) + # else: + # # ?TODO, see above but, possibly unmasking sub-exc + # # entries if there are > 1 + # await pause(shield=True) else: raise From d17864a432240df6d506bd8b38ac2a81f4998fee Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 25 Jul 2025 11:02:22 -0400 Subject: [PATCH 03/17] Adjust test suites to new `maybe_raise_from_masking_exc()` changes --- tests/test_trioisms.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/test_trioisms.py b/tests/test_trioisms.py index ca1e6d55..6aaf20b3 100644 --- a/tests/test_trioisms.py +++ b/tests/test_trioisms.py @@ -119,9 +119,8 @@ def test_acm_embedded_nursery_propagates_enter_err( tractor.trionics.maybe_raise_from_masking_exc( tn=tn, unmask_from=( - trio.Cancelled - if unmask_from_canc - else None + (trio.Cancelled,) if unmask_from_canc + else () ), ) ): @@ -136,8 +135,7 @@ def test_acm_embedded_nursery_propagates_enter_err( with tractor.devx.maybe_open_crash_handler( pdb=debug_mode, ) as bxerr: - if bxerr: - assert not bxerr.value + assert not bxerr.value async with ( wraps_tn_that_always_cancels() as tn, @@ -145,11 +143,12 @@ def test_acm_embedded_nursery_propagates_enter_err( assert not tn.cancel_scope.cancel_called assert 0 - assert ( - (err := bxerr.value) - and - type(err) is AssertionError - ) + if debug_mode: + assert ( + (err := bxerr.value) + and + type(err) is AssertionError + ) with pytest.raises(ExceptionGroup) as excinfo: trio.run(_main) From ba793fadd95b8af93d68c48c427f165000c5972e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 19 Aug 2025 13:04:46 -0400 Subject: [PATCH 04/17] Pass `tuple` from `._invoke()` unmasker usage To match the `maybe_raise_from_masking_exc()` sig change. --- tractor/_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tractor/_rpc.py b/tractor/_rpc.py index 573aa77b..377c51a4 100644 --- a/tractor/_rpc.py +++ b/tractor/_rpc.py @@ -655,7 +655,7 @@ async def _invoke( # *should* never be interfered with!! maybe_raise_from_masking_exc( tn=tn, - unmask_from=Cancelled, + unmask_from=(Cancelled,), ) as _mbme, # maybe boxed masked exc ): ctx._scope_nursery = tn From 25c5847f2eef69ec484d7fab153a392d954988d4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 25 Jul 2025 11:16:02 -0400 Subject: [PATCH 05/17] Drop `tn` input from `maybe_raise_from_masking_exc()` Including all caller usage throughout. Moving to a non-`except*` impl means it's never needed as a signal from the caller - we can just catch the beg outright (like we should have always been doing).. --- tests/test_trioisms.py | 1 - tractor/trionics/_taskc.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/test_trioisms.py b/tests/test_trioisms.py index 6aaf20b3..431730f3 100644 --- a/tests/test_trioisms.py +++ b/tests/test_trioisms.py @@ -117,7 +117,6 @@ def test_acm_embedded_nursery_propagates_enter_err( async with ( trio.open_nursery() as tn, tractor.trionics.maybe_raise_from_masking_exc( - tn=tn, unmask_from=( (trio.Cancelled,) if unmask_from_canc else () diff --git a/tractor/trionics/_taskc.py b/tractor/trionics/_taskc.py index fb743bbc..4e336b59 100644 --- a/tractor/trionics/_taskc.py +++ b/tractor/trionics/_taskc.py @@ -65,7 +65,6 @@ def find_masked_excs( # @acm async def maybe_raise_from_masking_exc( - tn: trio.Nursery|None = None, unmask_from: ( BaseException| tuple[BaseException] From cec028295357c137fee6780278d76e50cd56f8de Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 28 Jul 2025 12:50:06 -0400 Subject: [PATCH 06/17] Add `never_warn_on: dict` support to unmasker Such that key->value pairs can be defined which should *never be* unmasked where values of - the keys are exc-types which might be masked, and - the values are exc-types which masked the equivalent key. For example, the default includes: - KBI->taskc: a kbi should never be unmasked from its masking `trio.Cancelled`. For the impl, a new `do_warn: bool` in the fn-body determines the primary guard for whether a warning or re-raising is necessary. --- tractor/trionics/_taskc.py | 52 +++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/tractor/trionics/_taskc.py b/tractor/trionics/_taskc.py index 4e336b59..b2503434 100644 --- a/tractor/trionics/_taskc.py +++ b/tractor/trionics/_taskc.py @@ -22,7 +22,10 @@ from __future__ import annotations from contextlib import ( asynccontextmanager as acm, ) -from typing import TYPE_CHECKING +from typing import ( + Type, + TYPE_CHECKING, +) import trio from tractor.log import get_logger @@ -80,9 +83,19 @@ async def maybe_raise_from_masking_exc( # ^TODO? other cases? ), - always_warn_on: tuple[BaseException] = ( + always_warn_on: tuple[Type[BaseException]] = ( trio.Cancelled, ), + + # don't ever unmask or warn on any masking pair, + # { -> } + never_warn_on: dict[ + Type[BaseException], + Type[BaseException], + ] = { + KeyboardInterrupt: trio.Cancelled, + trio.Cancelled: trio.Cancelled, + }, # ^XXX, special case(s) where we warn-log bc likely # there will be no operational diff since the exc # is always expected to be consumed. @@ -144,7 +157,10 @@ async def maybe_raise_from_masking_exc( maybe_masker=exc_match, unmask_from=set(unmask_from), ): - masked.append((exc_ctx, exc_match)) + masked.append(( + exc_ctx, + exc_match, + )) boxed_maybe_exc.value = exc_match note: str = ( f'\n' @@ -156,18 +172,36 @@ async def maybe_raise_from_masking_exc( f'\n' f'{extra_note}\n' ) - exc_ctx.add_note(note) - if type(exc_match) in always_warn_on: + do_warn: bool = ( + never_warn_on.get( + type(exc_ctx) # masking type + ) + is not + type(exc_match) # masked type + ) + + if do_warn: + exc_ctx.add_note(note) + + if ( + do_warn + and + type(exc_match) in always_warn_on + ): log.warning(note) - if raise_unmasked: - + if ( + do_warn + and + raise_unmasked + ): if len(masked) < 2: raise exc_ctx from exc_match + + # ??TODO, see above but, possibly unmasking sub-exc + # entries if there are > 1 # else: - # # ?TODO, see above but, possibly unmasking sub-exc - # # entries if there are > 1 # await pause(shield=True) else: raise From ed18ecd06445e3f0baffbfe5d6555befb41feebd Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 19 Aug 2025 15:23:54 -0400 Subject: [PATCH 07/17] Drop `tn` arg to `maybe_raise_from_masking_exc()` in `._rpc` --- tractor/_rpc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tractor/_rpc.py b/tractor/_rpc.py index 377c51a4..68ce56ea 100644 --- a/tractor/_rpc.py +++ b/tractor/_rpc.py @@ -654,7 +654,6 @@ async def _invoke( # scope ensures unasking of the `await coro` below # *should* never be interfered with!! maybe_raise_from_masking_exc( - tn=tn, unmask_from=(Cancelled,), ) as _mbme, # maybe boxed masked exc ): From 5ab642bdf04067ca491450a27d2e3489aad84a93 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 19 Aug 2025 19:59:05 -0400 Subject: [PATCH 08/17] Drop more `typing.Optional` usage --- tractor/trionics/_mngrs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tractor/trionics/_mngrs.py b/tractor/trionics/_mngrs.py index 3acfbeda..581497df 100644 --- a/tractor/trionics/_mngrs.py +++ b/tractor/trionics/_mngrs.py @@ -31,7 +31,6 @@ from typing import ( AsyncIterator, Callable, Hashable, - Optional, Sequence, TypeVar, TYPE_CHECKING, @@ -204,7 +203,7 @@ class _Cache: a kept-alive-while-in-use async resource. ''' - service_tn: Optional[trio.Nursery] = None + service_tn: trio.Nursery|None = None locks: dict[Hashable, trio.Lock] = {} users: int = 0 values: dict[Any, Any] = {} @@ -213,7 +212,7 @@ class _Cache: tuple[trio.Nursery, trio.Event] ] = {} # nurseries: dict[int, trio.Nursery] = {} - no_more_users: Optional[trio.Event] = None + no_more_users: trio.Event|None = None @classmethod async def run_ctx( From 93aa39db07dad44db2d240662d5eabc9ac54a492 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Jul 2025 15:13:38 -0400 Subject: [PATCH 09/17] Always pop `._Cache.resources` AFTER `mng.__aexit__()` The correct ordering is to de-alloc the surrounding `service_n` + `trio.Event` **after** the `mng` teardown ensuring the `mng.__aexit__()` never can hit a ref-error if it touches either (like if a `tn` is passed to `maybe_open_context()`! --- tractor/trionics/_mngrs.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tractor/trionics/_mngrs.py b/tractor/trionics/_mngrs.py index 581497df..57897c4e 100644 --- a/tractor/trionics/_mngrs.py +++ b/tractor/trionics/_mngrs.py @@ -222,16 +222,18 @@ class _Cache: task_status: trio.TaskStatus[T] = trio.TASK_STATUS_IGNORED, ) -> None: - async with mng as value: - _, no_more_users = cls.resources[ctx_key] - cls.values[ctx_key] = value - task_status.started(value) - try: - await no_more_users.wait() - finally: - # discard nursery ref so it won't be re-used (an error)? - value = cls.values.pop(ctx_key) - cls.resources.pop(ctx_key) + try: + async with mng as value: + _, no_more_users = cls.resources[ctx_key] + try: + cls.values[ctx_key] = value + task_status.started(value) + await no_more_users.wait() + finally: + value = cls.values.pop(ctx_key) + finally: + # discard nursery ref so it won't be re-used (an error)? + cls.resources.pop(ctx_key) @acm From e9f36891919f29dafe7305bf77de3d7f2cacecac Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 4 Sep 2025 14:51:25 -0400 Subject: [PATCH 10/17] Add "ignore-case-handling" to exc unmasker Since it turns out there's even case(s) in `trio` core that are guilty (of implementing things like checkpoints in exc handlers), this adds facility for ignoring explicit cases via `inspect.FrameInfo` field matching from the unmasked `exc_ctx` within `maybe_raise_from_masking_exc()`. Impl deats, - use `inspect.getinnerframes()/getmodule()` to extract the equivalent "guilty place in code" which raised the masked error which we'd like to ignore and **not unmask**. - start a `_mask_cases: dict` which describes the entries to ignore by matching against a specific `FrameInfo`'s fields from indexed from `getinnerframes()`. - describe in that table the case i hit with `trio.WouldBlock` being always masked by a `Cancelled` due to way `trio.Lock.acquire()` implements the blocking case in the would-block handler.. - always call into a new `is_expected_masking_case()` predicate (from `maybe_raise_from_masking_exc()`) on matching `exc_ctx` types. --- tractor/trionics/_taskc.py | 82 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tractor/trionics/_taskc.py b/tractor/trionics/_taskc.py index b2503434..d299bbb0 100644 --- a/tractor/trionics/_taskc.py +++ b/tractor/trionics/_taskc.py @@ -22,6 +22,10 @@ from __future__ import annotations from contextlib import ( asynccontextmanager as acm, ) +import inspect +from types import ( + TracebackType, +) from typing import ( Type, TYPE_CHECKING, @@ -63,6 +67,66 @@ def find_masked_excs( return None +_mask_cases: dict[ + Type[Exception], # masked exc type + dict[ + int, # inner-frame index into `inspect.getinnerframes()` + # `FrameInfo.function/filename: str`s to match + tuple[str, str], + ], +] = { + trio.WouldBlock: { + # `trio.Lock.acquire()` has a checkpoint inside the + # `WouldBlock`-no_wait path's handler.. + -5: { # "5th frame up" from checkpoint + 'filename': 'trio/_sync.py', + 'function': 'acquire', + # 'lineno': 605, # matters? + }, + } +} + + +def is_expected_masking_case( + cases: dict, + exc_ctx: Exception, + exc_match: BaseException, + +) -> bool|inspect.FrameInfo: + ''' + Determine whether the provided masked exception is from a known + bug/special/unintentional-`trio`-impl case which we do not wish + to unmask. + + Return any guilty `inspect.FrameInfo` ow `False`. + + ''' + exc_tb: TracebackType = exc_match.__traceback__ + if cases := _mask_cases.get(type(exc_ctx)): + inner: list[inspect.FrameInfo] = inspect.getinnerframes(exc_tb) + + # from tractor.devx.debug import mk_pdb + # mk_pdb().set_trace() + for iframe, matchon in cases.items(): + try: + masker_frame: inspect.FrameInfo = inner[iframe] + except IndexError: + continue + + for field, in_field in matchon.items(): + val = getattr( + masker_frame, + field, + ) + if in_field not in val: + break + else: + return masker_frame + + return False + + + # XXX, relevant discussion @ `trio`-core, # https://github.com/python-trio/trio/issues/455 # @@ -197,6 +261,24 @@ async def maybe_raise_from_masking_exc( raise_unmasked ): if len(masked) < 2: + # don't unmask already known "special" cases.. + if ( + (cases := _mask_cases.get(type(exc_ctx))) + and + (masker_frame := is_expected_masking_case( + cases, + exc_ctx, + exc_match, + )) + ): + log.warning( + f'Ignoring already-known/non-ideally-valid masker code @\n' + f'{masker_frame}\n' + f'\n' + f'NOT raising {exc_ctx} from masker {exc_match!r}\n' + ) + raise exc_match + raise exc_ctx from exc_match # ??TODO, see above but, possibly unmasking sub-exc From 759174729c4da5d6e279d295de32ce6bb72a9b8c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 5 Sep 2025 18:22:39 -0400 Subject: [PATCH 11/17] Prep masking `.aclose()` script for test suite So we can parametrize in various toggles to `main()` including, - `child_errors_mid_stream: bool` which now also drives whether an additional, and otherwise non-affecting, `_tn` is allocated in the `finite_stream_to_rent()` subtask, only in the early stream termination case does it seem to produce a masked outcome? * see surrounding notes within. - `raise_unmasked: bool` to toggle whether the embedded unmasker fn will actually raise the masked user RTE; this enables demoing the masked outcomes via simple switch and makes it easy to wrap them as `pytest.xfail()` outcomes. Also in support, - use `.trionics.collapse_eg()` around the root tn to ensure when unmasking we can catch the EG-unwrapped RTE easily from a test. - flip stream `msg` logs to `.debug()` to reduce console noise. - tweak mod's script iface to report/trace unexpected non-RTEs. --- examples/trio/send_chan_aclose_masks_beg.py | 70 ++++++++++++++++++--- 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/examples/trio/send_chan_aclose_masks_beg.py b/examples/trio/send_chan_aclose_masks_beg.py index 74611682..e7f895b7 100644 --- a/examples/trio/send_chan_aclose_masks_beg.py +++ b/examples/trio/send_chan_aclose_masks_beg.py @@ -9,8 +9,10 @@ import tractor import trio -log = tractor.log.get_logger(__name__) -tractor.log.get_console_log('info') +log = tractor.log.get_logger( + name=__name__ +) + @cm def teardown_on_exc( @@ -54,6 +56,7 @@ def teardown_on_exc( async def finite_stream_to_rent( tx: trio.abc.SendChannel, child_errors_mid_stream: bool, + raise_unmasked: bool, task_status: trio.TaskStatus[ trio.CancelScope, @@ -68,20 +71,41 @@ async def finite_stream_to_rent( # inside the child task! # # TODO, uncomment next LoC to see the supprsessed beg[RTE]! - # tractor.trionics.maybe_raise_from_masking_exc(), + tractor.trionics.maybe_raise_from_masking_exc( + raise_unmasked=raise_unmasked, + ), tx as tx, # .aclose() is the guilty masker chkpt! - trio.open_nursery() as _tn, + + # XXX, this ONLY matters in the + # `child_errors_mid_stream=False` case oddly!? + # THAT IS, if no tn is opened in that case then the + # test will not fail; it raises the RTE correctly? + # + # -> so it seems this new scope somehow affects the form of + # eventual in the parent EG? + tractor.trionics.maybe_open_nursery( + nursery=( + None + if not child_errors_mid_stream + else True + ), + ) as _tn, ): # pass our scope back to parent for supervision\ # control. - task_status.started(_tn.cancel_scope) + cs: trio.CancelScope|None = ( + None + if _tn is True + else _tn.cancel_scope + ) + task_status.started(cs) with teardown_on_exc( raise_from_handler=not child_errors_mid_stream, ): for i in range(100): - log.info( + log.debug( f'Child tx {i!r}\n' ) if ( @@ -107,23 +131,31 @@ async def main( # bug and raises. # child_errors_mid_stream: bool, + + raise_unmasked: bool = False, + loglevel: str = 'info', ): + tractor.log.get_console_log(level=loglevel) + + # the `.aclose()` being checkpoints on these + # is the source of the problem.. tx, rx = trio.open_memory_channel(1) async with ( + tractor.trionics.collapse_eg(), trio.open_nursery() as tn, rx as rx, ): - _child_cs = await tn.start( partial( finite_stream_to_rent, child_errors_mid_stream=child_errors_mid_stream, + raise_unmasked=raise_unmasked, tx=tx, ) ) async for msg in rx: - log.info( + log.debug( f'Rent rx {msg!r}\n' ) @@ -139,7 +171,25 @@ async def main( tn.cancel_scope.cancel() +# XXX, manual test as script if __name__ == '__main__': - + tractor.log.get_console_log(level='info') for case in [True, False]: - trio.run(main, case) + log.info( + f'\n' + f'------ RUNNING SCRIPT TRIAL ------\n' + f'child_errors_midstream: {case!r}\n' + ) + try: + trio.run(partial( + main, + child_errors_mid_stream=case, + # raise_unmasked=True, + loglevel='info', + )) + except Exception as _exc: + exc = _exc + log.exception( + 'Should have raised an RTE or Cancelled?\n' + ) + breakpoint() From 04c3d5e239c5ede11d2da3e6352f01f510b56d06 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 5 Sep 2025 18:43:58 -0400 Subject: [PATCH 12/17] Wrap `send_chan_aclose_masks_beg.py` as test suite Call it `test_trioisms::test_unmask_aclose_as_checkpoint_on_aexit` and parametrize all script-mod`.main()` toggles including `.xfails()` for the `raise_unmasked=False` cases. --- tests/test_trioisms.py | 66 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/tests/test_trioisms.py b/tests/test_trioisms.py index 431730f3..18e0c80a 100644 --- a/tests/test_trioisms.py +++ b/tests/test_trioisms.py @@ -6,11 +6,18 @@ want to see changed. from contextlib import ( asynccontextmanager as acm, ) +from types import ModuleType + +from functools import partial import pytest +from _pytest import pathlib from tractor.trionics import collapse_eg import trio from trio import TaskStatus +from tractor._testing import ( + examples_dir, +) @pytest.mark.parametrize( @@ -106,8 +113,9 @@ def test_acm_embedded_nursery_propagates_enter_err( debug_mode: bool, ): ''' - Demo how a masking `trio.Cancelled` could be handled by unmasking from the - `.__context__` field when a user (by accident) re-raises from a `finally:`. + Demo how a masking `trio.Cancelled` could be handled by unmasking + from the `.__context__` field when a user (by accident) re-raises + from a `finally:`. ''' import tractor @@ -158,13 +166,13 @@ def test_acm_embedded_nursery_propagates_enter_err( assert len(assert_eg.exceptions) == 1 - def test_gatherctxs_with_memchan_breaks_multicancelled( debug_mode: bool, ): ''' - Demo how a using an `async with sndchan` inside a `.trionics.gather_contexts()` task - will break a strict-eg-tn's multi-cancelled absorption.. + Demo how a using an `async with sndchan` inside + a `.trionics.gather_contexts()` task will break a strict-eg-tn's + multi-cancelled absorption.. ''' from tractor import ( @@ -190,7 +198,6 @@ def test_gatherctxs_with_memchan_breaks_multicancelled( f'Closed {task!r}\n' ) - async def main(): async with ( # XXX should ensure ONLY the KBI @@ -211,3 +218,50 @@ def test_gatherctxs_with_memchan_breaks_multicancelled( with pytest.raises(KeyboardInterrupt): trio.run(main) + + +@pytest.mark.parametrize( + 'raise_unmasked', [ + True, + pytest.param( + False, + marks=pytest.mark.xfail( + reason="see examples/trio/send_chan_aclose_masks.py" + ) + ), + ] +) +@pytest.mark.parametrize( + 'child_errors_mid_stream', + [True, False], +) +def test_unmask_aclose_as_checkpoint_on_aexit( + raise_unmasked: bool, + child_errors_mid_stream: bool, + debug_mode: bool, +): + ''' + Verify that our unmasker util works over the common case where + a mem-chan's `.aclose()` is included in an `@acm` stack + and it being currently a checkpoint, can `trio.Cancelled`-mask an embedded + exception from user code resulting in a silent failure which + appears like graceful cancellation. + + This test suite is mostly implemented as an example script so it + could more easily be shared with `trio`-core peeps as `tractor`-less + minimum reproducing example. + + ''' + mod: ModuleType = pathlib.import_path( + examples_dir() + / 'trio' + / 'send_chan_aclose_masks_beg.py', + root=examples_dir(), + consider_namespace_packages=False, + ) + with pytest.raises(RuntimeError): + trio.run(partial( + mod.main, + raise_unmasked=raise_unmasked, + child_errors_mid_stream=child_errors_mid_stream, + )) From 9aebe7d8f993ab62404f85abcb694e11e22c48cb Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 5 Sep 2025 22:23:51 -0400 Subject: [PATCH 13/17] Only read `_mask_cases` if truthy, allow disabling for xfails --- tractor/trionics/_taskc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tractor/trionics/_taskc.py b/tractor/trionics/_taskc.py index d299bbb0..dc534037 100644 --- a/tractor/trionics/_taskc.py +++ b/tractor/trionics/_taskc.py @@ -163,6 +163,7 @@ async def maybe_raise_from_masking_exc( # ^XXX, special case(s) where we warn-log bc likely # there will be no operational diff since the exc # is always expected to be consumed. + ) -> BoxedMaybeException: ''' Maybe un-mask and re-raise exception(s) suppressed by a known @@ -263,6 +264,8 @@ async def maybe_raise_from_masking_exc( if len(masked) < 2: # don't unmask already known "special" cases.. if ( + _mask_cases + and (cases := _mask_cases.get(type(exc_ctx))) and (masker_frame := is_expected_masking_case( From 542d4c784068c9bc542b83293bfb1eaac58681d1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 6 Sep 2025 13:39:08 -0400 Subject: [PATCH 14/17] Ignore `examples/trio/` in docs-examples test suite --- tests/test_docs_examples.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_docs_examples.py b/tests/test_docs_examples.py index 6250e0aa..b4cf85eb 100644 --- a/tests/test_docs_examples.py +++ b/tests/test_docs_examples.py @@ -95,6 +95,7 @@ def run_example_in_subproc( and 'integration' not in p[0] and 'advanced_faults' not in p[0] and 'multihost' not in p[0] + and 'trio' not in p[0] ) ], ids=lambda t: t[1], From 9c6b90ef041881cf32a67ef62c49e8f331cb875a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 6 Sep 2025 14:03:02 -0400 Subject: [PATCH 15/17] Add a ignore-masking-case script + suite Demonstrating the guilty `trio.Lock.acquire()` impl which puts a checkpoint inside its `trio.WouldBlock` handler and which will always appear to mask the "sync path" case on (graceful) cancellation. This first script draft demos the issue from within a `tractor.context` ep bc that's where it was orig discovered, however i'm going to factor out the `tractor` code and instead just use a `.trionics.maybe_raise_from_masking_exc()` to demo its low-level ignore-case feature. Further, this script exposed a previously unhandled remote graceful cancellation case which hangs: - parent actor spawns child and opens a >1 ctxs with it, - the parent then OoB (out-of-band) cancels the child actor (with `Portal.cancel_actor()`), - since the open ctxs raise a ctxc with a `.canceller == parent.uid` the `Context._is_self_cancelled()` will eval `True`, - the `Context._scope` will NOT be cancelled in `._maybe_cancel_and_set_remote_error()` resulting in any bg-task which is waiting on a `Portal.open_context()` to not be cancelled/unblocked. So my plan is to factor this ^^ scenario into a standalone unit test as well as another test which consumes from al low-level `trio`-only version of **this** script-scenario to sanity check the interaction of the unmasker-with-ignore-cases usage implicitly around a ctx ep. --- examples/trio/lockacquire_not_unmasked.py | 97 +++++++++++++++++++++++ tests/test_trioisms.py | 35 ++++++++ 2 files changed, 132 insertions(+) create mode 100644 examples/trio/lockacquire_not_unmasked.py diff --git a/examples/trio/lockacquire_not_unmasked.py b/examples/trio/lockacquire_not_unmasked.py new file mode 100644 index 00000000..3e507133 --- /dev/null +++ b/examples/trio/lockacquire_not_unmasked.py @@ -0,0 +1,97 @@ +from functools import partial + +import tractor +import trio + + +log = tractor.log.get_logger( + name=__name__ +) + + +async def acquire_singleton_lock( + _lock = trio.Lock(), +) -> None: + log.info('TRYING TO LOCK ACQUIRE') + await _lock.acquire() + log.info('ACQUIRED') + + +@tractor.context +async def acquire_actor_global_lock( + ctx: tractor.Context, + + ignore_special_cases: bool, +): + if not ignore_special_cases: + from tractor.trionics import _taskc + _taskc._mask_cases.clear() + + await acquire_singleton_lock() + await ctx.started('locked') + + # block til cancelled + await trio.sleep_forever() + + +async def main( + ignore_special_cases: bool, + loglevel: str = 'info', + debug_mode: bool = True, + + _fail_after: float = 2, +): + tractor.log.get_console_log(level=loglevel) + + with trio.fail_after(_fail_after): + async with ( + tractor.trionics.collapse_eg(), + tractor.open_nursery( + debug_mode=debug_mode, + loglevel=loglevel, + ) as an, + trio.open_nursery() as tn, + ): + ptl = await an.start_actor( + 'locker', + enable_modules=[__name__], + ) + + async def _open_ctx( + task_status=trio.TASK_STATUS_IGNORED, + ): + async with ptl.open_context( + acquire_actor_global_lock, + ignore_special_cases=ignore_special_cases, + ) as pair: + task_status.started(pair) + await trio.sleep_forever() + + first_ctx, first = await tn.start(_open_ctx,) + assert first == 'locked' + + with trio.move_on_after(0.5):# as cs: + await _open_ctx() + + # await tractor.pause() + print('cancelling first IPC ctx!') + await first_ctx.cancel() + + await ptl.cancel_actor() + # await tractor.pause() + + +# XXX, manual test as script +if __name__ == '__main__': + tractor.log.get_console_log(level='info') + for case in [False, True]: + log.info( + f'\n' + f'------ RUNNING SCRIPT TRIAL ------\n' + f'child_errors_midstream: {case!r}\n' + ) + trio.run(partial( + main, + ignore_special_cases=case, + loglevel='info', + )) diff --git a/tests/test_trioisms.py b/tests/test_trioisms.py index 18e0c80a..f68bc2e8 100644 --- a/tests/test_trioisms.py +++ b/tests/test_trioisms.py @@ -265,3 +265,38 @@ def test_unmask_aclose_as_checkpoint_on_aexit( raise_unmasked=raise_unmasked, child_errors_mid_stream=child_errors_mid_stream, )) + + + +@pytest.mark.parametrize( + 'ignore_special_cases', [ + True, + pytest.param( + False, + marks=pytest.mark.xfail( + reason="see examples/trio/lockacquire_not_umasked.py" + ) + ), + ] +) +def test_cancelled_lockacquire_in_ipctx_not_unmaskeed( + ignore_special_cases: bool, + loglevel: str, + debug_mode: bool, +): + mod: ModuleType = pathlib.import_path( + examples_dir() + / 'trio' + / 'lockacquire_not_unmasked.py', + root=examples_dir(), + consider_namespace_packages=False, + ) + async def _main(): + with trio.fail_after(2): + await mod.main( + ignore_special_cases=ignore_special_cases, + loglevel=loglevel, + debug_mode=debug_mode, + ) + + trio.run(_main) From 07781e38cd5dd0b60b071b5d9ca6f68457ff38da Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 7 Sep 2025 18:47:10 -0400 Subject: [PATCH 16/17] Reduce "ignore cases" script to `trio`-only Remove all the `tractor` usage (with IPC ctxs) and just get us a min-reproducing-example with a multi-task-single `trio.Lock`. The wrapping test suite runs the exact same with an ignore case and an `.xfail()` for when we let the `trio.WouldBlock` be unmasked. --- examples/trio/lockacquire_not_unmasked.py | 98 ++++++++++------------- 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/examples/trio/lockacquire_not_unmasked.py b/examples/trio/lockacquire_not_unmasked.py index 3e507133..2f979a00 100644 --- a/examples/trio/lockacquire_not_unmasked.py +++ b/examples/trio/lockacquire_not_unmasked.py @@ -1,3 +1,6 @@ +from contextlib import ( + asynccontextmanager as acm, +) from functools import partial import tractor @@ -8,87 +11,72 @@ log = tractor.log.get_logger( name=__name__ ) +_lock: trio.Lock|None = None + +@acm async def acquire_singleton_lock( - _lock = trio.Lock(), ) -> None: + global _lock + if _lock is None: + log.info('Allocating LOCK') + _lock = trio.Lock() + log.info('TRYING TO LOCK ACQUIRE') - await _lock.acquire() - log.info('ACQUIRED') + async with _lock: + log.info('ACQUIRED') + yield _lock + + log.info('RELEASED') -@tractor.context -async def acquire_actor_global_lock( - ctx: tractor.Context, - ignore_special_cases: bool, +async def hold_lock_forever( + task_status=trio.TASK_STATUS_IGNORED ): - if not ignore_special_cases: - from tractor.trionics import _taskc - _taskc._mask_cases.clear() - - await acquire_singleton_lock() - await ctx.started('locked') - - # block til cancelled - await trio.sleep_forever() + async with ( + tractor.trionics.maybe_raise_from_masking_exc(), + acquire_singleton_lock() as lock, + ): + task_status.started(lock) + await trio.sleep_forever() async def main( ignore_special_cases: bool, loglevel: str = 'info', debug_mode: bool = True, - - _fail_after: float = 2, ): - tractor.log.get_console_log(level=loglevel) + async with ( + trio.open_nursery() as tn, - with trio.fail_after(_fail_after): - async with ( - tractor.trionics.collapse_eg(), - tractor.open_nursery( - debug_mode=debug_mode, - loglevel=loglevel, - ) as an, - trio.open_nursery() as tn, - ): - ptl = await an.start_actor( - 'locker', - enable_modules=[__name__], + # tractor.trionics.maybe_raise_from_masking_exc() + # ^^^ XXX NOTE, interestingly putting the unmasker + # here does not exhibit the same behaviour ?? + ): + if not ignore_special_cases: + from tractor.trionics import _taskc + _taskc._mask_cases.clear() + + _lock = await tn.start( + hold_lock_forever, + ) + with trio.move_on_after(0.2): + await tn.start( + hold_lock_forever, ) - async def _open_ctx( - task_status=trio.TASK_STATUS_IGNORED, - ): - async with ptl.open_context( - acquire_actor_global_lock, - ignore_special_cases=ignore_special_cases, - ) as pair: - task_status.started(pair) - await trio.sleep_forever() - - first_ctx, first = await tn.start(_open_ctx,) - assert first == 'locked' - - with trio.move_on_after(0.5):# as cs: - await _open_ctx() - - # await tractor.pause() - print('cancelling first IPC ctx!') - await first_ctx.cancel() - - await ptl.cancel_actor() - # await tractor.pause() + tn.cancel_scope.cancel() # XXX, manual test as script if __name__ == '__main__': tractor.log.get_console_log(level='info') - for case in [False, True]: + for case in [True, False]: log.info( f'\n' f'------ RUNNING SCRIPT TRIAL ------\n' - f'child_errors_midstream: {case!r}\n' + f'ignore_special_cases: {case!r}\n' ) trio.run(partial( main, From 62a364a1d3ca4d89b23578d8f5ab073c8f71f6e8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 11 Sep 2025 10:01:25 -0400 Subject: [PATCH 17/17] Tweaks from copilot, type fix, typos, language. --- tests/test_trioisms.py | 2 +- tractor/trionics/_taskc.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_trioisms.py b/tests/test_trioisms.py index f68bc2e8..516d30e1 100644 --- a/tests/test_trioisms.py +++ b/tests/test_trioisms.py @@ -279,7 +279,7 @@ def test_unmask_aclose_as_checkpoint_on_aexit( ), ] ) -def test_cancelled_lockacquire_in_ipctx_not_unmaskeed( +def test_cancelled_lockacquire_in_ipctx_not_unmasked( ignore_special_cases: bool, loglevel: str, debug_mode: bool, diff --git a/tractor/trionics/_taskc.py b/tractor/trionics/_taskc.py index dc534037..0298912d 100644 --- a/tractor/trionics/_taskc.py +++ b/tractor/trionics/_taskc.py @@ -72,7 +72,7 @@ _mask_cases: dict[ dict[ int, # inner-frame index into `inspect.getinnerframes()` # `FrameInfo.function/filename: str`s to match - tuple[str, str], + dict[str, str], ], ] = { trio.WouldBlock: { @@ -275,7 +275,8 @@ async def maybe_raise_from_masking_exc( )) ): log.warning( - f'Ignoring already-known/non-ideally-valid masker code @\n' + f'Ignoring already-known, non-ideal-but-valid ' + f'masker code @\n' f'{masker_frame}\n' f'\n' f'NOT raising {exc_ctx} from masker {exc_match!r}\n'