Merge pull request #426 from goodboy/remote_exc_type_registry
Fix remote exc relay + add `reg_err_types()` testsmulticast_revertable_streams
Bd
GitHub
commit
ec8e8a2786
|
|
@ -0,0 +1,333 @@
|
|||
'''
|
||||
Verify that externally registered remote actor error
|
||||
types are correctly relayed, boxed, and re-raised across
|
||||
IPC actor hops via `reg_err_types()`.
|
||||
|
||||
Also ensure that when custom error types are NOT registered
|
||||
the framework indicates the lookup failure to the user.
|
||||
|
||||
'''
|
||||
import pytest
|
||||
import trio
|
||||
import tractor
|
||||
from tractor import (
|
||||
Context,
|
||||
Portal,
|
||||
RemoteActorError,
|
||||
)
|
||||
from tractor._exceptions import (
|
||||
get_err_type,
|
||||
reg_err_types,
|
||||
)
|
||||
|
||||
|
||||
# -- custom app-level errors for testing --
|
||||
class CustomAppError(Exception):
|
||||
'''
|
||||
A hypothetical user-app error that should be
|
||||
boxed+relayed by `tractor` IPC when registered.
|
||||
|
||||
'''
|
||||
|
||||
|
||||
class AnotherAppError(Exception):
|
||||
'''
|
||||
A second custom error for multi-type registration.
|
||||
|
||||
'''
|
||||
|
||||
|
||||
class UnregisteredAppError(Exception):
|
||||
'''
|
||||
A custom error that is intentionally NEVER
|
||||
registered via `reg_err_types()` so we can
|
||||
verify the framework's failure indication.
|
||||
|
||||
'''
|
||||
|
||||
|
||||
# -- remote-task endpoints --
|
||||
@tractor.context
|
||||
async def raise_custom_err(
|
||||
ctx: Context,
|
||||
) -> None:
|
||||
'''
|
||||
Remote ep that raises a `CustomAppError`
|
||||
after sync-ing with the caller.
|
||||
|
||||
'''
|
||||
await ctx.started()
|
||||
raise CustomAppError(
|
||||
'the app exploded remotely'
|
||||
)
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def raise_another_err(
|
||||
ctx: Context,
|
||||
) -> None:
|
||||
'''
|
||||
Remote ep that raises `AnotherAppError`.
|
||||
|
||||
'''
|
||||
await ctx.started()
|
||||
raise AnotherAppError(
|
||||
'another app-level kaboom'
|
||||
)
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def raise_unreg_err(
|
||||
ctx: Context,
|
||||
) -> None:
|
||||
'''
|
||||
Remote ep that raises an `UnregisteredAppError`
|
||||
which has NOT been `reg_err_types()`-registered.
|
||||
|
||||
'''
|
||||
await ctx.started()
|
||||
raise UnregisteredAppError(
|
||||
'this error type is unknown to tractor'
|
||||
)
|
||||
|
||||
|
||||
# -- unit tests for the type-registry plumbing --
|
||||
|
||||
class TestRegErrTypesPlumbing:
|
||||
'''
|
||||
Low-level checks on `reg_err_types()` and
|
||||
`get_err_type()` without requiring IPC.
|
||||
|
||||
'''
|
||||
|
||||
def test_unregistered_type_returns_none(self):
|
||||
'''
|
||||
An unregistered custom error name should yield
|
||||
`None` from `get_err_type()`.
|
||||
|
||||
'''
|
||||
result = get_err_type('CustomAppError')
|
||||
assert result is None
|
||||
|
||||
def test_register_and_lookup(self):
|
||||
'''
|
||||
After `reg_err_types()`, the custom type should
|
||||
be discoverable via `get_err_type()`.
|
||||
|
||||
'''
|
||||
reg_err_types([CustomAppError])
|
||||
result = get_err_type('CustomAppError')
|
||||
assert result is CustomAppError
|
||||
|
||||
def test_register_multiple_types(self):
|
||||
'''
|
||||
Registering a list of types should make each
|
||||
one individually resolvable.
|
||||
|
||||
'''
|
||||
reg_err_types([
|
||||
CustomAppError,
|
||||
AnotherAppError,
|
||||
])
|
||||
assert (
|
||||
get_err_type('CustomAppError')
|
||||
is CustomAppError
|
||||
)
|
||||
assert (
|
||||
get_err_type('AnotherAppError')
|
||||
is AnotherAppError
|
||||
)
|
||||
|
||||
def test_builtin_types_always_resolve(self):
|
||||
'''
|
||||
Builtin error types like `RuntimeError` and
|
||||
`ValueError` should always be found without
|
||||
any prior registration.
|
||||
|
||||
'''
|
||||
assert (
|
||||
get_err_type('RuntimeError')
|
||||
is RuntimeError
|
||||
)
|
||||
assert (
|
||||
get_err_type('ValueError')
|
||||
is ValueError
|
||||
)
|
||||
|
||||
def test_tractor_native_types_resolve(self):
|
||||
'''
|
||||
`tractor`-internal exc types (e.g.
|
||||
`ContextCancelled`) should always resolve.
|
||||
|
||||
'''
|
||||
assert (
|
||||
get_err_type('ContextCancelled')
|
||||
is tractor.ContextCancelled
|
||||
)
|
||||
|
||||
def test_boxed_type_str_without_ipc_msg(self):
|
||||
'''
|
||||
When a `RemoteActorError` is constructed
|
||||
without an IPC msg (and no resolvable type),
|
||||
`.boxed_type_str` should return `'<unknown>'`.
|
||||
|
||||
'''
|
||||
rae = RemoteActorError('test')
|
||||
assert rae.boxed_type_str == '<unknown>'
|
||||
|
||||
|
||||
# -- IPC-level integration tests --
|
||||
|
||||
def test_registered_custom_err_relayed(
|
||||
debug_mode: bool,
|
||||
tpt_proto: str,
|
||||
):
|
||||
'''
|
||||
When a custom error type is registered via
|
||||
`reg_err_types()` on BOTH sides of an IPC dialog,
|
||||
the parent should receive a `RemoteActorError`
|
||||
whose `.boxed_type` matches the original custom
|
||||
error type.
|
||||
|
||||
'''
|
||||
reg_err_types([CustomAppError])
|
||||
|
||||
async def main():
|
||||
async with tractor.open_nursery(
|
||||
debug_mode=debug_mode,
|
||||
enable_transports=[tpt_proto],
|
||||
) as an:
|
||||
ptl: Portal = await an.start_actor(
|
||||
'custom-err-raiser',
|
||||
enable_modules=[__name__],
|
||||
)
|
||||
async with ptl.open_context(
|
||||
raise_custom_err,
|
||||
) as (ctx, sent):
|
||||
assert not sent
|
||||
try:
|
||||
await ctx.wait_for_result()
|
||||
except RemoteActorError as rae:
|
||||
assert rae.boxed_type is CustomAppError
|
||||
assert rae.src_type is CustomAppError
|
||||
assert 'the app exploded remotely' in str(
|
||||
rae.tb_str
|
||||
)
|
||||
raise
|
||||
|
||||
with pytest.raises(RemoteActorError) as excinfo:
|
||||
trio.run(main)
|
||||
|
||||
rae = excinfo.value
|
||||
assert rae.boxed_type is CustomAppError
|
||||
|
||||
|
||||
def test_registered_another_err_relayed(
|
||||
debug_mode: bool,
|
||||
tpt_proto: str,
|
||||
):
|
||||
'''
|
||||
Same as above but for a different custom error
|
||||
type to verify multi-type registration works
|
||||
end-to-end over IPC.
|
||||
|
||||
'''
|
||||
reg_err_types([AnotherAppError])
|
||||
|
||||
async def main():
|
||||
async with tractor.open_nursery(
|
||||
debug_mode=debug_mode,
|
||||
enable_transports=[tpt_proto],
|
||||
) as an:
|
||||
ptl: Portal = await an.start_actor(
|
||||
'another-err-raiser',
|
||||
enable_modules=[__name__],
|
||||
)
|
||||
async with ptl.open_context(
|
||||
raise_another_err,
|
||||
) as (ctx, sent):
|
||||
assert not sent
|
||||
try:
|
||||
await ctx.wait_for_result()
|
||||
except RemoteActorError as rae:
|
||||
assert (
|
||||
rae.boxed_type
|
||||
is AnotherAppError
|
||||
)
|
||||
raise
|
||||
|
||||
await an.cancel()
|
||||
|
||||
with pytest.raises(RemoteActorError) as excinfo:
|
||||
trio.run(main)
|
||||
|
||||
rae = excinfo.value
|
||||
assert rae.boxed_type is AnotherAppError
|
||||
|
||||
|
||||
def test_unregistered_err_still_relayed(
|
||||
debug_mode: bool,
|
||||
tpt_proto: str,
|
||||
):
|
||||
'''
|
||||
Verify that even when a custom error type is NOT registered via
|
||||
`reg_err_types()`, the remote error is still relayed as
|
||||
a `RemoteActorError` with all string-level info preserved
|
||||
(traceback, type name, source actor uid).
|
||||
|
||||
The `.boxed_type` will be `None` (type obj can't be resolved) but
|
||||
`.boxed_type_str` and `.src_type_str` still report the original
|
||||
type name from the IPC msg.
|
||||
|
||||
This documents the expected limitation: without `reg_err_types()`
|
||||
the `.boxed_type` property can NOT resolve to the original Python
|
||||
type.
|
||||
|
||||
'''
|
||||
# NOTE: intentionally do NOT call
|
||||
# `reg_err_types([UnregisteredAppError])`
|
||||
|
||||
async def main():
|
||||
async with tractor.open_nursery(
|
||||
debug_mode=debug_mode,
|
||||
enable_transports=[tpt_proto],
|
||||
) as an:
|
||||
ptl: Portal = await an.start_actor(
|
||||
'unreg-err-raiser',
|
||||
enable_modules=[__name__],
|
||||
)
|
||||
async with ptl.open_context(
|
||||
raise_unreg_err,
|
||||
) as (ctx, sent):
|
||||
assert not sent
|
||||
await ctx.wait_for_result()
|
||||
|
||||
await an.cancel()
|
||||
|
||||
with pytest.raises(RemoteActorError) as excinfo:
|
||||
trio.run(main)
|
||||
|
||||
rae = excinfo.value
|
||||
|
||||
# the error IS relayed even without
|
||||
# registration; type obj is unresolvable but
|
||||
# all string-level info is preserved.
|
||||
assert rae.boxed_type is None # NOT `UnregisteredAppError`
|
||||
assert rae.src_type is None
|
||||
|
||||
# string names survive the IPC round-trip
|
||||
# via the `Error` msg fields.
|
||||
assert (
|
||||
rae.src_type_str
|
||||
==
|
||||
'UnregisteredAppError'
|
||||
)
|
||||
assert (
|
||||
rae.boxed_type_str
|
||||
==
|
||||
'UnregisteredAppError'
|
||||
)
|
||||
|
||||
# original traceback content is preserved
|
||||
assert 'this error type is unknown' in rae.tb_str
|
||||
assert 'UnregisteredAppError' in rae.tb_str
|
||||
|
|
@ -195,7 +195,7 @@ def reg_err_types(
|
|||
|
||||
Such that error types can be registered by an external
|
||||
`tractor`-use-app code base which are expected to be raised
|
||||
remotely; enables them being re-raised on the recevier side of
|
||||
remotely; enables them being re-raised on the receiver side of
|
||||
some inter-actor IPC dialog.
|
||||
|
||||
'''
|
||||
|
|
@ -211,7 +211,7 @@ def reg_err_types(
|
|||
)
|
||||
|
||||
|
||||
def get_err_type(type_name: str) -> BaseException|None:
|
||||
def get_err_type(type_name: str) -> Type[BaseException]|None:
|
||||
'''
|
||||
Look up an exception type by name from the set of locally known
|
||||
namespaces:
|
||||
|
|
@ -325,7 +325,8 @@ class RemoteActorError(Exception):
|
|||
# also pertains to our long long oustanding issue XD
|
||||
# https://github.com/goodboy/tractor/issues/5
|
||||
self._boxed_type: BaseException = boxed_type
|
||||
self._src_type: BaseException|None = None
|
||||
self._src_type: Type[BaseException]|None = None
|
||||
self._src_type_resolved: bool = False
|
||||
self._ipc_msg: Error|None = ipc_msg
|
||||
self._extra_msgdata = extra_msgdata
|
||||
|
||||
|
|
@ -434,24 +435,41 @@ class RemoteActorError(Exception):
|
|||
return self._ipc_msg.src_type_str
|
||||
|
||||
@property
|
||||
def src_type(self) -> str:
|
||||
def src_type(self) -> Type[BaseException]|None:
|
||||
'''
|
||||
Error type raised by original remote faulting actor.
|
||||
Error type raised by original remote faulting
|
||||
actor.
|
||||
|
||||
When the error has only been relayed a single actor-hop
|
||||
this will be the same as the `.boxed_type`.
|
||||
When the error has only been relayed a single
|
||||
actor-hop this will be the same as
|
||||
`.boxed_type`.
|
||||
|
||||
If the type can not be resolved locally (i.e.
|
||||
it was not registered via `reg_err_types()`)
|
||||
a warning is logged and `None` is returned;
|
||||
all string-level error info (`.src_type_str`,
|
||||
`.tb_str`, etc.) remains available.
|
||||
|
||||
'''
|
||||
if self._src_type is None:
|
||||
if not self._src_type_resolved:
|
||||
self._src_type_resolved = True
|
||||
|
||||
if self._ipc_msg is None:
|
||||
return None
|
||||
|
||||
self._src_type = get_err_type(
|
||||
self._ipc_msg.src_type_str
|
||||
)
|
||||
|
||||
if not self._src_type:
|
||||
raise TypeError(
|
||||
f'Failed to lookup src error type with '
|
||||
f'`tractor._exceptions.get_err_type()` :\n'
|
||||
f'{self.src_type_str}'
|
||||
log.warning(
|
||||
f'Failed to lookup src error type via\n'
|
||||
f'`tractor._exceptions.get_err_type()`:\n'
|
||||
f'\n'
|
||||
f'`{self._ipc_msg.src_type_str}`'
|
||||
f' is not registered!\n'
|
||||
f'\n'
|
||||
f'Call `reg_err_types()` to enable'
|
||||
f' full type reconstruction.\n'
|
||||
)
|
||||
|
||||
return self._src_type
|
||||
|
|
@ -459,20 +477,30 @@ class RemoteActorError(Exception):
|
|||
@property
|
||||
def boxed_type_str(self) -> str:
|
||||
'''
|
||||
String-name of the (last hop's) boxed error type.
|
||||
String-name of the (last hop's) boxed error
|
||||
type.
|
||||
|
||||
Falls back to the IPC-msg-encoded type-name
|
||||
str when the type can not be resolved locally
|
||||
(e.g. unregistered custom errors).
|
||||
|
||||
'''
|
||||
# TODO, maybe support also serializing the
|
||||
# `ExceptionGroup.exeptions: list[BaseException]` set under
|
||||
# certain conditions?
|
||||
# `ExceptionGroup.exceptions: list[BaseException]`
|
||||
# set under certain conditions?
|
||||
bt: Type[BaseException] = self.boxed_type
|
||||
if bt:
|
||||
return str(bt.__name__)
|
||||
|
||||
return ''
|
||||
# fallback to the str name from the IPC msg
|
||||
# when the type obj can't be resolved.
|
||||
if self._ipc_msg:
|
||||
return self._ipc_msg.boxed_type_str
|
||||
|
||||
return '<unknown>'
|
||||
|
||||
@property
|
||||
def boxed_type(self) -> Type[BaseException]:
|
||||
def boxed_type(self) -> Type[BaseException]|None:
|
||||
'''
|
||||
Error type boxed by last actor IPC hop.
|
||||
|
||||
|
|
@ -701,10 +729,22 @@ class RemoteActorError(Exception):
|
|||
failing actor's remote env.
|
||||
|
||||
'''
|
||||
# TODO: better tb insertion and all the fancier dunder
|
||||
# metadata stuff as per `.__context__` etc. and friends:
|
||||
# TODO: better tb insertion and all the fancier
|
||||
# dunder metadata stuff as per `.__context__`
|
||||
# etc. and friends:
|
||||
# https://github.com/python-trio/trio/issues/611
|
||||
src_type_ref: Type[BaseException] = self.src_type
|
||||
src_type_ref: Type[BaseException]|None = (
|
||||
self.src_type
|
||||
)
|
||||
if src_type_ref is None:
|
||||
# unresolvable type: fall back to
|
||||
# a `RuntimeError` preserving original
|
||||
# traceback + type name.
|
||||
return RuntimeError(
|
||||
f'{self.src_type_str}: '
|
||||
f'{self.tb_str}'
|
||||
)
|
||||
|
||||
return src_type_ref(self.tb_str)
|
||||
|
||||
# TODO: local recontruction of nested inception for a given
|
||||
|
|
@ -1233,14 +1273,31 @@ def unpack_error(
|
|||
if not isinstance(msg, Error):
|
||||
return None
|
||||
|
||||
# try to lookup a suitable error type from the local runtime
|
||||
# env then use it to construct a local instance.
|
||||
# boxed_type_str: str = error_dict['boxed_type_str']
|
||||
# try to lookup a suitable error type from the
|
||||
# local runtime env then use it to construct a
|
||||
# local instance.
|
||||
boxed_type_str: str = msg.boxed_type_str
|
||||
boxed_type: Type[BaseException] = get_err_type(boxed_type_str)
|
||||
boxed_type: Type[BaseException]|None = get_err_type(
|
||||
boxed_type_str
|
||||
)
|
||||
|
||||
# retrieve the error's msg-encoded remotoe-env info
|
||||
message: str = f'remote task raised a {msg.boxed_type_str!r}\n'
|
||||
if boxed_type is None:
|
||||
log.warning(
|
||||
f'Failed to resolve remote error type\n'
|
||||
f'`{boxed_type_str}` - boxing as\n'
|
||||
f'`RemoteActorError` with original\n'
|
||||
f'traceback preserved.\n'
|
||||
f'\n'
|
||||
f'Call `reg_err_types()` to enable\n'
|
||||
f'full type reconstruction.\n'
|
||||
)
|
||||
|
||||
# retrieve the error's msg-encoded remote-env
|
||||
# info
|
||||
message: str = (
|
||||
f'remote task raised a '
|
||||
f'{msg.boxed_type_str!r}\n'
|
||||
)
|
||||
|
||||
# TODO: do we even really need these checks for RAEs?
|
||||
if boxed_type_str in [
|
||||
|
|
|
|||
Loading…
Reference in New Issue