From 949cb2c9fe4558853d0bb8f2f039e417b7000053 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 27 Jan 2022 12:03:28 -0500 Subject: [PATCH 1/8] First draft "namespace path" named tuple; probably will discard --- tractor/_portal.py | 55 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/tractor/_portal.py b/tractor/_portal.py index 1dc6ee4..58dbce5 100644 --- a/tractor/_portal.py +++ b/tractor/_portal.py @@ -14,11 +14,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" +''' Memory boundary "Portals": an API for structured concurrency linked tasks running in disparate memory domains. -""" +''' +from __future__ import annotations +from typing import NamedTuple import importlib import inspect from typing import ( @@ -66,11 +68,38 @@ async def maybe_open_nursery( yield nursery -def func_deats(func: Callable) -> tuple[str, str]: - return ( - func.__module__, - func.__name__, - ) +class NamespacePath(NamedTuple): + ''' + A serializeable description of a (function) object location + described by the target's module path and its namespace key. + + ''' + mod: str + key: str + + def load(self) -> Callable: + return getattr( + importlib.import_module(self.mod), + self.key + ) + + @classmethod + def from_ref( + cls, + obj, + + ) -> NamespacePath: + return cls( + obj.__module__, + obj.__name__, + ) + + +# def func_deats(func: Callable) -> NamespacePath[str, str]: +# return NamespacePath( +# func.__module__, +# func.__name__, +# ) def _unwrap_msg( @@ -86,6 +115,7 @@ def _unwrap_msg( assert msg.get('cid'), "Received internal error at portal?" raise unpack_error(msg, channel) + class MessagingError(Exception): 'Some kind of unexpected SC messaging dialog issue' @@ -316,7 +346,8 @@ class Portal: raise TypeError( f'{func} must be a non-streaming async function!') - fn_mod_path, fn_name = func_deats(func) + # fn_mod_path, fn_name = func_deats(func) + fn_mod_path, fn_name = NamespacePath.from_ref(func) ctx = await self.actor.start_remote_task( self.channel, @@ -346,7 +377,8 @@ class Portal: raise TypeError( f'{async_gen_func} must be an async generator function!') - fn_mod_path, fn_name = func_deats(async_gen_func) + # fn_mod_path, fn_name = func_deats(async_gen_func) + fn_mod_path, fn_name = NamespacePath.from_ref(async_gen_func) ctx = await self.actor.start_remote_task( self.channel, fn_mod_path, @@ -412,7 +444,8 @@ class Portal: raise TypeError( f'{func} must be an async generator function!') - fn_mod_path, fn_name = func_deats(func) + # fn_mod_path, fn_name = func_deats(func) + fn_mod_path, fn_name = NamespacePath.from_ref(func) ctx = await self.actor.start_remote_task( self.channel, @@ -430,7 +463,7 @@ class Portal: first = msg['started'] ctx._started_called = True - except KeyError as kerr: + except KeyError: assert msg.get('cid'), ("Received internal error at context?") if msg.get('error'): From b6ae77b5acbe00cdff6e290ccf825d3a0e74d852 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 27 Jan 2022 13:39:46 -0500 Subject: [PATCH 2/8] Use `pkgutils.resolve_name()` and a `str` subtype Python 3.9's new object resolver + a `str` is much simpler then mucking with tuples (and easier to serialize). Include a `.to_tuple()` formatter since we still are passing the module namespace and function name separately inside the runtime's message format but in theory we might be able to simplify this depending on how we would change the support for `enable_modules:list[str]` in the spawn API. Thanks to @Fuyukai for pointing `resolve_name()` which I didn't know about before! --- tractor/_portal.py | 46 +++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/tractor/_portal.py b/tractor/_portal.py index 58dbce5..5f5b5d0 100644 --- a/tractor/_portal.py +++ b/tractor/_portal.py @@ -20,8 +20,8 @@ concurrency linked tasks running in disparate memory domains. ''' from __future__ import annotations -from typing import NamedTuple import importlib +from pkgutil import resolve_name import inspect from typing import ( Any, Optional, @@ -68,20 +68,21 @@ async def maybe_open_nursery( yield nursery -class NamespacePath(NamedTuple): +class NamespacePath(str): ''' - A serializeable description of a (function) object location + A serializeable description of a (function) Python object location described by the target's module path and its namespace key. ''' - mod: str - key: str + def load_ref(self) -> object: + return resolve_name(self) - def load(self) -> Callable: - return getattr( - importlib.import_module(self.mod), - self.key - ) + def to_tuple( + self, + + ) -> tuple[str, str]: + ref = self.load_ref() + return ref.__module__, ref.__name__ @classmethod def from_ref( @@ -89,17 +90,10 @@ class NamespacePath(NamedTuple): obj, ) -> NamespacePath: - return cls( - obj.__module__, - obj.__name__, - ) - - -# def func_deats(func: Callable) -> NamespacePath[str, str]: -# return NamespacePath( -# func.__module__, -# func.__name__, -# ) + return cls(':'.join( + (obj.__module__, + obj.__name__,) + )) def _unwrap_msg( @@ -346,8 +340,7 @@ class Portal: raise TypeError( f'{func} must be a non-streaming async function!') - # fn_mod_path, fn_name = func_deats(func) - fn_mod_path, fn_name = NamespacePath.from_ref(func) + fn_mod_path, fn_name = NamespacePath.from_ref(func).to_tuple() ctx = await self.actor.start_remote_task( self.channel, @@ -377,8 +370,8 @@ class Portal: raise TypeError( f'{async_gen_func} must be an async generator function!') - # fn_mod_path, fn_name = func_deats(async_gen_func) - fn_mod_path, fn_name = NamespacePath.from_ref(async_gen_func) + fn_mod_path, fn_name = NamespacePath.from_ref( + async_gen_func).to_tuple() ctx = await self.actor.start_remote_task( self.channel, fn_mod_path, @@ -444,8 +437,7 @@ class Portal: raise TypeError( f'{func} must be an async generator function!') - # fn_mod_path, fn_name = func_deats(func) - fn_mod_path, fn_name = NamespacePath.from_ref(func) + fn_mod_path, fn_name = NamespacePath.from_ref(func).to_tuple() ctx = await self.actor.start_remote_task( self.channel, From 2900ceb0035a8ff8f5fab984a1b520e23b3dc9e4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 27 Jan 2022 16:43:31 -0500 Subject: [PATCH 3/8] Not all objects have a `.__name__` --- tractor/_portal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tractor/_portal.py b/tractor/_portal.py index 5f5b5d0..7f540e9 100644 --- a/tractor/_portal.py +++ b/tractor/_portal.py @@ -82,7 +82,7 @@ class NamespacePath(str): ) -> tuple[str, str]: ref = self.load_ref() - return ref.__module__, ref.__name__ + return ref.__module__, getattr(ref, '__name__', '') @classmethod def from_ref( From c265f3f94e647e24ba217f3b04a6180a005b425b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 29 Jan 2022 12:43:57 -0500 Subject: [PATCH 4/8] Move namespace path type into `msg` mod --- tractor/_portal.py | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/tractor/_portal.py b/tractor/_portal.py index 7f540e9..672c9af 100644 --- a/tractor/_portal.py +++ b/tractor/_portal.py @@ -21,7 +21,6 @@ concurrency linked tasks running in disparate memory domains. ''' from __future__ import annotations import importlib -from pkgutil import resolve_name import inspect from typing import ( Any, Optional, @@ -38,6 +37,7 @@ from async_generator import asynccontextmanager from ._state import current_actor from ._ipc import Channel from .log import get_logger +from .msg import NamespacePath from ._exceptions import ( unpack_error, NoResult, @@ -68,34 +68,6 @@ async def maybe_open_nursery( yield nursery -class NamespacePath(str): - ''' - A serializeable description of a (function) Python object location - described by the target's module path and its namespace key. - - ''' - def load_ref(self) -> object: - return resolve_name(self) - - def to_tuple( - self, - - ) -> tuple[str, str]: - ref = self.load_ref() - return ref.__module__, getattr(ref, '__name__', '') - - @classmethod - def from_ref( - cls, - obj, - - ) -> NamespacePath: - return cls(':'.join( - (obj.__module__, - obj.__name__,) - )) - - def _unwrap_msg( msg: dict[str, Any], From 25a27e780d81fb2785b1677f112940522813af3d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 29 Jan 2022 13:59:12 -0500 Subject: [PATCH 5/8] Add todo resources for eventual capability-based module filtering --- tractor/msg.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tractor/msg.py b/tractor/msg.py index 16f4e5c..3c96893 100644 --- a/tractor/msg.py +++ b/tractor/msg.py @@ -15,6 +15,55 @@ # along with this program. If not, see . ''' -Coming soon! +Built-in messaging patterns, types, APIs and helpers. ''' + +# TODO: integration with our ``enable_modules: list[str]`` caps sys. + +# ``pkgutil.resolve_name()`` internally uses +# ``importlib.import_module()`` which can be filtered by inserting +# a ``MetaPathFinder`` into ``sys.meta_path`` (which we could do before +# entering the ``Actor._process_messages()`` loop). +# - https://github.com/python/cpython/blob/main/Lib/pkgutil.py#L645 +# - https://stackoverflow.com/questions/1350466/preventing-python-code-from-importing-certain-modules +# - https://stackoverflow.com/a/63320902 +# - https://docs.python.org/3/library/sys.html#sys.meta_path + +# the new "Implicit Namespace Packages" might be relevant? +# - https://www.python.org/dev/peps/pep-0420/ + +from __future__ import annotations +from pkgutil import resolve_name + + +class NamespacePath(str): + ''' + A serializeable description of a (function) Python object location + described by the target's module path and its namespace key. + + ''' + _ref: object = None + + def load_ref(self) -> object: + if self._ref is None: + self._ref = resolve_name(self) + return self._ref + + def to_tuple( + self, + + ) -> tuple[str, str]: + ref = self.load_ref() + return ref.__module__, getattr(ref, '__name__', '') + + @classmethod + def from_ref( + cls, + ref, + + ) -> NamespacePath: + return cls(':'.join( + (ref.__module__, + getattr(ref, '__name__', '')) + )) From adf9a1d0aa3f59ab395a33d828fd80fbb816737a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 30 Jan 2022 12:17:32 -0500 Subject: [PATCH 6/8] Add nooz --- nooz/295.misc.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 nooz/295.misc.rst diff --git a/nooz/295.misc.rst b/nooz/295.misc.rst new file mode 100644 index 0000000..1724e06 --- /dev/null +++ b/nooz/295.misc.rst @@ -0,0 +1,3 @@ +Add an experimental `tractor.msg.NamespacePath` type for passing Python +objects by "reference" through a ``str``-subtype message and using the +new ``pkgutil.resolve_name()`` for reference loading. From 56b29c27de168d8aa83342605a4233c8c5cfc8d3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 30 Jan 2022 12:19:21 -0500 Subject: [PATCH 7/8] Add msg serialization coding todo resources list --- tractor/msg.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tractor/msg.py b/tractor/msg.py index 3c96893..461d54e 100644 --- a/tractor/msg.py +++ b/tractor/msg.py @@ -33,6 +33,15 @@ Built-in messaging patterns, types, APIs and helpers. # the new "Implicit Namespace Packages" might be relevant? # - https://www.python.org/dev/peps/pep-0420/ +# add implicit serialized message type support so that paths can be +# handed directly to IPC primitives such as streams and `Portal.run()` +# calls: +# - via ``msgspec``: +# - https://jcristharif.com/msgspec/api.html#struct +# - https://jcristharif.com/msgspec/extending.html +# via ``msgpack-python``: +# - https://github.com/msgpack/msgpack-python#packingunpacking-of-custom-data-type + from __future__ import annotations from pkgutil import resolve_name From 87de28fd885fa2da6cba908a98d3419e04ea919a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 30 Jan 2022 12:21:41 -0500 Subject: [PATCH 8/8] Slight doc string update --- tractor/msg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tractor/msg.py b/tractor/msg.py index 461d54e..138718b 100644 --- a/tractor/msg.py +++ b/tractor/msg.py @@ -49,7 +49,9 @@ from pkgutil import resolve_name class NamespacePath(str): ''' A serializeable description of a (function) Python object location - described by the target's module path and its namespace key. + described by the target's module path and namespace key meant as + a message-native "packet" to allows actors to point-and-load objects + by absolute reference. ''' _ref: object = None