From 4a270f85ca940b15a67a5d9496dd88cc6847cd02 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 31 May 2024 12:03:18 -0400 Subject: [PATCH] Drop sub-decoder proto-cruft from `.msg._codec` It ended up getting necessarily implemented as the `PldRx` though at a different layer and won't be needed as part of `MsgCodec` most likely, though this original idea did provide the source of inspiration for how things work now! Also Move the commented TODO proto for a codec hook factory from `.types` to `._codec` where it prolly better fits and update some msg related todo/questions. --- tractor/msg/_codec.py | 196 ++++++++---------------------------------- tractor/msg/types.py | 62 +++++-------- 2 files changed, 58 insertions(+), 200 deletions(-) diff --git a/tractor/msg/_codec.py b/tractor/msg/_codec.py index e1c59e9..c1301bd 100644 --- a/tractor/msg/_codec.py +++ b/tractor/msg/_codec.py @@ -52,10 +52,6 @@ from msgspec import ( msgpack, Raw, ) -# from trio.lowlevel import ( -# RunVar, -# RunVarToken, -# ) # TODO: see notes below from @mikenerone.. # from tricycle import TreeVar @@ -368,160 +364,16 @@ class MsgCodec(Struct): # https://jcristharif.com/msgspec/usage.html#typed-decoding return self._dec.decode(msg) - # TODO: a sub-decoder system as well? - # payload_msg_specs: Union[Type[Struct]] = Any - # see related comments in `.msg.types` - # _payload_decs: ( - # dict[ - # str, - # msgpack.Decoder, - # ] - # |None - # ) = None - # OR - # ) = { - # # pre-seed decoders for std-py-type-set for use when - # # `MsgType.pld == None|Any`. - # None: msgpack.Decoder(Any), - # Any: msgpack.Decoder(Any), - # } - # - # -[ ] do we still want to try and support the sub-decoder with - # `.Raw` technique in the case that the `Generic` approach gives - # future grief? - # - # -[ ] - # -> https://jcristharif.com/msgspec/api.html#raw - # - #def mk_pld_subdec( - # self, - # payload_types: Union[Type[Struct]], - #) -> msgpack.Decoder: - # # TODO: sub-decoder suppor for `.pld: Raw`? - # # => see similar notes inside `.msg.types`.. - # # - # # not sure we'll end up needing this though it might have - # # unforeseen advantages in terms of enabling encrypted - # # appliciation layer (only) payloads? - # # - # # register sub-payload decoders to load `.pld: Raw` - # # decoded `Msg`-packets using a dynamic lookup (table) - # # instead of a pre-defined msg-spec via `Generic` - # # parameterization. - # # - # ( - # tags, - # payload_dec, - # ) = mk_tagged_union_dec( - # tagged_structs=list(payload_types.__args__), - # ) - # # register sub-decoders by tag - # subdecs: dict[str, msgpack.Decoder]|None = self._payload_decs - # for name in tags: - # subdecs.setdefault( - # name, - # payload_dec, - # ) - - # return payload_dec - - # sub-decoders for retreiving embedded - # payload data and decoding to a sender - # side defined (struct) type. - # def dec_payload( - # codec: MsgCodec, - # msg: Msg, - - # ) -> Any|Struct: - - # msg: PayloadMsg = codec.dec.decode(msg) - # payload_tag: str = msg.header.payload_tag - # payload_dec: msgpack.Decoder = codec._payload_decs[payload_tag] - # return payload_dec.decode(msg.pld) - - # def enc_payload( - # codec: MsgCodec, - # payload: Any, - # cid: str, - - # ) -> bytes: - - # # tag_field: str|None = None - - # plbytes = codec.enc.encode(payload) - # if b'msg_type' in plbytes: - # assert isinstance(payload, Struct) - - # # tag_field: str = type(payload).__name__ - # payload = msgspec.Raw(plbytes) - - # msg = Msg( - # cid=cid, - # pld=payload, - # # Header( - # # payload_tag=tag_field, - # # # dialog_id, - # # ), - # ) - # return codec.enc.encode(msg) - - - -# TODO: sub-decoded `Raw` fields? -# -[ ] see `MsgCodec._payload_decs` notes +# [x] TODO: a sub-decoder system as well? => No! # -# XXX if we wanted something more complex then field name str-keys -# we might need a header field type to describe the lookup sys? -# class Header(Struct, tag=True): -# ''' -# A msg header which defines payload properties - -# ''' -# payload_tag: str|None = None - - - #def mk_tagged_union_dec( - # tagged_structs: list[Struct], - - #) -> tuple[ - # list[str], - # msgpack.Decoder, - #]: - # ''' - # Create a `msgpack.Decoder` for an input `list[msgspec.Struct]` - # and return a `list[str]` of each struct's `tag_field: str` value - # which can be used to "map to" the initialized dec. - - # ''' - # # See "tagged unions" docs: - # # https://jcristharif.com/msgspec/structs.html#tagged-unions - - # # "The quickest way to enable tagged unions is to set tag=True when - # # defining every struct type in the union. In this case tag_field - # # defaults to "type", and tag defaults to the struct class name - # # (e.g. "Get")." - # first: Struct = tagged_structs[0] - # types_union: Union[Type[Struct]] = Union[ - # first - # ]|Any - # tags: list[str] = [first.__name__] - - # for struct in tagged_structs[1:]: - # types_union |= struct - # tags.append( - # getattr( - # struct, - # struct.__struct_config__.tag_field, - # struct.__name__, - # ) - # ) - - # dec = msgpack.Decoder(types_union) - # return ( - # tags, - # dec, - # ) +# -[x] do we still want to try and support the sub-decoder with +# `.Raw` technique in the case that the `Generic` approach gives +# future grief? +# => NO, since we went with the `PldRx` approach instead B) +# +# IF however you want to see the code that was staged for this +# from wayyy back, see the pure removal commit. def mk_codec( @@ -644,10 +496,6 @@ _def_tractor_codec: MsgCodec = mk_codec( # 3. We similarly set the pending values for the child nurseries # of the *current* task. # - -# TODO: STOP USING THIS, since it's basically a global and won't -# allow sub-IPC-ctxs to limit the msg-spec however desired.. -# _ctxvar_MsgCodec: MsgCodec = RunVar( _ctxvar_MsgCodec: ContextVar[MsgCodec] = ContextVar( 'msgspec_codec', default=_def_tractor_codec, @@ -782,3 +630,31 @@ def limit_msg_spec( # # import pdbp; pdbp.set_trace() # assert ext_codec.pld_spec == extended_spec # yield ext_codec + + +# TODO: make something similar to this inside `._codec` such that +# user can just pass a type table of some sort? +# -[ ] we would need to decode all msgs to `pretty_struct.Struct` +# and then call `.to_dict()` on them? +# -[x] we're going to need to re-impl all the stuff changed in the +# runtime port such that it can handle dicts or `Msg`s? +# +# def mk_dict_msg_codec_hooks() -> tuple[Callable, Callable]: +# ''' +# Deliver a `enc_hook()`/`dec_hook()` pair which does +# manual convertion from our above native `Msg` set +# to `dict` equivalent (wire msgs) in order to keep legacy compat +# with the original runtime implementation. +# +# Note: this is is/was primarly used while moving the core +# runtime over to using native `Msg`-struct types wherein we +# start with the send side emitting without loading +# a typed-decoder and then later flipping the switch over to +# load to the native struct types once all runtime usage has +# been adjusted appropriately. +# +# ''' +# return ( +# # enc_to_dict, +# dec_from_dict, +# ) diff --git a/tractor/msg/types.py b/tractor/msg/types.py index ad6d6fb..0fc0ee9 100644 --- a/tractor/msg/types.py +++ b/tractor/msg/types.py @@ -26,7 +26,6 @@ from __future__ import annotations import types from typing import ( Any, - # Callable, Generic, Literal, Type, @@ -161,7 +160,6 @@ class SpawnSpec( bind_addrs: list[tuple[str, int]] - # TODO: caps based RPC support in the payload? # # -[ ] integration with our ``enable_modules: list[str]`` caps sys. @@ -314,8 +312,9 @@ class Started( pld: PayloadT|Raw -# TODO: instead of using our existing `Start` -# for this (as we did with the original `{'cmd': ..}` style) +# TODO: cancel request dedicated msg? +# -[ ] instead of using our existing `Start`? +# # class Cancel: # cid: str @@ -477,12 +476,16 @@ def from_dict_msg( ) return msgT(**dict_msg) -# TODO: should be make a msg version of `ContextCancelled?` -# and/or with a scope field or a full `ActorCancelled`? +# TODO: should be make a set of cancel msgs? +# -[ ] a version of `ContextCancelled`? +# |_ and/or with a scope field? +# -[ ] or, a full `ActorCancelled`? +# # class Cancelled(MsgType): # cid: str - -# TODO what about overruns? +# +# -[ ] what about overruns? +# # class Overrun(MsgType): # cid: str @@ -564,10 +567,17 @@ def mk_msg_spec( Create a payload-(data-)type-parameterized IPC message specification. Allows generating IPC msg types from the above builtin set - with a payload (field) restricted data-type via the `Msg.pld: - PayloadT` type var. This allows runtime-task contexts to use - the python type system to limit/filter payload values as - determined by the input `payload_type_union: Union[Type]`. + with a payload (field) restricted data-type, the `Msg.pld: PayloadT`. + + This allows runtime-task contexts to use the python type system + to limit/filter payload values as determined by the input + `payload_type_union: Union[Type]`. + + Notes: originally multiple approaches for constructing the + type-union passed to `msgspec` were attempted as selected via the + `spec_build_method`, but it turns out only the defaul method + 'indexed_generics' seems to work reliably in all use cases. As + such, the others will likely be removed in the near future. ''' submsg_types: list[MsgType] = Msg.__subclasses__() @@ -707,31 +717,3 @@ def mk_msg_spec( + ipc_msg_types, ) - - -# TODO: make something similar to this inside `._codec` such that -# user can just pass a type table of some sort? -# -[ ] we would need to decode all msgs to `pretty_struct.Struct` -# and then call `.to_dict()` on them? -# -[ ] we're going to need to re-impl all the stuff changed in the -# runtime port such that it can handle dicts or `Msg`s? -# -# def mk_dict_msg_codec_hooks() -> tuple[Callable, Callable]: -# ''' -# Deliver a `enc_hook()`/`dec_hook()` pair which does -# manual convertion from our above native `Msg` set -# to `dict` equivalent (wire msgs) in order to keep legacy compat -# with the original runtime implementation. -# -# Note: this is is/was primarly used while moving the core -# runtime over to using native `Msg`-struct types wherein we -# start with the send side emitting without loading -# a typed-decoder and then later flipping the switch over to -# load to the native struct types once all runtime usage has -# been adjusted appropriately. -# -# ''' -# return ( -# # enc_to_dict, -# dec_from_dict, -# )