Compare commits
No commits in common. "leslies_extra_appendix" and "main" have entirely different histories.
leslies_ex
...
main
19
default.nix
19
default.nix
|
|
@ -1,19 +0,0 @@
|
|||
{ pkgs ? import <nixpkgs> {} }:
|
||||
let
|
||||
nativeBuildInputs = with pkgs; [
|
||||
stdenv.cc.cc.lib
|
||||
uv
|
||||
];
|
||||
|
||||
in
|
||||
pkgs.mkShell {
|
||||
inherit nativeBuildInputs;
|
||||
|
||||
LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath nativeBuildInputs;
|
||||
TMPDIR = "/tmp";
|
||||
|
||||
shellHook = ''
|
||||
set -e
|
||||
uv venv .venv --python=3.12
|
||||
'';
|
||||
}
|
||||
|
|
@ -120,7 +120,6 @@ async def main(
|
|||
break_parent_ipc_after: int|bool = False,
|
||||
break_child_ipc_after: int|bool = False,
|
||||
pre_close: bool = False,
|
||||
tpt_proto: str = 'tcp',
|
||||
|
||||
) -> None:
|
||||
|
||||
|
|
@ -132,7 +131,6 @@ async def main(
|
|||
# a hang since it never engages due to broken IPC
|
||||
debug_mode=debug_mode,
|
||||
loglevel=loglevel,
|
||||
enable_transports=[tpt_proto],
|
||||
|
||||
) as an,
|
||||
):
|
||||
|
|
@ -147,8 +145,7 @@ async def main(
|
|||
_testing.expect_ctxc(
|
||||
yay=(
|
||||
break_parent_ipc_after
|
||||
or
|
||||
break_child_ipc_after
|
||||
or break_child_ipc_after
|
||||
),
|
||||
# TODO: we CAN'T remove this right?
|
||||
# since we need the ctxc to bubble up from either
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ async def main(service_name):
|
|||
async with tractor.open_nursery() as an:
|
||||
await an.start_actor(service_name)
|
||||
|
||||
async with tractor.get_registry() as portal:
|
||||
async with tractor.get_registry('127.0.0.1', 1616) as portal:
|
||||
print(f"Arbiter is listening on {portal.channel}")
|
||||
|
||||
async with tractor.wait_for_actor(service_name) as sockaddr:
|
||||
|
|
|
|||
|
|
@ -45,8 +45,6 @@ dependencies = [
|
|||
"pdbp>=1.6,<2", # windows only (from `pdbp`)
|
||||
# typed IPC msging
|
||||
"msgspec>=0.19.0",
|
||||
"cffi>=1.17.1",
|
||||
"bidict>=0.23.1",
|
||||
]
|
||||
|
||||
# ------ project ------
|
||||
|
|
@ -64,7 +62,6 @@ dev = [
|
|||
"pyperclip>=1.9.0",
|
||||
"prompt-toolkit>=3.0.50",
|
||||
"xonsh>=0.19.2",
|
||||
"psutil>=7.0.0",
|
||||
]
|
||||
# TODO, add these with sane versions; were originally in
|
||||
# `requirements-docs.txt`..
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
"""
|
||||
Top level of the testing suites!
|
||||
|
||||
``tractor`` testing!!
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import sys
|
||||
import subprocess
|
||||
import os
|
||||
|
|
@ -32,11 +30,7 @@ else:
|
|||
_KILL_SIGNAL = signal.SIGKILL
|
||||
_INT_SIGNAL = signal.SIGINT
|
||||
_INT_RETURN_CODE = 1 if sys.version_info < (3, 8) else -signal.SIGINT.value
|
||||
_PROC_SPAWN_WAIT = (
|
||||
0.6
|
||||
if sys.version_info < (3, 7)
|
||||
else 0.4
|
||||
)
|
||||
_PROC_SPAWN_WAIT = 0.6 if sys.version_info < (3, 7) else 0.4
|
||||
|
||||
|
||||
no_windows = pytest.mark.skipif(
|
||||
|
|
@ -45,9 +39,7 @@ no_windows = pytest.mark.skipif(
|
|||
)
|
||||
|
||||
|
||||
def pytest_addoption(
|
||||
parser: pytest.Parser,
|
||||
):
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--ll",
|
||||
action="store",
|
||||
|
|
@ -64,8 +56,7 @@ def pytest_addoption(
|
|||
)
|
||||
|
||||
parser.addoption(
|
||||
"--tpdb",
|
||||
"--debug-mode",
|
||||
"--tpdb", "--debug-mode",
|
||||
action="store_true",
|
||||
dest='tractor_debug_mode',
|
||||
# default=False,
|
||||
|
|
@ -76,17 +67,6 @@ def pytest_addoption(
|
|||
),
|
||||
)
|
||||
|
||||
# provide which IPC transport protocols opting-in test suites
|
||||
# should accumulatively run against.
|
||||
parser.addoption(
|
||||
"--tpt-proto",
|
||||
nargs='+', # accumulate-multiple-args
|
||||
action="store",
|
||||
dest='tpt_protos',
|
||||
default=['tcp'],
|
||||
help="Transport protocol to use under the `tractor.ipc.Channel`",
|
||||
)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
backend = config.option.spawn_backend
|
||||
|
|
@ -94,7 +74,7 @@ def pytest_configure(config):
|
|||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def debug_mode(request) -> bool:
|
||||
def debug_mode(request):
|
||||
debug_mode: bool = request.config.option.tractor_debug_mode
|
||||
# if debug_mode:
|
||||
# breakpoint()
|
||||
|
|
@ -115,35 +95,11 @@ def spawn_backend(request) -> str:
|
|||
return request.config.option.spawn_backend
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def tpt_protos(request) -> list[str]:
|
||||
|
||||
# allow quoting on CLI
|
||||
proto_keys: list[str] = [
|
||||
proto_key.replace('"', '').replace("'", "")
|
||||
for proto_key in request.config.option.tpt_protos
|
||||
]
|
||||
|
||||
# ?TODO, eventually support multiple protos per test-sesh?
|
||||
if len(proto_keys) > 1:
|
||||
pytest.fail(
|
||||
'We only support one `--tpt-proto <key>` atm!\n'
|
||||
)
|
||||
|
||||
# XXX ensure we support the protocol by name via lookup!
|
||||
for proto_key in proto_keys:
|
||||
addr_type = tractor._addr._address_types[proto_key]
|
||||
assert addr_type.proto_key == proto_key
|
||||
|
||||
yield proto_keys
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def tpt_proto(
|
||||
tpt_protos: list[str],
|
||||
) -> str:
|
||||
yield tpt_protos[0]
|
||||
|
||||
# @pytest.fixture(scope='function', autouse=True)
|
||||
# def debug_enabled(request) -> str:
|
||||
# from tractor import _state
|
||||
# if _state._runtime_vars['_debug_mode']:
|
||||
# breakpoint()
|
||||
|
||||
_ci_env: bool = os.environ.get('CI', False)
|
||||
|
||||
|
|
@ -151,7 +107,7 @@ _ci_env: bool = os.environ.get('CI', False)
|
|||
@pytest.fixture(scope='session')
|
||||
def ci_env() -> bool:
|
||||
'''
|
||||
Detect CI environment.
|
||||
Detect CI envoirment.
|
||||
|
||||
'''
|
||||
return _ci_env
|
||||
|
|
@ -159,45 +115,30 @@ def ci_env() -> bool:
|
|||
|
||||
# TODO: also move this to `._testing` for now?
|
||||
# -[ ] possibly generalize and re-use for multi-tree spawning
|
||||
# along with the new stuff for multi-addrs?
|
||||
# along with the new stuff for multi-addrs in distribute_dis
|
||||
# branch?
|
||||
#
|
||||
# choose random port at import time
|
||||
_rando_port: str = random.randint(1000, 9999)
|
||||
# choose randomly at import time
|
||||
_reg_addr: tuple[str, int] = (
|
||||
'127.0.0.1',
|
||||
random.randint(1000, 9999),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def reg_addr(
|
||||
tpt_proto: str,
|
||||
) -> tuple[str, int|str]:
|
||||
def reg_addr() -> tuple[str, int]:
|
||||
|
||||
# globally override the runtime to the per-test-session-dynamic
|
||||
# addr so that all tests never conflict with any other actor
|
||||
# tree using the default.
|
||||
from tractor import (
|
||||
_addr,
|
||||
)
|
||||
addr_type = _addr._address_types[tpt_proto]
|
||||
def_reg_addr: tuple[str, int] = _addr._default_lo_addrs[tpt_proto]
|
||||
from tractor import _root
|
||||
_root._default_lo_addrs = [_reg_addr]
|
||||
|
||||
testrun_reg_addr: tuple[str, int]
|
||||
match tpt_proto:
|
||||
case 'tcp':
|
||||
testrun_reg_addr = (
|
||||
addr_type.def_bindspace,
|
||||
_rando_port,
|
||||
)
|
||||
|
||||
# NOTE, file-name uniqueness (no-collisions) will be based on
|
||||
# the runtime-directory and root (pytest-proc's) pid.
|
||||
case 'uds':
|
||||
testrun_reg_addr = addr_type.get_random().unwrap()
|
||||
|
||||
assert def_reg_addr != testrun_reg_addr
|
||||
return testrun_reg_addr
|
||||
return _reg_addr
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
spawn_backend: str = metafunc.config.option.spawn_backend
|
||||
spawn_backend = metafunc.config.option.spawn_backend
|
||||
|
||||
if not spawn_backend:
|
||||
# XXX some weird windows bug with `pytest`?
|
||||
|
|
@ -210,53 +151,45 @@ def pytest_generate_tests(metafunc):
|
|||
'trio',
|
||||
)
|
||||
|
||||
# NOTE: used-to-be-used-to dyanmically parametrize tests for when
|
||||
# NOTE: used to be used to dyanmically parametrize tests for when
|
||||
# you just passed --spawn-backend=`mp` on the cli, but now we expect
|
||||
# that cli input to be manually specified, BUT, maybe we'll do
|
||||
# something like this again in the future?
|
||||
if 'start_method' in metafunc.fixturenames:
|
||||
metafunc.parametrize(
|
||||
"start_method",
|
||||
[spawn_backend],
|
||||
scope='module',
|
||||
)
|
||||
metafunc.parametrize("start_method", [spawn_backend], scope='module')
|
||||
|
||||
# TODO, parametrize any `tpt_proto: str` declaring tests!
|
||||
# proto_tpts: list[str] = metafunc.config.option.proto_tpts
|
||||
# if 'tpt_proto' in metafunc.fixturenames:
|
||||
# metafunc.parametrize(
|
||||
# 'tpt_proto',
|
||||
# proto_tpts, # TODO, double check this list usage!
|
||||
# scope='module',
|
||||
|
||||
# TODO: a way to let test scripts (like from `examples/`)
|
||||
# guarantee they won't registry addr collide!
|
||||
# @pytest.fixture
|
||||
# def open_test_runtime(
|
||||
# reg_addr: tuple,
|
||||
# ) -> AsyncContextManager:
|
||||
# return partial(
|
||||
# tractor.open_nursery,
|
||||
# registry_addrs=[reg_addr],
|
||||
# )
|
||||
|
||||
|
||||
def sig_prog(
|
||||
proc: subprocess.Popen,
|
||||
sig: int,
|
||||
canc_timeout: float = 0.1,
|
||||
) -> int:
|
||||
def sig_prog(proc, sig):
|
||||
"Kill the actor-process with ``sig``."
|
||||
proc.send_signal(sig)
|
||||
time.sleep(canc_timeout)
|
||||
time.sleep(0.1)
|
||||
if not proc.poll():
|
||||
# TODO: why sometimes does SIGINT not work on teardown?
|
||||
# seems to happen only when trace logging enabled?
|
||||
proc.send_signal(_KILL_SIGNAL)
|
||||
ret: int = proc.wait()
|
||||
ret = proc.wait()
|
||||
assert ret
|
||||
|
||||
|
||||
# TODO: factor into @cm and move to `._testing`?
|
||||
@pytest.fixture
|
||||
def daemon(
|
||||
debug_mode: bool,
|
||||
loglevel: str,
|
||||
testdir,
|
||||
reg_addr: tuple[str, int],
|
||||
tpt_proto: str,
|
||||
|
||||
) -> subprocess.Popen:
|
||||
):
|
||||
'''
|
||||
Run a daemon root actor as a separate actor-process tree and
|
||||
"remote registrar" for discovery-protocol related tests.
|
||||
|
|
@ -268,99 +201,27 @@ def daemon(
|
|||
|
||||
code: str = (
|
||||
"import tractor; "
|
||||
"tractor.run_daemon([], "
|
||||
"registry_addrs={reg_addrs}, "
|
||||
"debug_mode={debug_mode}, "
|
||||
"loglevel={ll})"
|
||||
"tractor.run_daemon([], registry_addrs={reg_addrs}, loglevel={ll})"
|
||||
).format(
|
||||
reg_addrs=str([reg_addr]),
|
||||
ll="'{}'".format(loglevel) if loglevel else None,
|
||||
debug_mode=debug_mode,
|
||||
)
|
||||
cmd: list[str] = [
|
||||
sys.executable,
|
||||
'-c', code,
|
||||
]
|
||||
# breakpoint()
|
||||
kwargs = {}
|
||||
if platform.system() == 'Windows':
|
||||
# without this, tests hang on windows forever
|
||||
kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
|
||||
|
||||
proc: subprocess.Popen = testdir.popen(
|
||||
proc = testdir.popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# UDS sockets are **really** fast to bind()/listen()/connect()
|
||||
# so it's often required that we delay a bit more starting
|
||||
# the first actor-tree..
|
||||
if tpt_proto == 'uds':
|
||||
global _PROC_SPAWN_WAIT
|
||||
_PROC_SPAWN_WAIT = 0.6
|
||||
|
||||
time.sleep(_PROC_SPAWN_WAIT)
|
||||
|
||||
assert not proc.returncode
|
||||
time.sleep(_PROC_SPAWN_WAIT)
|
||||
yield proc
|
||||
sig_prog(proc, _INT_SIGNAL)
|
||||
|
||||
# XXX! yeah.. just be reaaal careful with this bc sometimes it
|
||||
# can lock up on the `_io.BufferedReader` and hang..
|
||||
stderr: str = proc.stderr.read().decode()
|
||||
if stderr:
|
||||
print(
|
||||
f'Daemon actor tree produced STDERR:\n'
|
||||
f'{proc.args}\n'
|
||||
f'\n'
|
||||
f'{stderr}\n'
|
||||
)
|
||||
if proc.returncode != -2:
|
||||
raise RuntimeError(
|
||||
'Daemon actor tree failed !?\n'
|
||||
f'{proc.args}\n'
|
||||
)
|
||||
|
||||
|
||||
# @pytest.fixture(autouse=True)
|
||||
# def shared_last_failed(pytestconfig):
|
||||
# val = pytestconfig.cache.get("example/value", None)
|
||||
# breakpoint()
|
||||
# if val is None:
|
||||
# pytestconfig.cache.set("example/value", val)
|
||||
# return val
|
||||
|
||||
|
||||
# TODO: a way to let test scripts (like from `examples/`)
|
||||
# guarantee they won't `registry_addrs` collide!
|
||||
# -[ ] maybe use some kinda standard `def main()` arg-spec that
|
||||
# we can introspect from a fixture that is called from the test
|
||||
# body?
|
||||
# -[ ] test and figure out typing for below prototype! Bp
|
||||
#
|
||||
# @pytest.fixture
|
||||
# def set_script_runtime_args(
|
||||
# reg_addr: tuple,
|
||||
# ) -> Callable[[...], None]:
|
||||
|
||||
# def import_n_partial_in_args_n_triorun(
|
||||
# script: Path, # under examples?
|
||||
# **runtime_args,
|
||||
# ) -> Callable[[], Any]: # a `partial`-ed equiv of `trio.run()`
|
||||
|
||||
# # NOTE, below is taken from
|
||||
# # `.test_advanced_faults.test_ipc_channel_break_during_stream`
|
||||
# mod: ModuleType = import_path(
|
||||
# examples_dir() / 'advanced_faults'
|
||||
# / 'ipc_failure_during_stream.py',
|
||||
# root=examples_dir(),
|
||||
# consider_namespace_packages=False,
|
||||
# )
|
||||
# return partial(
|
||||
# trio.run,
|
||||
# partial(
|
||||
# mod.main,
|
||||
# **runtime_args,
|
||||
# )
|
||||
# )
|
||||
# return import_n_partial_in_args_n_triorun
|
||||
|
|
|
|||
|
|
@ -10,9 +10,6 @@ import pytest
|
|||
from _pytest.pathlib import import_path
|
||||
import trio
|
||||
import tractor
|
||||
from tractor import (
|
||||
TransportClosed,
|
||||
)
|
||||
from tractor._testing import (
|
||||
examples_dir,
|
||||
break_ipc,
|
||||
|
|
@ -77,7 +74,6 @@ def test_ipc_channel_break_during_stream(
|
|||
spawn_backend: str,
|
||||
ipc_break: dict|None,
|
||||
pre_aclose_msgstream: bool,
|
||||
tpt_proto: str,
|
||||
):
|
||||
'''
|
||||
Ensure we can have an IPC channel break its connection during
|
||||
|
|
@ -95,7 +91,7 @@ def test_ipc_channel_break_during_stream(
|
|||
# non-`trio` spawners should never hit the hang condition that
|
||||
# requires the user to do ctl-c to cancel the actor tree.
|
||||
# expect_final_exc = trio.ClosedResourceError
|
||||
expect_final_exc = TransportClosed
|
||||
expect_final_exc = tractor.TransportClosed
|
||||
|
||||
mod: ModuleType = import_path(
|
||||
examples_dir() / 'advanced_faults'
|
||||
|
|
@ -108,8 +104,6 @@ def test_ipc_channel_break_during_stream(
|
|||
# period" wherein the user eventually hits ctl-c to kill the
|
||||
# root-actor tree.
|
||||
expect_final_exc: BaseException = KeyboardInterrupt
|
||||
expect_final_cause: BaseException|None = None
|
||||
|
||||
if (
|
||||
# only expect EoC if trans is broken on the child side,
|
||||
ipc_break['break_child_ipc_after'] is not False
|
||||
|
|
@ -144,9 +138,6 @@ def test_ipc_channel_break_during_stream(
|
|||
# a user sending ctl-c by raising a KBI.
|
||||
if pre_aclose_msgstream:
|
||||
expect_final_exc = KeyboardInterrupt
|
||||
if tpt_proto == 'uds':
|
||||
expect_final_exc = TransportClosed
|
||||
expect_final_cause = trio.BrokenResourceError
|
||||
|
||||
# XXX OLD XXX
|
||||
# if child calls `MsgStream.aclose()` then expect EoC.
|
||||
|
|
@ -166,10 +157,6 @@ def test_ipc_channel_break_during_stream(
|
|||
if pre_aclose_msgstream:
|
||||
expect_final_exc = KeyboardInterrupt
|
||||
|
||||
if tpt_proto == 'uds':
|
||||
expect_final_exc = TransportClosed
|
||||
expect_final_cause = trio.BrokenResourceError
|
||||
|
||||
# NOTE when the parent IPC side dies (even if the child does as well
|
||||
# but the child fails BEFORE the parent) we always expect the
|
||||
# IPC layer to raise a closed-resource, NEVER do we expect
|
||||
|
|
@ -182,8 +169,8 @@ def test_ipc_channel_break_during_stream(
|
|||
and
|
||||
ipc_break['break_child_ipc_after'] is False
|
||||
):
|
||||
# expect_final_exc = trio.ClosedResourceError
|
||||
expect_final_exc = tractor.TransportClosed
|
||||
expect_final_cause = trio.ClosedResourceError
|
||||
|
||||
# BOTH but, PARENT breaks FIRST
|
||||
elif (
|
||||
|
|
@ -194,8 +181,8 @@ def test_ipc_channel_break_during_stream(
|
|||
ipc_break['break_parent_ipc_after']
|
||||
)
|
||||
):
|
||||
# expect_final_exc = trio.ClosedResourceError
|
||||
expect_final_exc = tractor.TransportClosed
|
||||
expect_final_cause = trio.ClosedResourceError
|
||||
|
||||
with pytest.raises(
|
||||
expected_exception=(
|
||||
|
|
@ -211,7 +198,6 @@ def test_ipc_channel_break_during_stream(
|
|||
start_method=spawn_backend,
|
||||
loglevel=loglevel,
|
||||
pre_close=pre_aclose_msgstream,
|
||||
tpt_proto=tpt_proto,
|
||||
**ipc_break,
|
||||
)
|
||||
)
|
||||
|
|
@ -234,15 +220,10 @@ def test_ipc_channel_break_during_stream(
|
|||
)
|
||||
cause: Exception = tc.__cause__
|
||||
assert (
|
||||
# type(cause) is trio.ClosedResourceError
|
||||
type(cause) is expect_final_cause
|
||||
|
||||
# TODO, should we expect a certain exc-message (per
|
||||
# tpt) as well??
|
||||
# and
|
||||
# cause.args[0] == 'another task closed this fd'
|
||||
type(cause) is trio.ClosedResourceError
|
||||
and
|
||||
cause.args[0] == 'another task closed this fd'
|
||||
)
|
||||
|
||||
raise
|
||||
|
||||
# get raw instance from pytest wrapper
|
||||
|
|
|
|||
|
|
@ -7,9 +7,7 @@ import platform
|
|||
from functools import partial
|
||||
import itertools
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
import subprocess
|
||||
import tractor
|
||||
from tractor._testing import tractor_test
|
||||
import trio
|
||||
|
|
@ -28,7 +26,7 @@ async def test_reg_then_unreg(reg_addr):
|
|||
portal = await n.start_actor('actor', enable_modules=[__name__])
|
||||
uid = portal.channel.uid
|
||||
|
||||
async with tractor.get_registry(reg_addr) as aportal:
|
||||
async with tractor.get_registry(*reg_addr) as aportal:
|
||||
# this local actor should be the arbiter
|
||||
assert actor is aportal.actor
|
||||
|
||||
|
|
@ -154,25 +152,15 @@ async def unpack_reg(actor_or_portal):
|
|||
async def spawn_and_check_registry(
|
||||
reg_addr: tuple,
|
||||
use_signal: bool,
|
||||
debug_mode: bool = False,
|
||||
remote_arbiter: bool = False,
|
||||
with_streaming: bool = False,
|
||||
maybe_daemon: tuple[
|
||||
subprocess.Popen,
|
||||
psutil.Process,
|
||||
]|None = None,
|
||||
|
||||
) -> None:
|
||||
|
||||
if maybe_daemon:
|
||||
popen, proc = maybe_daemon
|
||||
# breakpoint()
|
||||
|
||||
async with tractor.open_root_actor(
|
||||
registry_addrs=[reg_addr],
|
||||
debug_mode=debug_mode,
|
||||
):
|
||||
async with tractor.get_registry(reg_addr) as portal:
|
||||
async with tractor.get_registry(*reg_addr) as portal:
|
||||
# runtime needs to be up to call this
|
||||
actor = tractor.current_actor()
|
||||
|
||||
|
|
@ -188,11 +176,11 @@ async def spawn_and_check_registry(
|
|||
extra = 2 # local root actor + remote arbiter
|
||||
|
||||
# ensure current actor is registered
|
||||
registry: dict = await get_reg()
|
||||
registry = await get_reg()
|
||||
assert actor.uid in registry
|
||||
|
||||
try:
|
||||
async with tractor.open_nursery() as an:
|
||||
async with tractor.open_nursery() as n:
|
||||
async with trio.open_nursery(
|
||||
strict_exception_groups=False,
|
||||
) as trion:
|
||||
|
|
@ -201,17 +189,17 @@ async def spawn_and_check_registry(
|
|||
for i in range(3):
|
||||
name = f'a{i}'
|
||||
if with_streaming:
|
||||
portals[name] = await an.start_actor(
|
||||
portals[name] = await n.start_actor(
|
||||
name=name, enable_modules=[__name__])
|
||||
|
||||
else: # no streaming
|
||||
portals[name] = await an.run_in_actor(
|
||||
portals[name] = await n.run_in_actor(
|
||||
trio.sleep_forever, name=name)
|
||||
|
||||
# wait on last actor to come up
|
||||
async with tractor.wait_for_actor(name):
|
||||
registry = await get_reg()
|
||||
for uid in an._children:
|
||||
for uid in n._children:
|
||||
assert uid in registry
|
||||
|
||||
assert len(portals) + extra == len(registry)
|
||||
|
|
@ -244,7 +232,6 @@ async def spawn_and_check_registry(
|
|||
@pytest.mark.parametrize('use_signal', [False, True])
|
||||
@pytest.mark.parametrize('with_streaming', [False, True])
|
||||
def test_subactors_unregister_on_cancel(
|
||||
debug_mode: bool,
|
||||
start_method,
|
||||
use_signal,
|
||||
reg_addr,
|
||||
|
|
@ -261,7 +248,6 @@ def test_subactors_unregister_on_cancel(
|
|||
spawn_and_check_registry,
|
||||
reg_addr,
|
||||
use_signal,
|
||||
debug_mode=debug_mode,
|
||||
remote_arbiter=False,
|
||||
with_streaming=with_streaming,
|
||||
),
|
||||
|
|
@ -271,8 +257,7 @@ def test_subactors_unregister_on_cancel(
|
|||
@pytest.mark.parametrize('use_signal', [False, True])
|
||||
@pytest.mark.parametrize('with_streaming', [False, True])
|
||||
def test_subactors_unregister_on_cancel_remote_daemon(
|
||||
daemon: subprocess.Popen,
|
||||
debug_mode: bool,
|
||||
daemon,
|
||||
start_method,
|
||||
use_signal,
|
||||
reg_addr,
|
||||
|
|
@ -288,13 +273,8 @@ def test_subactors_unregister_on_cancel_remote_daemon(
|
|||
spawn_and_check_registry,
|
||||
reg_addr,
|
||||
use_signal,
|
||||
debug_mode=debug_mode,
|
||||
remote_arbiter=True,
|
||||
with_streaming=with_streaming,
|
||||
maybe_daemon=(
|
||||
daemon,
|
||||
psutil.Process(daemon.pid)
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -320,7 +300,7 @@ async def close_chans_before_nursery(
|
|||
async with tractor.open_root_actor(
|
||||
registry_addrs=[reg_addr],
|
||||
):
|
||||
async with tractor.get_registry(reg_addr) as aportal:
|
||||
async with tractor.get_registry(*reg_addr) as aportal:
|
||||
try:
|
||||
get_reg = partial(unpack_reg, aportal)
|
||||
|
||||
|
|
@ -393,7 +373,7 @@ def test_close_channel_explicit(
|
|||
|
||||
@pytest.mark.parametrize('use_signal', [False, True])
|
||||
def test_close_channel_explicit_remote_arbiter(
|
||||
daemon: subprocess.Popen,
|
||||
daemon,
|
||||
start_method,
|
||||
use_signal,
|
||||
reg_addr,
|
||||
|
|
|
|||
|
|
@ -66,9 +66,6 @@ def run_example_in_subproc(
|
|||
# due to backpressure!!!
|
||||
proc = testdir.popen(
|
||||
cmdargs,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
**kwargs,
|
||||
)
|
||||
assert not proc.returncode
|
||||
|
|
@ -122,14 +119,10 @@ def test_example(
|
|||
code = ex.read()
|
||||
|
||||
with run_example_in_subproc(code) as proc:
|
||||
err = None
|
||||
try:
|
||||
if not proc.poll():
|
||||
_, err = proc.communicate(timeout=15)
|
||||
|
||||
except subprocess.TimeoutExpired as e:
|
||||
proc.kill()
|
||||
err = e.stderr
|
||||
proc.wait()
|
||||
err, _ = proc.stderr.read(), proc.stdout.read()
|
||||
# print(f'STDERR: {err}')
|
||||
# print(f'STDOUT: {out}')
|
||||
|
||||
# if we get some gnarly output let's aggregate and raise
|
||||
if err:
|
||||
|
|
|
|||
|
|
@ -871,7 +871,7 @@ async def serve_subactors(
|
|||
)
|
||||
await ipc.send((
|
||||
peer.chan.uid,
|
||||
peer.chan.raddr.unwrap(),
|
||||
peer.chan.raddr,
|
||||
))
|
||||
|
||||
print('Spawner exiting spawn serve loop!')
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ async def test_self_is_registered_localportal(reg_addr):
|
|||
"Verify waiting on the arbiter to register itself using a local portal."
|
||||
actor = tractor.current_actor()
|
||||
assert actor.is_arbiter
|
||||
async with tractor.get_registry(reg_addr) as portal:
|
||||
async with tractor.get_registry(*reg_addr) as portal:
|
||||
assert isinstance(portal, tractor._portal.LocalPortal)
|
||||
|
||||
with trio.fail_after(0.2):
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ def test_abort_on_sigint(daemon):
|
|||
@tractor_test
|
||||
async def test_cancel_remote_arbiter(daemon, reg_addr):
|
||||
assert not tractor.current_actor().is_arbiter
|
||||
async with tractor.get_registry(reg_addr) as portal:
|
||||
async with tractor.get_registry(*reg_addr) as portal:
|
||||
await portal.cancel_actor()
|
||||
|
||||
time.sleep(0.1)
|
||||
|
|
@ -41,7 +41,7 @@ async def test_cancel_remote_arbiter(daemon, reg_addr):
|
|||
|
||||
# no arbiter socket should exist
|
||||
with pytest.raises(OSError):
|
||||
async with tractor.get_registry(reg_addr) as portal:
|
||||
async with tractor.get_registry(*reg_addr) as portal:
|
||||
pass
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,208 +0,0 @@
|
|||
import time
|
||||
|
||||
import trio
|
||||
import pytest
|
||||
import tractor
|
||||
from tractor.ipc import (
|
||||
open_ringbuf,
|
||||
RBToken,
|
||||
RingBuffSender,
|
||||
RingBuffReceiver
|
||||
)
|
||||
from tractor._testing.samples import generate_sample_messages
|
||||
|
||||
# in case you don't want to melt your cores, uncomment dis!
|
||||
pytestmark = pytest.mark.skip
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def child_read_shm(
|
||||
ctx: tractor.Context,
|
||||
msg_amount: int,
|
||||
token: RBToken,
|
||||
total_bytes: int,
|
||||
) -> None:
|
||||
recvd_bytes = 0
|
||||
await ctx.started()
|
||||
start_ts = time.time()
|
||||
async with RingBuffReceiver(token) as receiver:
|
||||
while recvd_bytes < total_bytes:
|
||||
msg = await receiver.receive_some()
|
||||
recvd_bytes += len(msg)
|
||||
|
||||
# make sure we dont hold any memoryviews
|
||||
# before the ctx manager aclose()
|
||||
msg = None
|
||||
|
||||
end_ts = time.time()
|
||||
elapsed = end_ts - start_ts
|
||||
elapsed_ms = int(elapsed * 1000)
|
||||
|
||||
print(f'\n\telapsed ms: {elapsed_ms}')
|
||||
print(f'\tmsg/sec: {int(msg_amount / elapsed):,}')
|
||||
print(f'\tbytes/sec: {int(recvd_bytes / elapsed):,}')
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def child_write_shm(
|
||||
ctx: tractor.Context,
|
||||
msg_amount: int,
|
||||
rand_min: int,
|
||||
rand_max: int,
|
||||
token: RBToken,
|
||||
) -> None:
|
||||
msgs, total_bytes = generate_sample_messages(
|
||||
msg_amount,
|
||||
rand_min=rand_min,
|
||||
rand_max=rand_max,
|
||||
)
|
||||
await ctx.started(total_bytes)
|
||||
async with RingBuffSender(token) as sender:
|
||||
for msg in msgs:
|
||||
await sender.send_all(msg)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'msg_amount,rand_min,rand_max,buf_size',
|
||||
[
|
||||
# simple case, fixed payloads, large buffer
|
||||
(100_000, 0, 0, 10 * 1024),
|
||||
|
||||
# guaranteed wrap around on every write
|
||||
(100, 10 * 1024, 20 * 1024, 10 * 1024),
|
||||
|
||||
# large payload size, but large buffer
|
||||
(10_000, 256 * 1024, 512 * 1024, 10 * 1024 * 1024)
|
||||
],
|
||||
ids=[
|
||||
'fixed_payloads_large_buffer',
|
||||
'wrap_around_every_write',
|
||||
'large_payloads_large_buffer',
|
||||
]
|
||||
)
|
||||
def test_ringbuf(
|
||||
msg_amount: int,
|
||||
rand_min: int,
|
||||
rand_max: int,
|
||||
buf_size: int
|
||||
):
|
||||
async def main():
|
||||
with open_ringbuf(
|
||||
'test_ringbuf',
|
||||
buf_size=buf_size
|
||||
) as token:
|
||||
proc_kwargs = {
|
||||
'pass_fds': (token.write_eventfd, token.wrap_eventfd)
|
||||
}
|
||||
|
||||
common_kwargs = {
|
||||
'msg_amount': msg_amount,
|
||||
'token': token,
|
||||
}
|
||||
async with tractor.open_nursery() as an:
|
||||
send_p = await an.start_actor(
|
||||
'ring_sender',
|
||||
enable_modules=[__name__],
|
||||
proc_kwargs=proc_kwargs
|
||||
)
|
||||
recv_p = await an.start_actor(
|
||||
'ring_receiver',
|
||||
enable_modules=[__name__],
|
||||
proc_kwargs=proc_kwargs
|
||||
)
|
||||
async with (
|
||||
send_p.open_context(
|
||||
child_write_shm,
|
||||
rand_min=rand_min,
|
||||
rand_max=rand_max,
|
||||
**common_kwargs
|
||||
) as (sctx, total_bytes),
|
||||
recv_p.open_context(
|
||||
child_read_shm,
|
||||
**common_kwargs,
|
||||
total_bytes=total_bytes,
|
||||
) as (sctx, _sent),
|
||||
):
|
||||
await recv_p.result()
|
||||
|
||||
await send_p.cancel_actor()
|
||||
await recv_p.cancel_actor()
|
||||
|
||||
|
||||
trio.run(main)
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def child_blocked_receiver(
|
||||
ctx: tractor.Context,
|
||||
token: RBToken
|
||||
):
|
||||
async with RingBuffReceiver(token) as receiver:
|
||||
await ctx.started()
|
||||
await receiver.receive_some()
|
||||
|
||||
|
||||
def test_ring_reader_cancel():
|
||||
async def main():
|
||||
with open_ringbuf('test_ring_cancel_reader') as token:
|
||||
async with (
|
||||
tractor.open_nursery() as an,
|
||||
RingBuffSender(token) as _sender,
|
||||
):
|
||||
recv_p = await an.start_actor(
|
||||
'ring_blocked_receiver',
|
||||
enable_modules=[__name__],
|
||||
proc_kwargs={
|
||||
'pass_fds': (token.write_eventfd, token.wrap_eventfd)
|
||||
}
|
||||
)
|
||||
async with (
|
||||
recv_p.open_context(
|
||||
child_blocked_receiver,
|
||||
token=token
|
||||
) as (sctx, _sent),
|
||||
):
|
||||
await trio.sleep(1)
|
||||
await an.cancel()
|
||||
|
||||
|
||||
with pytest.raises(tractor._exceptions.ContextCancelled):
|
||||
trio.run(main)
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def child_blocked_sender(
|
||||
ctx: tractor.Context,
|
||||
token: RBToken
|
||||
):
|
||||
async with RingBuffSender(token) as sender:
|
||||
await ctx.started()
|
||||
await sender.send_all(b'this will wrap')
|
||||
|
||||
|
||||
def test_ring_sender_cancel():
|
||||
async def main():
|
||||
with open_ringbuf(
|
||||
'test_ring_cancel_sender',
|
||||
buf_size=1
|
||||
) as token:
|
||||
async with tractor.open_nursery() as an:
|
||||
recv_p = await an.start_actor(
|
||||
'ring_blocked_sender',
|
||||
enable_modules=[__name__],
|
||||
proc_kwargs={
|
||||
'pass_fds': (token.write_eventfd, token.wrap_eventfd)
|
||||
}
|
||||
)
|
||||
async with (
|
||||
recv_p.open_context(
|
||||
child_blocked_sender,
|
||||
token=token
|
||||
) as (sctx, _sent),
|
||||
):
|
||||
await trio.sleep(1)
|
||||
await an.cancel()
|
||||
|
||||
|
||||
with pytest.raises(tractor._exceptions.ContextCancelled):
|
||||
trio.run(main)
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
'''
|
||||
Runtime boot/init sanity.
|
||||
|
||||
'''
|
||||
|
||||
import pytest
|
||||
import trio
|
||||
|
||||
import tractor
|
||||
from tractor._exceptions import RuntimeFailure
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_new_root_in_sub(
|
||||
ctx: tractor.Context,
|
||||
) -> None:
|
||||
|
||||
async with tractor.open_root_actor():
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'open_root_in',
|
||||
['root', 'sub'],
|
||||
ids='open_2nd_root_in={}'.format,
|
||||
)
|
||||
def test_only_one_root_actor(
|
||||
open_root_in: str,
|
||||
reg_addr: tuple,
|
||||
debug_mode: bool
|
||||
):
|
||||
'''
|
||||
Verify we specially fail whenever more then one root actor
|
||||
is attempted to be opened within an already opened tree.
|
||||
|
||||
'''
|
||||
async def main():
|
||||
async with tractor.open_nursery() as an:
|
||||
|
||||
if open_root_in == 'root':
|
||||
async with tractor.open_root_actor(
|
||||
registry_addrs=[reg_addr],
|
||||
):
|
||||
pass
|
||||
|
||||
ptl: tractor.Portal = await an.start_actor(
|
||||
name='bad_rooty_boi',
|
||||
enable_modules=[__name__],
|
||||
)
|
||||
|
||||
async with ptl.open_context(
|
||||
open_new_root_in_sub,
|
||||
) as (ctx, first):
|
||||
pass
|
||||
|
||||
if open_root_in == 'root':
|
||||
with pytest.raises(
|
||||
RuntimeFailure
|
||||
) as excinfo:
|
||||
trio.run(main)
|
||||
|
||||
else:
|
||||
with pytest.raises(
|
||||
tractor.RemoteActorError,
|
||||
) as excinfo:
|
||||
trio.run(main)
|
||||
|
||||
assert excinfo.value.boxed_type is RuntimeFailure
|
||||
|
||||
|
||||
def test_implicit_root_via_first_nursery(
|
||||
reg_addr: tuple,
|
||||
debug_mode: bool
|
||||
):
|
||||
'''
|
||||
The first `ActorNursery` open should implicitly call
|
||||
`_root.open_root_actor()`.
|
||||
|
||||
'''
|
||||
async def main():
|
||||
async with tractor.open_nursery() as an:
|
||||
assert an._implicit_runtime_started
|
||||
assert tractor.current_actor().aid.name == 'root'
|
||||
|
||||
trio.run(main)
|
||||
|
|
@ -1,167 +0,0 @@
|
|||
"""
|
||||
Shared mem primitives and APIs.
|
||||
|
||||
"""
|
||||
import uuid
|
||||
|
||||
# import numpy
|
||||
import pytest
|
||||
import trio
|
||||
import tractor
|
||||
from tractor.ipc._shm import (
|
||||
open_shm_list,
|
||||
attach_shm_list,
|
||||
)
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def child_attach_shml_alot(
|
||||
ctx: tractor.Context,
|
||||
shm_key: str,
|
||||
) -> None:
|
||||
|
||||
await ctx.started(shm_key)
|
||||
|
||||
# now try to attach a boatload of times in a loop..
|
||||
for _ in range(1000):
|
||||
shml = attach_shm_list(
|
||||
key=shm_key,
|
||||
readonly=False,
|
||||
)
|
||||
assert shml.shm.name == shm_key
|
||||
await trio.sleep(0.001)
|
||||
|
||||
|
||||
def test_child_attaches_alot():
|
||||
async def main():
|
||||
async with tractor.open_nursery() as an:
|
||||
|
||||
# allocate writeable list in parent
|
||||
key = f'shml_{uuid.uuid4()}'
|
||||
shml = open_shm_list(
|
||||
key=key,
|
||||
)
|
||||
|
||||
portal = await an.start_actor(
|
||||
'shm_attacher',
|
||||
enable_modules=[__name__],
|
||||
)
|
||||
|
||||
async with (
|
||||
portal.open_context(
|
||||
child_attach_shml_alot,
|
||||
shm_key=shml.key,
|
||||
) as (ctx, start_val),
|
||||
):
|
||||
assert start_val == key
|
||||
await ctx.result()
|
||||
|
||||
await portal.cancel_actor()
|
||||
|
||||
trio.run(main)
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def child_read_shm_list(
|
||||
ctx: tractor.Context,
|
||||
shm_key: str,
|
||||
use_str: bool,
|
||||
frame_size: int,
|
||||
) -> None:
|
||||
|
||||
# attach in child
|
||||
shml = attach_shm_list(
|
||||
key=shm_key,
|
||||
# dtype=str if use_str else float,
|
||||
)
|
||||
await ctx.started(shml.key)
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
async for i in stream:
|
||||
print(f'(child): reading shm list index: {i}')
|
||||
|
||||
if use_str:
|
||||
expect = str(float(i))
|
||||
else:
|
||||
expect = float(i)
|
||||
|
||||
if frame_size == 1:
|
||||
val = shml[i]
|
||||
assert expect == val
|
||||
print(f'(child): reading value: {val}')
|
||||
else:
|
||||
frame = shml[i - frame_size:i]
|
||||
print(f'(child): reading frame: {frame}')
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'use_str',
|
||||
[False, True],
|
||||
ids=lambda i: f'use_str_values={i}',
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
'frame_size',
|
||||
[1, 2**6, 2**10],
|
||||
ids=lambda i: f'frame_size={i}',
|
||||
)
|
||||
def test_parent_writer_child_reader(
|
||||
use_str: bool,
|
||||
frame_size: int,
|
||||
):
|
||||
|
||||
async def main():
|
||||
async with tractor.open_nursery(
|
||||
# debug_mode=True,
|
||||
) as an:
|
||||
|
||||
portal = await an.start_actor(
|
||||
'shm_reader',
|
||||
enable_modules=[__name__],
|
||||
debug_mode=True,
|
||||
)
|
||||
|
||||
# allocate writeable list in parent
|
||||
key = 'shm_list'
|
||||
seq_size = int(2 * 2 ** 10)
|
||||
shml = open_shm_list(
|
||||
key=key,
|
||||
size=seq_size,
|
||||
dtype=str if use_str else float,
|
||||
readonly=False,
|
||||
)
|
||||
|
||||
async with (
|
||||
portal.open_context(
|
||||
child_read_shm_list,
|
||||
shm_key=key,
|
||||
use_str=use_str,
|
||||
frame_size=frame_size,
|
||||
) as (ctx, sent),
|
||||
|
||||
ctx.open_stream() as stream,
|
||||
):
|
||||
|
||||
assert sent == key
|
||||
|
||||
for i in range(seq_size):
|
||||
|
||||
val = float(i)
|
||||
if use_str:
|
||||
val = str(val)
|
||||
|
||||
# print(f'(parent): writing {val}')
|
||||
shml[i] = val
|
||||
|
||||
# only on frame fills do we
|
||||
# signal to the child that a frame's
|
||||
# worth is ready.
|
||||
if (i % frame_size) == 0:
|
||||
print(f'(parent): signalling frame full on {val}')
|
||||
await stream.send(i)
|
||||
else:
|
||||
print(f'(parent): signalling final frame on {val}')
|
||||
await stream.send(i)
|
||||
|
||||
await portal.cancel_actor()
|
||||
|
||||
trio.run(main)
|
||||
|
|
@ -2,7 +2,6 @@
|
|||
Spawning basics
|
||||
|
||||
"""
|
||||
from functools import partial
|
||||
from typing import (
|
||||
Any,
|
||||
)
|
||||
|
|
@ -13,95 +12,74 @@ import tractor
|
|||
|
||||
from tractor._testing import tractor_test
|
||||
|
||||
data_to_pass_down = {
|
||||
'doggy': 10,
|
||||
'kitty': 4,
|
||||
}
|
||||
data_to_pass_down = {'doggy': 10, 'kitty': 4}
|
||||
|
||||
|
||||
async def spawn(
|
||||
should_be_root: bool,
|
||||
is_arbiter: bool,
|
||||
data: dict,
|
||||
reg_addr: tuple[str, int],
|
||||
|
||||
debug_mode: bool = False,
|
||||
):
|
||||
namespaces = [__name__]
|
||||
|
||||
await trio.sleep(0.1)
|
||||
actor = tractor.current_actor(err_on_no_runtime=False)
|
||||
|
||||
if should_be_root:
|
||||
assert actor is None # no runtime yet
|
||||
async with (
|
||||
tractor.open_root_actor(
|
||||
async with tractor.open_root_actor(
|
||||
arbiter_addr=reg_addr,
|
||||
),
|
||||
tractor.open_nursery() as an,
|
||||
):
|
||||
# now runtime exists
|
||||
actor: tractor.Actor = tractor.current_actor()
|
||||
assert actor.is_arbiter == should_be_root
|
||||
actor = tractor.current_actor()
|
||||
assert actor.is_arbiter == is_arbiter
|
||||
data = data_to_pass_down
|
||||
|
||||
# spawns subproc here
|
||||
portal: tractor.Portal = await an.run_in_actor(
|
||||
fn=spawn,
|
||||
if actor.is_arbiter:
|
||||
async with tractor.open_nursery() as nursery:
|
||||
|
||||
# spawning args
|
||||
# forks here
|
||||
portal = await nursery.run_in_actor(
|
||||
spawn,
|
||||
is_arbiter=False,
|
||||
name='sub-actor',
|
||||
enable_modules=[__name__],
|
||||
|
||||
# passed to a subactor-recursive RPC invoke
|
||||
# of this same `spawn()` fn.
|
||||
should_be_root=False,
|
||||
data=data_to_pass_down,
|
||||
data=data,
|
||||
reg_addr=reg_addr,
|
||||
enable_modules=namespaces,
|
||||
)
|
||||
|
||||
assert len(an._children) == 1
|
||||
assert len(nursery._children) == 1
|
||||
assert portal.channel.uid in tractor.current_actor()._peers
|
||||
|
||||
# get result from child subactor
|
||||
# be sure we can still get the result
|
||||
result = await portal.result()
|
||||
assert result == 10
|
||||
return result
|
||||
else:
|
||||
assert actor.is_arbiter == should_be_root
|
||||
return 10
|
||||
|
||||
|
||||
def test_run_in_actor_same_func_in_child(
|
||||
reg_addr: tuple,
|
||||
debug_mode: bool,
|
||||
def test_local_arbiter_subactor_global_state(
|
||||
reg_addr,
|
||||
):
|
||||
result = trio.run(
|
||||
partial(
|
||||
spawn,
|
||||
should_be_root=True,
|
||||
data=data_to_pass_down,
|
||||
reg_addr=reg_addr,
|
||||
debug_mode=debug_mode,
|
||||
)
|
||||
True,
|
||||
data_to_pass_down,
|
||||
reg_addr,
|
||||
)
|
||||
assert result == 10
|
||||
|
||||
|
||||
async def movie_theatre_question():
|
||||
'''
|
||||
A question asked in a dark theatre, in a tangent
|
||||
"""A question asked in a dark theatre, in a tangent
|
||||
(errr, I mean different) process.
|
||||
|
||||
'''
|
||||
"""
|
||||
return 'have you ever seen a portal?'
|
||||
|
||||
|
||||
@tractor_test
|
||||
async def test_movie_theatre_convo(start_method):
|
||||
'''
|
||||
The main ``tractor`` routine.
|
||||
"""The main ``tractor`` routine.
|
||||
"""
|
||||
async with tractor.open_nursery() as n:
|
||||
|
||||
'''
|
||||
async with tractor.open_nursery(debug_mode=True) as an:
|
||||
|
||||
portal = await an.start_actor(
|
||||
portal = await n.start_actor(
|
||||
'frank',
|
||||
# enable the actor to run funcs from this current module
|
||||
enable_modules=[__name__],
|
||||
|
|
@ -140,8 +118,8 @@ async def test_most_beautiful_word(
|
|||
with trio.fail_after(1):
|
||||
async with tractor.open_nursery(
|
||||
debug_mode=debug_mode,
|
||||
) as an:
|
||||
portal = await an.run_in_actor(
|
||||
) as n:
|
||||
portal = await n.run_in_actor(
|
||||
cellar_door,
|
||||
return_value=return_value,
|
||||
name='some_linguist',
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ from ._root import (
|
|||
run_daemon as run_daemon,
|
||||
open_root_actor as open_root_actor,
|
||||
)
|
||||
from .ipc import Channel as Channel
|
||||
from ._ipc import Channel as Channel
|
||||
from ._portal import Portal as Portal
|
||||
from ._runtime import Actor as Actor
|
||||
# from . import hilevel as hilevel
|
||||
|
|
|
|||
583
tractor/_addr.py
583
tractor/_addr.py
|
|
@ -1,583 +0,0 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
import os
|
||||
# import tempfile
|
||||
from uuid import uuid4
|
||||
from typing import (
|
||||
Protocol,
|
||||
ClassVar,
|
||||
# TypeVar,
|
||||
# Union,
|
||||
Type,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
from bidict import bidict
|
||||
# import trio
|
||||
from trio import (
|
||||
socket,
|
||||
SocketListener,
|
||||
open_tcp_listeners,
|
||||
)
|
||||
|
||||
from .log import get_logger
|
||||
from ._state import (
|
||||
get_rt_dir,
|
||||
current_actor,
|
||||
is_root_process,
|
||||
_def_tpt_proto,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._runtime import Actor
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# TODO, maybe breakout the netns key to a struct?
|
||||
# class NetNs(Struct)[str, int]:
|
||||
# ...
|
||||
|
||||
# TODO, can't we just use a type alias
|
||||
# for this? namely just some `tuple[str, int, str, str]`?
|
||||
#
|
||||
# -[ ] would also just be simpler to keep this as SockAddr[tuple]
|
||||
# or something, implying it's just a simple pair of values which can
|
||||
# presumably be mapped to all transports?
|
||||
# -[ ] `pydoc socket.socket.getsockname()` delivers a 4-tuple for
|
||||
# ipv6 `(hostaddr, port, flowinfo, scope_id)`.. so how should we
|
||||
# handle that?
|
||||
# -[ ] as a further alternative to this wrap()/unwrap() approach we
|
||||
# could just implement `enc/dec_hook()`s for the `Address`-types
|
||||
# and just deal with our internal objs directly and always and
|
||||
# leave it to the codec layer to figure out marshalling?
|
||||
# |_ would mean only one spot to do the `.unwrap()` (which we may
|
||||
# end up needing to call from the hook()s anyway?)
|
||||
# -[x] rename to `UnwrappedAddress[Descriptor]` ??
|
||||
# seems like the right name as per,
|
||||
# https://www.geeksforgeeks.org/introduction-to-address-descriptor/
|
||||
#
|
||||
UnwrappedAddress = (
|
||||
# tcp/udp/uds
|
||||
tuple[
|
||||
str, # host/domain(tcp), filesys-dir(uds)
|
||||
int|str, # port/path(uds)
|
||||
]
|
||||
# ?TODO? should we also include another 2 fields from
|
||||
# our `Aid` msg such that we include the runtime `Actor.uid`
|
||||
# of `.name` and `.uuid`?
|
||||
# - would ensure uniqueness across entire net?
|
||||
# - allows for easier runtime-level filtering of "actors by
|
||||
# service name"
|
||||
)
|
||||
|
||||
|
||||
# TODO, maybe rename to `SocketAddress`?
|
||||
class Address(Protocol):
|
||||
proto_key: ClassVar[str]
|
||||
unwrapped_type: ClassVar[UnwrappedAddress]
|
||||
|
||||
# TODO, i feel like an `.is_bound()` is a better thing to
|
||||
# support?
|
||||
# Lke, what use does this have besides a noop and if it's not
|
||||
# valid why aren't we erroring on creation/use?
|
||||
@property
|
||||
def is_valid(self) -> bool:
|
||||
...
|
||||
|
||||
# TODO, maybe `.netns` is a better name?
|
||||
@property
|
||||
def namespace(self) -> tuple[str, int]|None:
|
||||
'''
|
||||
The if-available, OS-specific "network namespace" key.
|
||||
|
||||
'''
|
||||
...
|
||||
|
||||
@property
|
||||
def bindspace(self) -> str:
|
||||
'''
|
||||
Deliver the socket address' "bindable space" from
|
||||
a `socket.socket.bind()` and thus from the perspective of
|
||||
specific transport protocol domain.
|
||||
|
||||
I.e. for most (layer-4) network-socket protocols this is
|
||||
normally the ipv4/6 address, for UDS this is normally
|
||||
a filesystem (sub-directory).
|
||||
|
||||
For (distributed) network protocols this is normally the routing
|
||||
layer's domain/(ip-)address, though it might also include a "network namespace"
|
||||
key different then the default.
|
||||
|
||||
For local-host-only transports this is either an explicit
|
||||
namespace (with types defined by the OS: netns, Cgroup, IPC,
|
||||
pid, etc. on linux) or failing that the sub-directory in the
|
||||
filesys in which socket/shm files are located *under*.
|
||||
|
||||
'''
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def from_addr(cls, addr: UnwrappedAddress) -> Address:
|
||||
...
|
||||
|
||||
def unwrap(self) -> UnwrappedAddress:
|
||||
'''
|
||||
Deliver the underying minimum field set in
|
||||
a primitive python data type-structure.
|
||||
'''
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_random(
|
||||
cls,
|
||||
current_actor: Actor,
|
||||
bindspace: str|None = None,
|
||||
) -> Address:
|
||||
...
|
||||
|
||||
# TODO, this should be something like a `.get_def_registar_addr()`
|
||||
# or similar since,
|
||||
# - it should be a **host singleton** (not root/tree singleton)
|
||||
# - we **only need this value** when one isn't provided to the
|
||||
# runtime at boot and we want to implicitly provide a host-wide
|
||||
# registrar.
|
||||
# - each rooted-actor-tree should likely have its own
|
||||
# micro-registry (likely the root being it), also see
|
||||
@classmethod
|
||||
def get_root(cls) -> Address:
|
||||
...
|
||||
|
||||
def __repr__(self) -> str:
|
||||
...
|
||||
|
||||
def __eq__(self, other) -> bool:
|
||||
...
|
||||
|
||||
async def open_listener(
|
||||
self,
|
||||
**kwargs,
|
||||
) -> SocketListener:
|
||||
...
|
||||
|
||||
async def close_listener(self):
|
||||
...
|
||||
|
||||
|
||||
class TCPAddress(Address):
|
||||
proto_key: str = 'tcp'
|
||||
unwrapped_type: type = tuple[str, int]
|
||||
def_bindspace: str = '127.0.0.1'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int
|
||||
):
|
||||
if (
|
||||
not isinstance(host, str)
|
||||
or
|
||||
not isinstance(port, int)
|
||||
):
|
||||
raise TypeError(
|
||||
f'Expected host {host!r} to be str and port {port!r} to be int'
|
||||
)
|
||||
|
||||
self._host: str = host
|
||||
self._port: int = port
|
||||
|
||||
@property
|
||||
def is_valid(self) -> bool:
|
||||
return self._port != 0
|
||||
|
||||
@property
|
||||
def bindspace(self) -> str:
|
||||
return self._host
|
||||
|
||||
@property
|
||||
def domain(self) -> str:
|
||||
return self._host
|
||||
|
||||
@classmethod
|
||||
def from_addr(
|
||||
cls,
|
||||
addr: tuple[str, int]
|
||||
) -> TCPAddress:
|
||||
match addr:
|
||||
case (str(), int()):
|
||||
return TCPAddress(addr[0], addr[1])
|
||||
case _:
|
||||
raise ValueError(
|
||||
f'Invalid unwrapped address for {cls}\n'
|
||||
f'{addr}\n'
|
||||
)
|
||||
|
||||
def unwrap(self) -> tuple[str, int]:
|
||||
return (
|
||||
self._host,
|
||||
self._port,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_random(
|
||||
cls,
|
||||
bindspace: str = def_bindspace,
|
||||
) -> TCPAddress:
|
||||
return TCPAddress(bindspace, 0)
|
||||
|
||||
@classmethod
|
||||
def get_root(cls) -> Address:
|
||||
return TCPAddress(
|
||||
'127.0.0.1',
|
||||
1616,
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f'{type(self).__name__}[{self.unwrap()}]'
|
||||
)
|
||||
|
||||
def __eq__(self, other) -> bool:
|
||||
if not isinstance(other, TCPAddress):
|
||||
raise TypeError(
|
||||
f'Can not compare {type(other)} with {type(self)}'
|
||||
)
|
||||
|
||||
return (
|
||||
self._host == other._host
|
||||
and
|
||||
self._port == other._port
|
||||
)
|
||||
|
||||
async def open_listener(
|
||||
self,
|
||||
**kwargs,
|
||||
) -> SocketListener:
|
||||
listeners: list[SocketListener] = await open_tcp_listeners(
|
||||
host=self._host,
|
||||
port=self._port,
|
||||
**kwargs
|
||||
)
|
||||
assert len(listeners) == 1
|
||||
listener = listeners[0]
|
||||
self._host, self._port = listener.socket.getsockname()[:2]
|
||||
return listener
|
||||
|
||||
async def close_listener(self):
|
||||
...
|
||||
|
||||
|
||||
def unwrap_sockpath(
|
||||
sockpath: Path,
|
||||
) -> tuple[Path, Path]:
|
||||
return (
|
||||
sockpath.parent,
|
||||
sockpath.name,
|
||||
)
|
||||
|
||||
|
||||
class UDSAddress(Address):
|
||||
# TODO, maybe we should use better field and value
|
||||
# -[x] really this is a `.protocol_key` not a "name" of anything.
|
||||
# -[ ] consider a 'unix' proto-key instead?
|
||||
# -[ ] need to check what other mult-transport frameworks do
|
||||
# like zmq, nng, uri-spec et al!
|
||||
proto_key: str = 'uds'
|
||||
unwrapped_type: type = tuple[str, int]
|
||||
def_bindspace: Path = get_rt_dir()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
filedir: Path|str|None,
|
||||
# TODO, i think i want `.filename` here?
|
||||
filename: str|Path,
|
||||
|
||||
# XXX, in the sense you can also pass
|
||||
# a "non-real-world-process-id" such as is handy to represent
|
||||
# our host-local default "port-like" key for the very first
|
||||
# root actor to create a registry address.
|
||||
maybe_pid: int|None = None,
|
||||
):
|
||||
fdir = self._filedir = Path(
|
||||
filedir
|
||||
or
|
||||
self.def_bindspace
|
||||
).absolute()
|
||||
fpath = self._filename = Path(filename)
|
||||
fp: Path = fdir / fpath
|
||||
assert (
|
||||
fp.is_absolute()
|
||||
and
|
||||
fp == self.sockpath
|
||||
)
|
||||
|
||||
# to track which "side" is the peer process by reading socket
|
||||
# credentials-info.
|
||||
self._pid: int = maybe_pid
|
||||
|
||||
@property
|
||||
def sockpath(self) -> Path:
|
||||
return self._filedir / self._filename
|
||||
|
||||
@property
|
||||
def is_valid(self) -> bool:
|
||||
'''
|
||||
We block socket files not allocated under the runtime subdir.
|
||||
|
||||
'''
|
||||
return self.bindspace in self.sockpath.parents
|
||||
|
||||
@property
|
||||
def bindspace(self) -> Path:
|
||||
'''
|
||||
We replicate the "ip-set-of-hosts" part of a UDS socket as
|
||||
just the sub-directory in which we allocate socket files.
|
||||
|
||||
'''
|
||||
return self._filedir or self.def_bindspace
|
||||
|
||||
@classmethod
|
||||
def from_addr(
|
||||
cls,
|
||||
addr: (
|
||||
tuple[Path|str, Path|str]|Path|str
|
||||
),
|
||||
) -> UDSAddress:
|
||||
match addr:
|
||||
case tuple()|list():
|
||||
filedir = Path(addr[0])
|
||||
filename = Path(addr[1])
|
||||
# sockpath: Path = Path(addr[0])
|
||||
# filedir, filename = unwrap_sockpath(sockpath)
|
||||
# pid: int = addr[1]
|
||||
return UDSAddress(
|
||||
filedir=filedir,
|
||||
filename=filename,
|
||||
# maybe_pid=pid,
|
||||
)
|
||||
# NOTE, in case we ever decide to just `.unwrap()`
|
||||
# to a `Path|str`?
|
||||
case str()|Path():
|
||||
sockpath: Path = Path(addr)
|
||||
return UDSAddress(*unwrap_sockpath(sockpath))
|
||||
case _:
|
||||
# import pdbp; pdbp.set_trace()
|
||||
raise TypeError(
|
||||
f'Bad unwrapped-address for {cls} !\n'
|
||||
f'{addr!r}\n'
|
||||
)
|
||||
|
||||
def unwrap(self) -> tuple[str, int]:
|
||||
# XXX NOTE, since this gets passed DIRECTLY to
|
||||
# `.ipc._uds.open_unix_socket_w_passcred()`
|
||||
return (
|
||||
str(self._filedir),
|
||||
str(self._filename),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_random(
|
||||
cls,
|
||||
bindspace: Path|None = None, # default netns
|
||||
) -> UDSAddress:
|
||||
|
||||
filedir: Path = bindspace or cls.def_bindspace
|
||||
pid: int = os.getpid()
|
||||
actor: Actor|None = current_actor(
|
||||
err_on_no_runtime=False,
|
||||
)
|
||||
if actor:
|
||||
sockname: str = '::'.join(actor.uid) + f'@{pid}'
|
||||
else:
|
||||
prefix: str = '<unknown-actor>'
|
||||
if is_root_process():
|
||||
prefix: str = 'root'
|
||||
sockname: str = f'{prefix}@{pid}'
|
||||
|
||||
sockpath: Path = Path(f'{sockname}.sock')
|
||||
return UDSAddress(
|
||||
filedir=filedir,
|
||||
filename=sockpath,
|
||||
maybe_pid=pid,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_root(cls) -> Address:
|
||||
def_uds_filename: Path = 'registry@1616.sock'
|
||||
return UDSAddress(
|
||||
filedir=None,
|
||||
filename=def_uds_filename,
|
||||
# maybe_pid=1616,
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f'{type(self).__name__}'
|
||||
f'['
|
||||
f'({self._filedir}, {self._filename})'
|
||||
f']'
|
||||
)
|
||||
|
||||
def __eq__(self, other) -> bool:
|
||||
if not isinstance(other, UDSAddress):
|
||||
raise TypeError(
|
||||
f'Can not compare {type(other)} with {type(self)}'
|
||||
)
|
||||
|
||||
return self.sockpath == other.sockpath
|
||||
|
||||
# async def open_listener(self, **kwargs) -> SocketListener:
|
||||
async def open_listener(
|
||||
self,
|
||||
**kwargs,
|
||||
) -> SocketListener:
|
||||
sock = self._sock = socket.socket(
|
||||
socket.AF_UNIX,
|
||||
socket.SOCK_STREAM
|
||||
)
|
||||
log.info(
|
||||
f'Attempting to bind UDS socket\n'
|
||||
f'>[\n'
|
||||
f'|_{self}\n'
|
||||
)
|
||||
|
||||
bindpath: Path = self.sockpath
|
||||
await sock.bind(str(bindpath))
|
||||
sock.listen(1)
|
||||
log.info(
|
||||
f'Listening on UDS socket\n'
|
||||
f'[>\n'
|
||||
f' |_{self}\n'
|
||||
)
|
||||
return SocketListener(self._sock)
|
||||
|
||||
def close_listener(self):
|
||||
self._sock.close()
|
||||
os.unlink(self.sockpath)
|
||||
|
||||
|
||||
_address_types: bidict[str, Type[Address]] = {
|
||||
'tcp': TCPAddress,
|
||||
'uds': UDSAddress
|
||||
}
|
||||
|
||||
|
||||
# TODO! really these are discovery sys default addrs ONLY useful for
|
||||
# when none is provided to a root actor on first boot.
|
||||
_default_lo_addrs: dict[
|
||||
str,
|
||||
UnwrappedAddress
|
||||
] = {
|
||||
'tcp': TCPAddress.get_root().unwrap(),
|
||||
'uds': UDSAddress.get_root().unwrap(),
|
||||
}
|
||||
|
||||
|
||||
def get_address_cls(name: str) -> Type[Address]:
|
||||
return _address_types[name]
|
||||
|
||||
|
||||
def is_wrapped_addr(addr: any) -> bool:
|
||||
return type(addr) in _address_types.values()
|
||||
|
||||
|
||||
def mk_uuid() -> str:
|
||||
'''
|
||||
Encapsulate creation of a uuid4 as `str` as used
|
||||
for creating `Actor.uid: tuple[str, str]` and/or
|
||||
`.msg.types.Aid`.
|
||||
|
||||
'''
|
||||
return str(uuid4())
|
||||
|
||||
|
||||
def wrap_address(
|
||||
addr: UnwrappedAddress
|
||||
) -> Address:
|
||||
'''
|
||||
Wrap an `UnwrappedAddress` as an `Address`-type based
|
||||
on matching builtin python data-structures which we adhoc
|
||||
use for each.
|
||||
|
||||
XXX NOTE, careful care must be placed to ensure
|
||||
`UnwrappedAddress` cases are **definitely unique** otherwise the
|
||||
wrong transport backend may be loaded and will break many
|
||||
low-level things in our runtime in a not-fun-to-debug way!
|
||||
|
||||
XD
|
||||
|
||||
'''
|
||||
if is_wrapped_addr(addr):
|
||||
return addr
|
||||
|
||||
cls: Type|None = None
|
||||
# if 'sock' in addr[0]:
|
||||
# import pdbp; pdbp.set_trace()
|
||||
match addr:
|
||||
|
||||
# classic network socket-address as tuple/list
|
||||
case (
|
||||
(str(), int())
|
||||
|
|
||||
[str(), int()]
|
||||
):
|
||||
cls = TCPAddress
|
||||
|
||||
case (
|
||||
# (str()|Path(), str()|Path()),
|
||||
# ^TODO? uhh why doesn't this work!?
|
||||
|
||||
(_, filename)
|
||||
) if type(filename) is str:
|
||||
cls = UDSAddress
|
||||
|
||||
# likely an unset UDS or TCP reg address as defaulted in
|
||||
# `_state._runtime_vars['_root_mailbox']`
|
||||
#
|
||||
# TODO? figure out when/if we even need this?
|
||||
case (
|
||||
None
|
||||
|
|
||||
[None, None]
|
||||
):
|
||||
cls: Type[Address] = get_address_cls(_def_tpt_proto)
|
||||
addr: UnwrappedAddress = cls.get_root().unwrap()
|
||||
|
||||
case _:
|
||||
# import pdbp; pdbp.set_trace()
|
||||
raise TypeError(
|
||||
f'Can not wrap unwrapped-address ??\n'
|
||||
f'type(addr): {type(addr)!r}\n'
|
||||
f'addr: {addr!r}\n'
|
||||
)
|
||||
|
||||
return cls.from_addr(addr)
|
||||
|
||||
|
||||
def default_lo_addrs(
|
||||
transports: list[str],
|
||||
) -> list[Type[Address]]:
|
||||
'''
|
||||
Return the default, host-singleton, registry address
|
||||
for an input transport key set.
|
||||
|
||||
'''
|
||||
return [
|
||||
_default_lo_addrs[transport]
|
||||
for transport in transports
|
||||
]
|
||||
|
|
@ -31,12 +31,8 @@ def parse_uid(arg):
|
|||
return str(name), str(uuid) # ensures str encoding
|
||||
|
||||
def parse_ipaddr(arg):
|
||||
try:
|
||||
return literal_eval(arg)
|
||||
|
||||
except (ValueError, SyntaxError):
|
||||
# UDS: try to interpret as a straight up str
|
||||
return arg
|
||||
host, port = literal_eval(arg)
|
||||
return (str(host), int(port))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
@ -50,8 +46,8 @@ if __name__ == "__main__":
|
|||
args = parser.parse_args()
|
||||
|
||||
subactor = Actor(
|
||||
name=args.uid[0],
|
||||
uuid=args.uid[1],
|
||||
args.uid[0],
|
||||
uid=args.uid[1],
|
||||
loglevel=args.loglevel,
|
||||
spawn_method="trio"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ from .msg import (
|
|||
pretty_struct,
|
||||
_ops as msgops,
|
||||
)
|
||||
from .ipc import (
|
||||
from ._ipc import (
|
||||
Channel,
|
||||
)
|
||||
from ._streaming import (
|
||||
|
|
@ -105,7 +105,7 @@ from ._state import (
|
|||
if TYPE_CHECKING:
|
||||
from ._portal import Portal
|
||||
from ._runtime import Actor
|
||||
from .ipc import MsgTransport
|
||||
from ._ipc import MsgTransport
|
||||
from .devx._frame_stack import (
|
||||
CallerInfo,
|
||||
)
|
||||
|
|
@ -366,7 +366,7 @@ class Context:
|
|||
# f' ---\n'
|
||||
f' |_ipc: {self.dst_maddr}\n'
|
||||
# f' dst_maddr{ds}{self.dst_maddr}\n'
|
||||
f" uid{ds}'{self.chan.aid}'\n"
|
||||
f" uid{ds}'{self.chan.uid}'\n"
|
||||
f" cid{ds}'{self.cid}'\n"
|
||||
# f' ---\n'
|
||||
f'\n'
|
||||
|
|
@ -859,10 +859,19 @@ class Context:
|
|||
@property
|
||||
def dst_maddr(self) -> str:
|
||||
chan: Channel = self.chan
|
||||
dst_addr, dst_port = chan.raddr
|
||||
trans: MsgTransport = chan.transport
|
||||
# cid: str = self.cid
|
||||
# cid_head, cid_tail = cid[:6], cid[-6:]
|
||||
return trans.maddr
|
||||
return (
|
||||
f'/ipv4/{dst_addr}'
|
||||
f'/{trans.name_key}/{dst_port}'
|
||||
# f'/{self.chan.uid[0]}'
|
||||
# f'/{self.cid}'
|
||||
|
||||
# f'/cid={cid_head}..{cid_tail}'
|
||||
# TODO: ? not use this ^ right ?
|
||||
)
|
||||
|
||||
dmaddr = dst_maddr
|
||||
|
||||
|
|
@ -945,9 +954,9 @@ class Context:
|
|||
reminfo: str = (
|
||||
# ' =>\n'
|
||||
# f'Context.cancel() => {self.chan.uid}\n'
|
||||
f'\n'
|
||||
f'c)=> {self.chan.uid}\n'
|
||||
f' |_[{self.dst_maddr}\n'
|
||||
# f'{self.chan.uid}\n'
|
||||
f' |_ @{self.dst_maddr}\n'
|
||||
f' >> {self.repr_rpc}\n'
|
||||
# f' >> {self._nsf}() -> {codec}[dict]:\n\n'
|
||||
# TODO: pull msg-type from spec re #320
|
||||
|
|
|
|||
|
|
@ -29,12 +29,7 @@ from contextlib import asynccontextmanager as acm
|
|||
|
||||
from tractor.log import get_logger
|
||||
from .trionics import gather_contexts
|
||||
from .ipc import _connect_chan, Channel
|
||||
from ._addr import (
|
||||
UnwrappedAddress,
|
||||
Address,
|
||||
wrap_address
|
||||
)
|
||||
from ._ipc import _connect_chan, Channel
|
||||
from ._portal import (
|
||||
Portal,
|
||||
open_portal,
|
||||
|
|
@ -43,7 +38,6 @@ from ._portal import (
|
|||
from ._state import (
|
||||
current_actor,
|
||||
_runtime_vars,
|
||||
_def_tpt_proto,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -55,7 +49,9 @@ log = get_logger(__name__)
|
|||
|
||||
@acm
|
||||
async def get_registry(
|
||||
addr: UnwrappedAddress|None = None,
|
||||
host: str,
|
||||
port: int,
|
||||
|
||||
) -> AsyncGenerator[
|
||||
Portal | LocalPortal | None,
|
||||
None,
|
||||
|
|
@ -73,15 +69,13 @@ async def get_registry(
|
|||
# (likely a re-entrant call from the arbiter actor)
|
||||
yield LocalPortal(
|
||||
actor,
|
||||
Channel(transport=None)
|
||||
# ^XXX, we DO NOT actually provide nor connect an
|
||||
# underlying transport since this is merely an API shim.
|
||||
Channel((host, port))
|
||||
)
|
||||
else:
|
||||
# TODO: try to look pre-existing connection from
|
||||
# `Actor._peers` and use it instead?
|
||||
async with (
|
||||
_connect_chan(addr) as chan,
|
||||
_connect_chan(host, port) as chan,
|
||||
open_portal(chan) as regstr_ptl,
|
||||
):
|
||||
yield regstr_ptl
|
||||
|
|
@ -95,10 +89,11 @@ async def get_root(
|
|||
|
||||
# TODO: rename mailbox to `_root_maddr` when we finally
|
||||
# add and impl libp2p multi-addrs?
|
||||
addr = _runtime_vars['_root_mailbox']
|
||||
host, port = _runtime_vars['_root_mailbox']
|
||||
assert host is not None
|
||||
|
||||
async with (
|
||||
_connect_chan(addr) as chan,
|
||||
_connect_chan(host, port) as chan,
|
||||
open_portal(chan, **kwargs) as portal,
|
||||
):
|
||||
yield portal
|
||||
|
|
@ -139,10 +134,10 @@ def get_peer_by_name(
|
|||
@acm
|
||||
async def query_actor(
|
||||
name: str,
|
||||
regaddr: UnwrappedAddress|None = None,
|
||||
regaddr: tuple[str, int]|None = None,
|
||||
|
||||
) -> AsyncGenerator[
|
||||
UnwrappedAddress|None,
|
||||
tuple[str, int]|None,
|
||||
None,
|
||||
]:
|
||||
'''
|
||||
|
|
@ -168,31 +163,31 @@ async def query_actor(
|
|||
return
|
||||
|
||||
reg_portal: Portal
|
||||
regaddr: Address = wrap_address(regaddr) or actor.reg_addrs[0]
|
||||
async with get_registry(regaddr) as reg_portal:
|
||||
regaddr: tuple[str, int] = regaddr or actor.reg_addrs[0]
|
||||
async with get_registry(*regaddr) as reg_portal:
|
||||
# TODO: return portals to all available actors - for now
|
||||
# just the last one that registered
|
||||
addr: UnwrappedAddress = await reg_portal.run_from_ns(
|
||||
sockaddr: tuple[str, int] = await reg_portal.run_from_ns(
|
||||
'self',
|
||||
'find_actor',
|
||||
name=name,
|
||||
)
|
||||
yield addr
|
||||
yield sockaddr
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_portal(
|
||||
addr: UnwrappedAddress,
|
||||
addr: tuple[str, int],
|
||||
name: str,
|
||||
):
|
||||
async with query_actor(
|
||||
name=name,
|
||||
regaddr=addr,
|
||||
) as addr:
|
||||
) as sockaddr:
|
||||
pass
|
||||
|
||||
if addr:
|
||||
async with _connect_chan(addr) as chan:
|
||||
if sockaddr:
|
||||
async with _connect_chan(*sockaddr) as chan:
|
||||
async with open_portal(chan) as portal:
|
||||
yield portal
|
||||
else:
|
||||
|
|
@ -202,8 +197,7 @@ async def maybe_open_portal(
|
|||
@acm
|
||||
async def find_actor(
|
||||
name: str,
|
||||
registry_addrs: list[UnwrappedAddress]|None = None,
|
||||
enable_transports: list[str] = [_def_tpt_proto],
|
||||
registry_addrs: list[tuple[str, int]]|None = None,
|
||||
|
||||
only_first: bool = True,
|
||||
raise_on_none: bool = False,
|
||||
|
|
@ -230,15 +224,15 @@ async def find_actor(
|
|||
# XXX NOTE: make sure to dynamically read the value on
|
||||
# every call since something may change it globally (eg.
|
||||
# like in our discovery test suite)!
|
||||
from ._addr import default_lo_addrs
|
||||
from . import _root
|
||||
registry_addrs = (
|
||||
_runtime_vars['_registry_addrs']
|
||||
or
|
||||
default_lo_addrs(enable_transports)
|
||||
_root._default_lo_addrs
|
||||
)
|
||||
|
||||
maybe_portals: list[
|
||||
AsyncContextManager[UnwrappedAddress]
|
||||
AsyncContextManager[tuple[str, int]]
|
||||
] = list(
|
||||
maybe_open_portal(
|
||||
addr=addr,
|
||||
|
|
@ -280,7 +274,7 @@ async def find_actor(
|
|||
@acm
|
||||
async def wait_for_actor(
|
||||
name: str,
|
||||
registry_addr: UnwrappedAddress | None = None,
|
||||
registry_addr: tuple[str, int] | None = None,
|
||||
|
||||
) -> AsyncGenerator[Portal, None]:
|
||||
'''
|
||||
|
|
@ -297,7 +291,7 @@ async def wait_for_actor(
|
|||
yield peer_portal
|
||||
return
|
||||
|
||||
regaddr: UnwrappedAddress = (
|
||||
regaddr: tuple[str, int] = (
|
||||
registry_addr
|
||||
or
|
||||
actor.reg_addrs[0]
|
||||
|
|
@ -305,8 +299,8 @@ async def wait_for_actor(
|
|||
# TODO: use `.trionics.gather_contexts()` like
|
||||
# above in `find_actor()` as well?
|
||||
reg_portal: Portal
|
||||
async with get_registry(regaddr) as reg_portal:
|
||||
addrs = await reg_portal.run_from_ns(
|
||||
async with get_registry(*regaddr) as reg_portal:
|
||||
sockaddrs = await reg_portal.run_from_ns(
|
||||
'self',
|
||||
'wait_for_actor',
|
||||
name=name,
|
||||
|
|
@ -314,8 +308,8 @@ async def wait_for_actor(
|
|||
|
||||
# get latest registered addr by default?
|
||||
# TODO: offer multi-portal yields in multi-homed case?
|
||||
addr: UnwrappedAddress = addrs[-1]
|
||||
sockaddr: tuple[str, int] = sockaddrs[-1]
|
||||
|
||||
async with _connect_chan(addr) as chan:
|
||||
async with _connect_chan(*sockaddr) as chan:
|
||||
async with open_portal(chan) as portal:
|
||||
yield portal
|
||||
|
|
|
|||
|
|
@ -37,7 +37,6 @@ from .log import (
|
|||
from . import _state
|
||||
from .devx import _debug
|
||||
from .to_asyncio import run_as_asyncio_guest
|
||||
from ._addr import UnwrappedAddress
|
||||
from ._runtime import (
|
||||
async_main,
|
||||
Actor,
|
||||
|
|
@ -53,10 +52,10 @@ log = get_logger(__name__)
|
|||
def _mp_main(
|
||||
|
||||
actor: Actor,
|
||||
accept_addrs: list[UnwrappedAddress],
|
||||
accept_addrs: list[tuple[str, int]],
|
||||
forkserver_info: tuple[Any, Any, Any, Any, Any],
|
||||
start_method: SpawnMethodKey,
|
||||
parent_addr: UnwrappedAddress | None = None,
|
||||
parent_addr: tuple[str, int] | None = None,
|
||||
infect_asyncio: bool = False,
|
||||
|
||||
) -> None:
|
||||
|
|
@ -207,7 +206,7 @@ def nest_from_op(
|
|||
def _trio_main(
|
||||
actor: Actor,
|
||||
*,
|
||||
parent_addr: UnwrappedAddress|None = None,
|
||||
parent_addr: tuple[str, int] | None = None,
|
||||
infect_asyncio: bool = False,
|
||||
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import builtins
|
|||
import importlib
|
||||
from pprint import pformat
|
||||
from pdb import bdb
|
||||
import sys
|
||||
from types import (
|
||||
TracebackType,
|
||||
)
|
||||
|
|
@ -64,29 +65,15 @@ if TYPE_CHECKING:
|
|||
from ._context import Context
|
||||
from .log import StackLevelAdapter
|
||||
from ._stream import MsgStream
|
||||
from .ipc import Channel
|
||||
from ._ipc import Channel
|
||||
|
||||
log = get_logger('tractor')
|
||||
|
||||
_this_mod = importlib.import_module(__name__)
|
||||
|
||||
|
||||
class RuntimeFailure(RuntimeError):
|
||||
'''
|
||||
General `Actor`-runtime failure due to,
|
||||
|
||||
- a bad runtime-env,
|
||||
- falied spawning (bad input to process),
|
||||
- API usage.
|
||||
|
||||
'''
|
||||
|
||||
|
||||
class ActorFailure(RuntimeFailure):
|
||||
'''
|
||||
`Actor` failed to boot before/after spawn
|
||||
|
||||
'''
|
||||
class ActorFailure(Exception):
|
||||
"General actor failure"
|
||||
|
||||
|
||||
class InternalError(RuntimeError):
|
||||
|
|
@ -139,12 +126,6 @@ class TrioTaskExited(Exception):
|
|||
'''
|
||||
|
||||
|
||||
class DebugRequestError(RuntimeError):
|
||||
'''
|
||||
Failed to request stdio lock from root actor!
|
||||
|
||||
'''
|
||||
|
||||
# NOTE: more or less should be close to these:
|
||||
# 'boxed_type',
|
||||
# 'src_type',
|
||||
|
|
@ -210,8 +191,6 @@ def get_err_type(type_name: str) -> BaseException|None:
|
|||
):
|
||||
return type_ref
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def pack_from_raise(
|
||||
local_err: (
|
||||
|
|
@ -542,6 +521,7 @@ class RemoteActorError(Exception):
|
|||
if val:
|
||||
_repr += f'{key}={val_str}{end_char}'
|
||||
|
||||
|
||||
return _repr
|
||||
|
||||
def reprol(self) -> str:
|
||||
|
|
@ -620,9 +600,56 @@ class RemoteActorError(Exception):
|
|||
the type name is already implicitly shown by python).
|
||||
|
||||
'''
|
||||
header: str = ''
|
||||
body: str = ''
|
||||
message: str = ''
|
||||
|
||||
# XXX when the currently raised exception is this instance,
|
||||
# we do not ever use the "type header" style repr.
|
||||
is_being_raised: bool = False
|
||||
if (
|
||||
(exc := sys.exception())
|
||||
and
|
||||
exc is self
|
||||
):
|
||||
is_being_raised: bool = True
|
||||
|
||||
with_type_header: bool = (
|
||||
with_type_header
|
||||
and
|
||||
not is_being_raised
|
||||
)
|
||||
|
||||
# <RemoteActorError( .. )> style
|
||||
if with_type_header:
|
||||
header: str = f'<{type(self).__name__}('
|
||||
|
||||
if message := self._message:
|
||||
|
||||
# split off the first line so, if needed, it isn't
|
||||
# indented the same like the "boxed content" which
|
||||
# since there is no `.tb_str` is just the `.message`.
|
||||
lines: list[str] = message.splitlines()
|
||||
first: str = lines[0]
|
||||
message: str = message.removeprefix(first)
|
||||
|
||||
# with a type-style header we,
|
||||
# - have no special message "first line" extraction/handling
|
||||
# - place the message a space in from the header:
|
||||
# `MsgTypeError( <message> ..`
|
||||
# ^-here
|
||||
# - indent the `.message` inside the type body.
|
||||
if with_type_header:
|
||||
first = f' {first} )>'
|
||||
|
||||
message: str = textwrap.indent(
|
||||
message,
|
||||
prefix=' '*2,
|
||||
)
|
||||
message: str = first + message
|
||||
|
||||
# IFF there is an embedded traceback-str we always
|
||||
# draw the ascii-box around it.
|
||||
body: str = ''
|
||||
if tb_str := self.tb_str:
|
||||
fields: str = self._mk_fields_str(
|
||||
_body_fields
|
||||
|
|
@ -643,15 +670,21 @@ class RemoteActorError(Exception):
|
|||
boxer_header=self.relay_uid,
|
||||
)
|
||||
|
||||
# !TODO, it'd be nice to import these top level without
|
||||
# cycles!
|
||||
from tractor.devx.pformat import (
|
||||
pformat_exc,
|
||||
)
|
||||
return pformat_exc(
|
||||
exc=self,
|
||||
with_type_header=with_type_header,
|
||||
body=body,
|
||||
tail = ''
|
||||
if (
|
||||
with_type_header
|
||||
and not message
|
||||
):
|
||||
tail: str = '>'
|
||||
|
||||
return (
|
||||
header
|
||||
+
|
||||
message
|
||||
+
|
||||
f'{body}'
|
||||
+
|
||||
tail
|
||||
)
|
||||
|
||||
__repr__ = pformat
|
||||
|
|
@ -929,7 +962,7 @@ class StreamOverrun(
|
|||
'''
|
||||
|
||||
|
||||
class TransportClosed(Exception):
|
||||
class TransportClosed(trio.BrokenResourceError):
|
||||
'''
|
||||
IPC transport (protocol) connection was closed or broke and
|
||||
indicates that the wrapping communication `Channel` can no longer
|
||||
|
|
@ -940,21 +973,16 @@ class TransportClosed(Exception):
|
|||
self,
|
||||
message: str,
|
||||
loglevel: str = 'transport',
|
||||
src_exc: Exception|None = None,
|
||||
cause: BaseException|None = None,
|
||||
raise_on_report: bool = False,
|
||||
|
||||
) -> None:
|
||||
self.message: str = message
|
||||
self._loglevel: str = loglevel
|
||||
self._loglevel = loglevel
|
||||
super().__init__(message)
|
||||
|
||||
self.src_exc = src_exc
|
||||
if (
|
||||
src_exc is not None
|
||||
and
|
||||
not self.__cause__
|
||||
):
|
||||
self.__cause__ = src_exc
|
||||
if cause is not None:
|
||||
self.__cause__ = cause
|
||||
|
||||
# flag to toggle whether the msg loop should raise
|
||||
# the exc in its `TransportClosed` handler block.
|
||||
|
|
@ -981,36 +1009,13 @@ class TransportClosed(Exception):
|
|||
f' {cause}\n' # exc repr
|
||||
)
|
||||
|
||||
getattr(
|
||||
log,
|
||||
self._loglevel
|
||||
)(message)
|
||||
getattr(log, self._loglevel)(message)
|
||||
|
||||
# some errors we want to blow up from
|
||||
# inside the RPC msg loop
|
||||
if self._raise_on_report:
|
||||
raise self from cause
|
||||
|
||||
def pformat(self) -> str:
|
||||
from tractor.devx.pformat import (
|
||||
pformat_exc,
|
||||
)
|
||||
src_err: Exception|None = self.src_exc or '<unknown>'
|
||||
src_msg: tuple[str] = src_err.args
|
||||
src_exc_repr: str = (
|
||||
f'{type(src_err).__name__}[ {src_msg} ]'
|
||||
)
|
||||
return pformat_exc(
|
||||
exc=self,
|
||||
# message=self.message, # implicit!
|
||||
body=(
|
||||
f' |_src_exc: {src_exc_repr}\n'
|
||||
),
|
||||
)
|
||||
|
||||
# delegate to `str`-ified pformat
|
||||
__repr__ = pformat
|
||||
|
||||
|
||||
class NoResult(RuntimeError):
|
||||
"No final result is expected for this actor"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,820 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Inter-process comms abstractions
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
AsyncIterator,
|
||||
)
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
contextmanager as cm,
|
||||
)
|
||||
import platform
|
||||
from pprint import pformat
|
||||
import struct
|
||||
import typing
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
runtime_checkable,
|
||||
Protocol,
|
||||
Type,
|
||||
TypeVar,
|
||||
)
|
||||
|
||||
import msgspec
|
||||
from tricycle import BufferedReceiveStream
|
||||
import trio
|
||||
|
||||
from tractor.log import get_logger
|
||||
from tractor._exceptions import (
|
||||
MsgTypeError,
|
||||
pack_from_raise,
|
||||
TransportClosed,
|
||||
_mk_send_mte,
|
||||
_mk_recv_mte,
|
||||
)
|
||||
from tractor.msg import (
|
||||
_ctxvar_MsgCodec,
|
||||
# _codec, XXX see `self._codec` sanity/debug checks
|
||||
MsgCodec,
|
||||
types as msgtypes,
|
||||
pretty_struct,
|
||||
)
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_is_windows = platform.system() == 'Windows'
|
||||
|
||||
|
||||
def get_stream_addrs(
|
||||
stream: trio.SocketStream
|
||||
) -> tuple[
|
||||
tuple[str, int], # local
|
||||
tuple[str, int], # remote
|
||||
]:
|
||||
'''
|
||||
Return the `trio` streaming transport prot's socket-addrs for
|
||||
both the local and remote sides as a pair.
|
||||
|
||||
'''
|
||||
# rn, should both be IP sockets
|
||||
lsockname = stream.socket.getsockname()
|
||||
rsockname = stream.socket.getpeername()
|
||||
return (
|
||||
tuple(lsockname[:2]),
|
||||
tuple(rsockname[:2]),
|
||||
)
|
||||
|
||||
|
||||
# from tractor.msg.types import MsgType
|
||||
# ?TODO? this should be our `Union[*msgtypes.__spec__]` alias now right..?
|
||||
# => BLEH, except can't bc prots must inherit typevar or param-spec
|
||||
# vars..
|
||||
MsgType = TypeVar('MsgType')
|
||||
|
||||
|
||||
# TODO: break up this mod into a subpkg so we can start adding new
|
||||
# backends and move this type stuff into a dedicated file.. Bo
|
||||
#
|
||||
@runtime_checkable
|
||||
class MsgTransport(Protocol[MsgType]):
|
||||
#
|
||||
# ^-TODO-^ consider using a generic def and indexing with our
|
||||
# eventual msg definition/types?
|
||||
# - https://docs.python.org/3/library/typing.html#typing.Protocol
|
||||
|
||||
stream: trio.SocketStream
|
||||
drained: list[MsgType]
|
||||
|
||||
def __init__(self, stream: trio.SocketStream) -> None:
|
||||
...
|
||||
|
||||
# XXX: should this instead be called `.sendall()`?
|
||||
async def send(self, msg: MsgType) -> None:
|
||||
...
|
||||
|
||||
async def recv(self) -> MsgType:
|
||||
...
|
||||
|
||||
def __aiter__(self) -> MsgType:
|
||||
...
|
||||
|
||||
def connected(self) -> bool:
|
||||
...
|
||||
|
||||
# defining this sync otherwise it causes a mypy error because it
|
||||
# can't figure out it's a generator i guess?..?
|
||||
def drain(self) -> AsyncIterator[dict]:
|
||||
...
|
||||
|
||||
@property
|
||||
def laddr(self) -> tuple[str, int]:
|
||||
...
|
||||
|
||||
@property
|
||||
def raddr(self) -> tuple[str, int]:
|
||||
...
|
||||
|
||||
|
||||
# TODO: typing oddity.. not sure why we have to inherit here, but it
|
||||
# seems to be an issue with `get_msg_transport()` returning
|
||||
# a `Type[Protocol]`; probably should make a `mypy` issue?
|
||||
class MsgpackTCPStream(MsgTransport):
|
||||
'''
|
||||
A ``trio.SocketStream`` delivering ``msgpack`` formatted data
|
||||
using the ``msgspec`` codec lib.
|
||||
|
||||
'''
|
||||
layer_key: int = 4
|
||||
name_key: str = 'tcp'
|
||||
|
||||
# TODO: better naming for this?
|
||||
# -[ ] check how libp2p does naming for such things?
|
||||
codec_key: str = 'msgpack'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
stream: trio.SocketStream,
|
||||
prefix_size: int = 4,
|
||||
|
||||
# XXX optionally provided codec pair for `msgspec`:
|
||||
# https://jcristharif.com/msgspec/extending.html#mapping-to-from-native-types
|
||||
#
|
||||
# TODO: define this as a `Codec` struct which can be
|
||||
# overriden dynamically by the application/runtime?
|
||||
codec: tuple[
|
||||
Callable[[Any], Any]|None, # coder
|
||||
Callable[[type, Any], Any]|None, # decoder
|
||||
]|None = None,
|
||||
|
||||
) -> None:
|
||||
|
||||
self.stream = stream
|
||||
assert self.stream.socket
|
||||
|
||||
# should both be IP sockets
|
||||
self._laddr, self._raddr = get_stream_addrs(stream)
|
||||
|
||||
# create read loop instance
|
||||
self._aiter_pkts = self._iter_packets()
|
||||
self._send_lock = trio.StrictFIFOLock()
|
||||
|
||||
# public i guess?
|
||||
self.drained: list[dict] = []
|
||||
|
||||
self.recv_stream = BufferedReceiveStream(
|
||||
transport_stream=stream
|
||||
)
|
||||
self.prefix_size = prefix_size
|
||||
|
||||
# allow for custom IPC msg interchange format
|
||||
# dynamic override Bo
|
||||
self._task = trio.lowlevel.current_task()
|
||||
|
||||
# XXX for ctxvar debug only!
|
||||
# self._codec: MsgCodec = (
|
||||
# codec
|
||||
# or
|
||||
# _codec._ctxvar_MsgCodec.get()
|
||||
# )
|
||||
|
||||
async def _iter_packets(self) -> AsyncGenerator[dict, None]:
|
||||
'''
|
||||
Yield `bytes`-blob decoded packets from the underlying TCP
|
||||
stream using the current task's `MsgCodec`.
|
||||
|
||||
This is a streaming routine implemented as an async generator
|
||||
func (which was the original design, but could be changed?)
|
||||
and is allocated by a `.__call__()` inside `.__init__()` where
|
||||
it is assigned to the `._aiter_pkts` attr.
|
||||
|
||||
'''
|
||||
decodes_failed: int = 0
|
||||
|
||||
while True:
|
||||
try:
|
||||
header: bytes = await self.recv_stream.receive_exactly(4)
|
||||
except (
|
||||
ValueError,
|
||||
ConnectionResetError,
|
||||
|
||||
# not sure entirely why we need this but without it we
|
||||
# seem to be getting racy failures here on
|
||||
# arbiter/registry name subs..
|
||||
trio.BrokenResourceError,
|
||||
|
||||
) as trans_err:
|
||||
|
||||
loglevel = 'transport'
|
||||
match trans_err:
|
||||
# case (
|
||||
# ConnectionResetError()
|
||||
# ):
|
||||
# loglevel = 'transport'
|
||||
|
||||
# peer actor (graceful??) TCP EOF but `tricycle`
|
||||
# seems to raise a 0-bytes-read?
|
||||
case ValueError() if (
|
||||
'unclean EOF' in trans_err.args[0]
|
||||
):
|
||||
pass
|
||||
|
||||
# peer actor (task) prolly shutdown quickly due
|
||||
# to cancellation
|
||||
case trio.BrokenResourceError() if (
|
||||
'Connection reset by peer' in trans_err.args[0]
|
||||
):
|
||||
pass
|
||||
|
||||
# unless the disconnect condition falls under "a
|
||||
# normal operation breakage" we usualy console warn
|
||||
# about it.
|
||||
case _:
|
||||
loglevel: str = 'warning'
|
||||
|
||||
|
||||
raise TransportClosed(
|
||||
message=(
|
||||
f'IPC transport already closed by peer\n'
|
||||
f'x]> {type(trans_err)}\n'
|
||||
f' |_{self}\n'
|
||||
),
|
||||
loglevel=loglevel,
|
||||
) from trans_err
|
||||
|
||||
# XXX definitely can happen if transport is closed
|
||||
# manually by another `trio.lowlevel.Task` in the
|
||||
# same actor; we use this in some simulated fault
|
||||
# testing for ex, but generally should never happen
|
||||
# under normal operation!
|
||||
#
|
||||
# NOTE: as such we always re-raise this error from the
|
||||
# RPC msg loop!
|
||||
except trio.ClosedResourceError as closure_err:
|
||||
raise TransportClosed(
|
||||
message=(
|
||||
f'IPC transport already manually closed locally?\n'
|
||||
f'x]> {type(closure_err)} \n'
|
||||
f' |_{self}\n'
|
||||
),
|
||||
loglevel='error',
|
||||
raise_on_report=(
|
||||
closure_err.args[0] == 'another task closed this fd'
|
||||
or
|
||||
closure_err.args[0] in ['another task closed this fd']
|
||||
),
|
||||
) from closure_err
|
||||
|
||||
# graceful TCP EOF disconnect
|
||||
if header == b'':
|
||||
raise TransportClosed(
|
||||
message=(
|
||||
f'IPC transport already gracefully closed\n'
|
||||
f']>\n'
|
||||
f' |_{self}\n'
|
||||
),
|
||||
loglevel='transport',
|
||||
# cause=??? # handy or no?
|
||||
)
|
||||
|
||||
size: int
|
||||
size, = struct.unpack("<I", header)
|
||||
|
||||
log.transport(f'received header {size}') # type: ignore
|
||||
msg_bytes: bytes = await self.recv_stream.receive_exactly(size)
|
||||
|
||||
log.transport(f"received {msg_bytes}") # type: ignore
|
||||
try:
|
||||
# NOTE: lookup the `trio.Task.context`'s var for
|
||||
# the current `MsgCodec`.
|
||||
codec: MsgCodec = _ctxvar_MsgCodec.get()
|
||||
|
||||
# XXX for ctxvar debug only!
|
||||
# if self._codec.pld_spec != codec.pld_spec:
|
||||
# assert (
|
||||
# task := trio.lowlevel.current_task()
|
||||
# ) is not self._task
|
||||
# self._task = task
|
||||
# self._codec = codec
|
||||
# log.runtime(
|
||||
# f'Using new codec in {self}.recv()\n'
|
||||
# f'codec: {self._codec}\n\n'
|
||||
# f'msg_bytes: {msg_bytes}\n'
|
||||
# )
|
||||
yield codec.decode(msg_bytes)
|
||||
|
||||
# XXX NOTE: since the below error derives from
|
||||
# `DecodeError` we need to catch is specially
|
||||
# and always raise such that spec violations
|
||||
# are never allowed to be caught silently!
|
||||
except msgspec.ValidationError as verr:
|
||||
msgtyperr: MsgTypeError = _mk_recv_mte(
|
||||
msg=msg_bytes,
|
||||
codec=codec,
|
||||
src_validation_error=verr,
|
||||
)
|
||||
# XXX deliver up to `Channel.recv()` where
|
||||
# a re-raise and `Error`-pack can inject the far
|
||||
# end actor `.uid`.
|
||||
yield msgtyperr
|
||||
|
||||
except (
|
||||
msgspec.DecodeError,
|
||||
UnicodeDecodeError,
|
||||
):
|
||||
if decodes_failed < 4:
|
||||
# ignore decoding errors for now and assume they have to
|
||||
# do with a channel drop - hope that receiving from the
|
||||
# channel will raise an expected error and bubble up.
|
||||
try:
|
||||
msg_str: str|bytes = msg_bytes.decode()
|
||||
except UnicodeDecodeError:
|
||||
msg_str = msg_bytes
|
||||
|
||||
log.exception(
|
||||
'Failed to decode msg?\n'
|
||||
f'{codec}\n\n'
|
||||
'Rxed bytes from wire:\n\n'
|
||||
f'{msg_str!r}\n'
|
||||
)
|
||||
decodes_failed += 1
|
||||
else:
|
||||
raise
|
||||
|
||||
async def send(
|
||||
self,
|
||||
msg: msgtypes.MsgType,
|
||||
|
||||
strict_types: bool = True,
|
||||
hide_tb: bool = False,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Send a msgpack encoded py-object-blob-as-msg over TCP.
|
||||
|
||||
If `strict_types == True` then a `MsgTypeError` will be raised on any
|
||||
invalid msg type
|
||||
|
||||
'''
|
||||
__tracebackhide__: bool = hide_tb
|
||||
|
||||
# XXX see `trio._sync.AsyncContextManagerMixin` for details
|
||||
# on the `.acquire()`/`.release()` sequencing..
|
||||
async with self._send_lock:
|
||||
|
||||
# NOTE: lookup the `trio.Task.context`'s var for
|
||||
# the current `MsgCodec`.
|
||||
codec: MsgCodec = _ctxvar_MsgCodec.get()
|
||||
|
||||
# XXX for ctxvar debug only!
|
||||
# if self._codec.pld_spec != codec.pld_spec:
|
||||
# self._codec = codec
|
||||
# log.runtime(
|
||||
# f'Using new codec in {self}.send()\n'
|
||||
# f'codec: {self._codec}\n\n'
|
||||
# f'msg: {msg}\n'
|
||||
# )
|
||||
|
||||
if type(msg) not in msgtypes.__msg_types__:
|
||||
if strict_types:
|
||||
raise _mk_send_mte(
|
||||
msg,
|
||||
codec=codec,
|
||||
)
|
||||
else:
|
||||
log.warning(
|
||||
'Sending non-`Msg`-spec msg?\n\n'
|
||||
f'{msg}\n'
|
||||
)
|
||||
|
||||
try:
|
||||
bytes_data: bytes = codec.encode(msg)
|
||||
except TypeError as _err:
|
||||
typerr = _err
|
||||
msgtyperr: MsgTypeError = _mk_send_mte(
|
||||
msg,
|
||||
codec=codec,
|
||||
message=(
|
||||
f'IPC-msg-spec violation in\n\n'
|
||||
f'{pretty_struct.Struct.pformat(msg)}'
|
||||
),
|
||||
src_type_error=typerr,
|
||||
)
|
||||
raise msgtyperr from typerr
|
||||
|
||||
# supposedly the fastest says,
|
||||
# https://stackoverflow.com/a/54027962
|
||||
size: bytes = struct.pack("<I", len(bytes_data))
|
||||
return await self.stream.send_all(size + bytes_data)
|
||||
|
||||
# ?TODO? does it help ever to dynamically show this
|
||||
# frame?
|
||||
# try:
|
||||
# <the-above_code>
|
||||
# except BaseException as _err:
|
||||
# err = _err
|
||||
# if not isinstance(err, MsgTypeError):
|
||||
# __tracebackhide__: bool = False
|
||||
# raise
|
||||
|
||||
@property
|
||||
def laddr(self) -> tuple[str, int]:
|
||||
return self._laddr
|
||||
|
||||
@property
|
||||
def raddr(self) -> tuple[str, int]:
|
||||
return self._raddr
|
||||
|
||||
async def recv(self) -> Any:
|
||||
return await self._aiter_pkts.asend(None)
|
||||
|
||||
async def drain(self) -> AsyncIterator[dict]:
|
||||
'''
|
||||
Drain the stream's remaining messages sent from
|
||||
the far end until the connection is closed by
|
||||
the peer.
|
||||
|
||||
'''
|
||||
try:
|
||||
async for msg in self._iter_packets():
|
||||
self.drained.append(msg)
|
||||
except TransportClosed:
|
||||
for msg in self.drained:
|
||||
yield msg
|
||||
|
||||
def __aiter__(self):
|
||||
return self._aiter_pkts
|
||||
|
||||
def connected(self) -> bool:
|
||||
return self.stream.socket.fileno() != -1
|
||||
|
||||
|
||||
def get_msg_transport(
|
||||
|
||||
key: tuple[str, str],
|
||||
|
||||
) -> Type[MsgTransport]:
|
||||
|
||||
return {
|
||||
('msgpack', 'tcp'): MsgpackTCPStream,
|
||||
}[key]
|
||||
|
||||
|
||||
class Channel:
|
||||
'''
|
||||
An inter-process channel for communication between (remote) actors.
|
||||
|
||||
Wraps a ``MsgStream``: transport + encoding IPC connection.
|
||||
|
||||
Currently we only support ``trio.SocketStream`` for transport
|
||||
(aka TCP) and the ``msgpack`` interchange format via the ``msgspec``
|
||||
codec libary.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
|
||||
self,
|
||||
destaddr: tuple[str, int]|None,
|
||||
|
||||
msg_transport_type_key: tuple[str, str] = ('msgpack', 'tcp'),
|
||||
|
||||
# TODO: optional reconnection support?
|
||||
# auto_reconnect: bool = False,
|
||||
# on_reconnect: typing.Callable[..., typing.Awaitable] = None,
|
||||
|
||||
) -> None:
|
||||
|
||||
# self._recon_seq = on_reconnect
|
||||
# self._autorecon = auto_reconnect
|
||||
|
||||
self._destaddr = destaddr
|
||||
self._transport_key = msg_transport_type_key
|
||||
|
||||
# Either created in ``.connect()`` or passed in by
|
||||
# user in ``.from_stream()``.
|
||||
self._stream: trio.SocketStream|None = None
|
||||
self._transport: MsgTransport|None = None
|
||||
|
||||
# set after handshake - always uid of far end
|
||||
self.uid: tuple[str, str]|None = None
|
||||
|
||||
self._aiter_msgs = self._iter_msgs()
|
||||
self._exc: Exception|None = None # set if far end actor errors
|
||||
self._closed: bool = False
|
||||
|
||||
# flag set by ``Portal.cancel_actor()`` indicating remote
|
||||
# (possibly peer) cancellation of the far end actor
|
||||
# runtime.
|
||||
self._cancel_called: bool = False
|
||||
|
||||
@property
|
||||
def msgstream(self) -> MsgTransport:
|
||||
log.info(
|
||||
'`Channel.msgstream` is an old name, use `._transport`'
|
||||
)
|
||||
return self._transport
|
||||
|
||||
@property
|
||||
def transport(self) -> MsgTransport:
|
||||
return self._transport
|
||||
|
||||
@classmethod
|
||||
def from_stream(
|
||||
cls,
|
||||
stream: trio.SocketStream,
|
||||
**kwargs,
|
||||
|
||||
) -> Channel:
|
||||
|
||||
src, dst = get_stream_addrs(stream)
|
||||
chan = Channel(
|
||||
destaddr=dst,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# set immediately here from provided instance
|
||||
chan._stream: trio.SocketStream = stream
|
||||
chan.set_msg_transport(stream)
|
||||
return chan
|
||||
|
||||
def set_msg_transport(
|
||||
self,
|
||||
stream: trio.SocketStream,
|
||||
type_key: tuple[str, str]|None = None,
|
||||
|
||||
# XXX optionally provided codec pair for `msgspec`:
|
||||
# https://jcristharif.com/msgspec/extending.html#mapping-to-from-native-types
|
||||
codec: MsgCodec|None = None,
|
||||
|
||||
) -> MsgTransport:
|
||||
type_key = (
|
||||
type_key
|
||||
or
|
||||
self._transport_key
|
||||
)
|
||||
# get transport type, then
|
||||
self._transport = get_msg_transport(
|
||||
type_key
|
||||
# instantiate an instance of the msg-transport
|
||||
)(
|
||||
stream,
|
||||
codec=codec,
|
||||
)
|
||||
return self._transport
|
||||
|
||||
@cm
|
||||
def apply_codec(
|
||||
self,
|
||||
codec: MsgCodec,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Temporarily override the underlying IPC msg codec for
|
||||
dynamic enforcement of messaging schema.
|
||||
|
||||
'''
|
||||
orig: MsgCodec = self._transport.codec
|
||||
try:
|
||||
self._transport.codec = codec
|
||||
yield
|
||||
finally:
|
||||
self._transport.codec = orig
|
||||
|
||||
# TODO: do a .src/.dst: str for maddrs?
|
||||
def __repr__(self) -> str:
|
||||
if not self._transport:
|
||||
return '<Channel with inactive transport?>'
|
||||
|
||||
return repr(
|
||||
self._transport.stream.socket._sock
|
||||
).replace( # type: ignore
|
||||
"socket.socket",
|
||||
"Channel",
|
||||
)
|
||||
|
||||
@property
|
||||
def laddr(self) -> tuple[str, int]|None:
|
||||
return self._transport.laddr if self._transport else None
|
||||
|
||||
@property
|
||||
def raddr(self) -> tuple[str, int]|None:
|
||||
return self._transport.raddr if self._transport else None
|
||||
|
||||
async def connect(
|
||||
self,
|
||||
destaddr: tuple[Any, ...] | None = None,
|
||||
**kwargs
|
||||
|
||||
) -> MsgTransport:
|
||||
|
||||
if self.connected():
|
||||
raise RuntimeError("channel is already connected?")
|
||||
|
||||
destaddr = destaddr or self._destaddr
|
||||
assert isinstance(destaddr, tuple)
|
||||
|
||||
stream = await trio.open_tcp_stream(
|
||||
*destaddr,
|
||||
**kwargs
|
||||
)
|
||||
transport = self.set_msg_transport(stream)
|
||||
|
||||
log.transport(
|
||||
f'Opened channel[{type(transport)}]: {self.laddr} -> {self.raddr}'
|
||||
)
|
||||
return transport
|
||||
|
||||
# TODO: something like,
|
||||
# `pdbp.hideframe_on(errors=[MsgTypeError])`
|
||||
# instead of the `try/except` hack we have rn..
|
||||
# seems like a pretty useful thing to have in general
|
||||
# along with being able to filter certain stack frame(s / sets)
|
||||
# possibly based on the current log-level?
|
||||
async def send(
|
||||
self,
|
||||
payload: Any,
|
||||
|
||||
hide_tb: bool = False,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Send a coded msg-blob over the transport.
|
||||
|
||||
'''
|
||||
__tracebackhide__: bool = hide_tb
|
||||
try:
|
||||
log.transport(
|
||||
'=> send IPC msg:\n\n'
|
||||
f'{pformat(payload)}\n'
|
||||
)
|
||||
# assert self._transport # but why typing?
|
||||
await self._transport.send(
|
||||
payload,
|
||||
hide_tb=hide_tb,
|
||||
)
|
||||
except BaseException as _err:
|
||||
err = _err # bind for introspection
|
||||
if not isinstance(_err, MsgTypeError):
|
||||
# assert err
|
||||
__tracebackhide__: bool = False
|
||||
else:
|
||||
assert err.cid
|
||||
|
||||
raise
|
||||
|
||||
async def recv(self) -> Any:
|
||||
assert self._transport
|
||||
return await self._transport.recv()
|
||||
|
||||
# TODO: auto-reconnect features like 0mq/nanomsg?
|
||||
# -[ ] implement it manually with nods to SC prot
|
||||
# possibly on multiple transport backends?
|
||||
# -> seems like that might be re-inventing scalability
|
||||
# prots tho no?
|
||||
# try:
|
||||
# return await self._transport.recv()
|
||||
# except trio.BrokenResourceError:
|
||||
# if self._autorecon:
|
||||
# await self._reconnect()
|
||||
# return await self.recv()
|
||||
# raise
|
||||
|
||||
async def aclose(self) -> None:
|
||||
|
||||
log.transport(
|
||||
f'Closing channel to {self.uid} '
|
||||
f'{self.laddr} -> {self.raddr}'
|
||||
)
|
||||
assert self._transport
|
||||
await self._transport.stream.aclose()
|
||||
self._closed = True
|
||||
|
||||
async def __aenter__(self):
|
||||
await self.connect()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
await self.aclose(*args)
|
||||
|
||||
def __aiter__(self):
|
||||
return self._aiter_msgs
|
||||
|
||||
# ?TODO? run any reconnection sequence?
|
||||
# -[ ] prolly should be impl-ed as deco-API?
|
||||
#
|
||||
# async def _reconnect(self) -> None:
|
||||
# """Handle connection failures by polling until a reconnect can be
|
||||
# established.
|
||||
# """
|
||||
# down = False
|
||||
# while True:
|
||||
# try:
|
||||
# with trio.move_on_after(3) as cancel_scope:
|
||||
# await self.connect()
|
||||
# cancelled = cancel_scope.cancelled_caught
|
||||
# if cancelled:
|
||||
# log.transport(
|
||||
# "Reconnect timed out after 3 seconds, retrying...")
|
||||
# continue
|
||||
# else:
|
||||
# log.transport("Stream connection re-established!")
|
||||
|
||||
# # on_recon = self._recon_seq
|
||||
# # if on_recon:
|
||||
# # await on_recon(self)
|
||||
|
||||
# break
|
||||
# except (OSError, ConnectionRefusedError):
|
||||
# if not down:
|
||||
# down = True
|
||||
# log.transport(
|
||||
# f"Connection to {self.raddr} went down, waiting"
|
||||
# " for re-establishment")
|
||||
# await trio.sleep(1)
|
||||
|
||||
async def _iter_msgs(
|
||||
self
|
||||
) -> AsyncGenerator[Any, None]:
|
||||
'''
|
||||
Yield `MsgType` IPC msgs decoded and deliverd from
|
||||
an underlying `MsgTransport` protocol.
|
||||
|
||||
This is a streaming routine alo implemented as an async-gen
|
||||
func (same a `MsgTransport._iter_pkts()`) gets allocated by
|
||||
a `.__call__()` inside `.__init__()` where it is assigned to
|
||||
the `._aiter_msgs` attr.
|
||||
|
||||
'''
|
||||
assert self._transport
|
||||
while True:
|
||||
try:
|
||||
async for msg in self._transport:
|
||||
match msg:
|
||||
# NOTE: if transport/interchange delivers
|
||||
# a type error, we pack it with the far
|
||||
# end peer `Actor.uid` and relay the
|
||||
# `Error`-msg upward to the `._rpc` stack
|
||||
# for normal RAE handling.
|
||||
case MsgTypeError():
|
||||
yield pack_from_raise(
|
||||
local_err=msg,
|
||||
cid=msg.cid,
|
||||
|
||||
# XXX we pack it here bc lower
|
||||
# layers have no notion of an
|
||||
# actor-id ;)
|
||||
src_uid=self.uid,
|
||||
)
|
||||
case _:
|
||||
yield msg
|
||||
|
||||
except trio.BrokenResourceError:
|
||||
|
||||
# if not self._autorecon:
|
||||
raise
|
||||
|
||||
await self.aclose()
|
||||
|
||||
# if self._autorecon: # attempt reconnect
|
||||
# await self._reconnect()
|
||||
# continue
|
||||
|
||||
def connected(self) -> bool:
|
||||
return self._transport.connected() if self._transport else False
|
||||
|
||||
|
||||
@acm
|
||||
async def _connect_chan(
|
||||
host: str,
|
||||
port: int
|
||||
|
||||
) -> typing.AsyncGenerator[Channel, None]:
|
||||
'''
|
||||
Create and connect a channel with disconnect on context manager
|
||||
teardown.
|
||||
|
||||
'''
|
||||
chan = Channel((host, port))
|
||||
await chan.connect()
|
||||
yield chan
|
||||
with trio.CancelScope(shield=True):
|
||||
await chan.aclose()
|
||||
|
|
@ -43,7 +43,7 @@ from .trionics import maybe_open_nursery
|
|||
from ._state import (
|
||||
current_actor,
|
||||
)
|
||||
from .ipc import Channel
|
||||
from ._ipc import Channel
|
||||
from .log import get_logger
|
||||
from .msg import (
|
||||
# Error,
|
||||
|
|
@ -107,10 +107,6 @@ class Portal:
|
|||
# point.
|
||||
self._expect_result_ctx: Context|None = None
|
||||
self._streams: set[MsgStream] = set()
|
||||
|
||||
# TODO, this should be PRIVATE (and never used publicly)! since it's just
|
||||
# a cached ref to the local runtime instead of calling
|
||||
# `current_actor()` everywhere.. XD
|
||||
self.actor: Actor = current_actor()
|
||||
|
||||
@property
|
||||
|
|
@ -175,7 +171,7 @@ class Portal:
|
|||
# not expecting a "main" result
|
||||
if self._expect_result_ctx is None:
|
||||
log.warning(
|
||||
f"Portal for {self.channel.aid} not expecting a final"
|
||||
f"Portal for {self.channel.uid} not expecting a final"
|
||||
" result?\nresult() should only be called if subactor"
|
||||
" was spawned with `ActorNursery.run_in_actor()`")
|
||||
return NoResult
|
||||
|
|
@ -222,7 +218,7 @@ class Portal:
|
|||
# IPC calls
|
||||
if self._streams:
|
||||
log.cancel(
|
||||
f"Cancelling all streams with {self.channel.aid}")
|
||||
f"Cancelling all streams with {self.channel.uid}")
|
||||
for stream in self._streams.copy():
|
||||
try:
|
||||
await stream.aclose()
|
||||
|
|
@ -267,7 +263,7 @@ class Portal:
|
|||
return False
|
||||
|
||||
reminfo: str = (
|
||||
f'c)=> {self.channel.aid}\n'
|
||||
f'c)=> {self.channel.uid}\n'
|
||||
f' |_{chan}\n'
|
||||
)
|
||||
log.cancel(
|
||||
|
|
@ -310,7 +306,7 @@ class Portal:
|
|||
):
|
||||
log.debug(
|
||||
'IPC chan for actor already closed or broken?\n\n'
|
||||
f'{self.channel.aid}\n'
|
||||
f'{self.channel.uid}\n'
|
||||
f' |_{self.channel}\n'
|
||||
)
|
||||
return False
|
||||
|
|
@ -508,12 +504,8 @@ class LocalPortal:
|
|||
return it's result.
|
||||
|
||||
'''
|
||||
obj = (
|
||||
self.actor
|
||||
if ns == 'self'
|
||||
else importlib.import_module(ns)
|
||||
)
|
||||
func: Callable = getattr(obj, func_name)
|
||||
obj = self.actor if ns == 'self' else importlib.import_module(ns)
|
||||
func = getattr(obj, func_name)
|
||||
return await func(**kwargs)
|
||||
|
||||
|
||||
|
|
@ -551,10 +543,8 @@ async def open_portal(
|
|||
await channel.connect()
|
||||
was_connected = True
|
||||
|
||||
if channel.aid is None:
|
||||
await channel._do_handshake(
|
||||
aid=actor.aid,
|
||||
)
|
||||
if channel.uid is None:
|
||||
await actor._do_handshake(channel)
|
||||
|
||||
msg_loop_cs: trio.CancelScope|None = None
|
||||
if start_msg_loop:
|
||||
|
|
|
|||
252
tractor/_root.py
252
tractor/_root.py
|
|
@ -18,9 +18,7 @@
|
|||
Root actor runtime ignition(s).
|
||||
|
||||
'''
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from functools import partial
|
||||
import importlib
|
||||
import inspect
|
||||
|
|
@ -28,10 +26,7 @@ import logging
|
|||
import os
|
||||
import signal
|
||||
import sys
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
)
|
||||
from typing import Callable
|
||||
import warnings
|
||||
|
||||
|
||||
|
|
@ -48,109 +43,33 @@ from .devx import _debug
|
|||
from . import _spawn
|
||||
from . import _state
|
||||
from . import log
|
||||
from .ipc import (
|
||||
_connect_chan,
|
||||
)
|
||||
from ._addr import (
|
||||
Address,
|
||||
UnwrappedAddress,
|
||||
default_lo_addrs,
|
||||
mk_uuid,
|
||||
wrap_address,
|
||||
)
|
||||
from ._exceptions import (
|
||||
RuntimeFailure,
|
||||
is_multi_cancelled,
|
||||
)
|
||||
from ._ipc import _connect_chan
|
||||
from ._exceptions import is_multi_cancelled
|
||||
|
||||
|
||||
# set at startup and after forks
|
||||
_default_host: str = '127.0.0.1'
|
||||
_default_port: int = 1616
|
||||
|
||||
# default registry always on localhost
|
||||
_default_lo_addrs: list[tuple[str, int]] = [(
|
||||
_default_host,
|
||||
_default_port,
|
||||
)]
|
||||
|
||||
|
||||
logger = log.get_logger('tractor')
|
||||
|
||||
|
||||
# TODO: stick this in a `@acm` defined in `devx._debug`?
|
||||
# -[ ] also maybe consider making this a `wrapt`-deco to
|
||||
# save an indent level?
|
||||
#
|
||||
@acm
|
||||
async def maybe_block_bp(
|
||||
debug_mode: bool,
|
||||
maybe_enable_greenback: bool,
|
||||
) -> bool:
|
||||
# Override the global debugger hook to make it play nice with
|
||||
# ``trio``, see much discussion in:
|
||||
# https://github.com/python-trio/trio/issues/1155#issuecomment-742964018
|
||||
builtin_bp_handler: Callable = sys.breakpointhook
|
||||
orig_bp_path: str|None = os.environ.get(
|
||||
'PYTHONBREAKPOINT',
|
||||
None,
|
||||
)
|
||||
bp_blocked: bool
|
||||
if (
|
||||
debug_mode
|
||||
and maybe_enable_greenback
|
||||
and (
|
||||
maybe_mod := await _debug.maybe_init_greenback(
|
||||
raise_not_found=False,
|
||||
)
|
||||
)
|
||||
):
|
||||
logger.info(
|
||||
f'Found `greenback` installed @ {maybe_mod}\n'
|
||||
'Enabling `tractor.pause_from_sync()` support!\n'
|
||||
)
|
||||
os.environ['PYTHONBREAKPOINT'] = (
|
||||
'tractor.devx._debug._sync_pause_from_builtin'
|
||||
)
|
||||
_state._runtime_vars['use_greenback'] = True
|
||||
bp_blocked = False
|
||||
|
||||
else:
|
||||
# TODO: disable `breakpoint()` by default (without
|
||||
# `greenback`) since it will break any multi-actor
|
||||
# usage by a clobbered TTY's stdstreams!
|
||||
def block_bps(*args, **kwargs):
|
||||
raise RuntimeError(
|
||||
'Trying to use `breakpoint()` eh?\n\n'
|
||||
'Welp, `tractor` blocks `breakpoint()` built-in calls by default!\n'
|
||||
'If you need to use it please install `greenback` and set '
|
||||
'`debug_mode=True` when opening the runtime '
|
||||
'(either via `.open_nursery()` or `open_root_actor()`)\n'
|
||||
)
|
||||
|
||||
sys.breakpointhook = block_bps
|
||||
# lol ok,
|
||||
# https://docs.python.org/3/library/sys.html#sys.breakpointhook
|
||||
os.environ['PYTHONBREAKPOINT'] = "0"
|
||||
bp_blocked = True
|
||||
|
||||
try:
|
||||
yield bp_blocked
|
||||
finally:
|
||||
# restore any prior built-in `breakpoint()` hook state
|
||||
if builtin_bp_handler is not None:
|
||||
sys.breakpointhook = builtin_bp_handler
|
||||
|
||||
if orig_bp_path is not None:
|
||||
os.environ['PYTHONBREAKPOINT'] = orig_bp_path
|
||||
|
||||
else:
|
||||
# clear env back to having no entry
|
||||
os.environ.pop('PYTHONBREAKPOINT', None)
|
||||
|
||||
|
||||
|
||||
@acm
|
||||
async def open_root_actor(
|
||||
|
||||
*,
|
||||
# defaults are above
|
||||
registry_addrs: list[UnwrappedAddress]|None = None,
|
||||
registry_addrs: list[tuple[str, int]]|None = None,
|
||||
|
||||
# defaults are above
|
||||
arbiter_addr: tuple[UnwrappedAddress]|None = None,
|
||||
|
||||
enable_transports: list[
|
||||
_state.TransportProtocolKey,
|
||||
] = [_state._def_tpt_proto],
|
||||
arbiter_addr: tuple[str, int]|None = None,
|
||||
|
||||
name: str|None = 'root',
|
||||
|
||||
|
|
@ -192,30 +111,55 @@ async def open_root_actor(
|
|||
Runtime init entry point for ``tractor``.
|
||||
|
||||
'''
|
||||
# XXX NEVER allow nested actor-trees!
|
||||
if already_actor := _state.current_actor(err_on_no_runtime=False):
|
||||
rtvs: dict[str, Any] = _state._runtime_vars
|
||||
root_mailbox: list[str, int] = rtvs['_root_mailbox']
|
||||
registry_addrs: list[list[str, int]] = rtvs['_registry_addrs']
|
||||
raise RuntimeFailure(
|
||||
f'A current actor already exists !?\n'
|
||||
f'({already_actor}\n'
|
||||
f'\n'
|
||||
f'You can NOT open a second root actor from within '
|
||||
f'an existing tree and the current root of this '
|
||||
f'already exists !!\n'
|
||||
f'\n'
|
||||
f'_root_mailbox: {root_mailbox!r}\n'
|
||||
f'_registry_addrs: {registry_addrs!r}\n'
|
||||
)
|
||||
|
||||
async with maybe_block_bp(
|
||||
debug_mode=debug_mode,
|
||||
maybe_enable_greenback=maybe_enable_greenback,
|
||||
):
|
||||
_debug.hide_runtime_frames()
|
||||
__tracebackhide__: bool = hide_tb
|
||||
|
||||
# TODO: stick this in a `@cm` defined in `devx._debug`?
|
||||
#
|
||||
# Override the global debugger hook to make it play nice with
|
||||
# ``trio``, see much discussion in:
|
||||
# https://github.com/python-trio/trio/issues/1155#issuecomment-742964018
|
||||
builtin_bp_handler: Callable = sys.breakpointhook
|
||||
orig_bp_path: str|None = os.environ.get(
|
||||
'PYTHONBREAKPOINT',
|
||||
None,
|
||||
)
|
||||
if (
|
||||
debug_mode
|
||||
and maybe_enable_greenback
|
||||
and (
|
||||
maybe_mod := await _debug.maybe_init_greenback(
|
||||
raise_not_found=False,
|
||||
)
|
||||
)
|
||||
):
|
||||
logger.info(
|
||||
f'Found `greenback` installed @ {maybe_mod}\n'
|
||||
'Enabling `tractor.pause_from_sync()` support!\n'
|
||||
)
|
||||
os.environ['PYTHONBREAKPOINT'] = (
|
||||
'tractor.devx._debug._sync_pause_from_builtin'
|
||||
)
|
||||
_state._runtime_vars['use_greenback'] = True
|
||||
|
||||
else:
|
||||
# TODO: disable `breakpoint()` by default (without
|
||||
# `greenback`) since it will break any multi-actor
|
||||
# usage by a clobbered TTY's stdstreams!
|
||||
def block_bps(*args, **kwargs):
|
||||
raise RuntimeError(
|
||||
'Trying to use `breakpoint()` eh?\n\n'
|
||||
'Welp, `tractor` blocks `breakpoint()` built-in calls by default!\n'
|
||||
'If you need to use it please install `greenback` and set '
|
||||
'`debug_mode=True` when opening the runtime '
|
||||
'(either via `.open_nursery()` or `open_root_actor()`)\n'
|
||||
)
|
||||
|
||||
sys.breakpointhook = block_bps
|
||||
# lol ok,
|
||||
# https://docs.python.org/3/library/sys.html#sys.breakpointhook
|
||||
os.environ['PYTHONBREAKPOINT'] = "0"
|
||||
|
||||
# attempt to retreive ``trio``'s sigint handler and stash it
|
||||
# on our debugger lock state.
|
||||
_debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
|
||||
|
|
@ -242,7 +186,6 @@ async def open_root_actor(
|
|||
if start_method is not None:
|
||||
_spawn.try_set_start_method(start_method)
|
||||
|
||||
# TODO! remove this ASAP!
|
||||
if arbiter_addr is not None:
|
||||
warnings.warn(
|
||||
'`arbiter_addr` is now deprecated\n'
|
||||
|
|
@ -252,11 +195,11 @@ async def open_root_actor(
|
|||
)
|
||||
registry_addrs = [arbiter_addr]
|
||||
|
||||
if not registry_addrs:
|
||||
registry_addrs: list[UnwrappedAddress] = default_lo_addrs(
|
||||
enable_transports
|
||||
registry_addrs: list[tuple[str, int]] = (
|
||||
registry_addrs
|
||||
or
|
||||
_default_lo_addrs
|
||||
)
|
||||
|
||||
assert registry_addrs
|
||||
|
||||
loglevel = (
|
||||
|
|
@ -305,10 +248,10 @@ async def open_root_actor(
|
|||
enable_stack_on_sig()
|
||||
|
||||
# closed into below ping task-func
|
||||
ponged_addrs: list[UnwrappedAddress] = []
|
||||
ponged_addrs: list[tuple[str, int]] = []
|
||||
|
||||
async def ping_tpt_socket(
|
||||
addr: UnwrappedAddress,
|
||||
addr: tuple[str, int],
|
||||
timeout: float = 1,
|
||||
) -> None:
|
||||
'''
|
||||
|
|
@ -328,7 +271,7 @@ async def open_root_actor(
|
|||
# be better to eventually have a "discovery" protocol
|
||||
# with basic handshake instead?
|
||||
with trio.move_on_after(timeout):
|
||||
async with _connect_chan(addr):
|
||||
async with _connect_chan(*addr):
|
||||
ponged_addrs.append(addr)
|
||||
|
||||
except OSError:
|
||||
|
|
@ -341,10 +284,10 @@ async def open_root_actor(
|
|||
for addr in registry_addrs:
|
||||
tn.start_soon(
|
||||
ping_tpt_socket,
|
||||
addr,
|
||||
tuple(addr), # TODO: just drop this requirement?
|
||||
)
|
||||
|
||||
trans_bind_addrs: list[UnwrappedAddress] = []
|
||||
trans_bind_addrs: list[tuple[str, int]] = []
|
||||
|
||||
# Create a new local root-actor instance which IS NOT THE
|
||||
# REGISTRAR
|
||||
|
|
@ -362,18 +305,15 @@ async def open_root_actor(
|
|||
|
||||
actor = Actor(
|
||||
name=name or 'anonymous',
|
||||
uuid=mk_uuid(),
|
||||
registry_addrs=ponged_addrs,
|
||||
loglevel=loglevel,
|
||||
enable_modules=enable_modules,
|
||||
)
|
||||
# DO NOT use the registry_addrs as the transport server
|
||||
# addrs for this new non-registar, root-actor.
|
||||
for addr in ponged_addrs:
|
||||
waddr: Address = wrap_address(addr)
|
||||
trans_bind_addrs.append(
|
||||
waddr.get_random(bindspace=waddr.bindspace)
|
||||
)
|
||||
for host, port in ponged_addrs:
|
||||
# NOTE: zero triggers dynamic OS port allocation
|
||||
trans_bind_addrs.append((host, 0))
|
||||
|
||||
# Start this local actor as the "registrar", aka a regular
|
||||
# actor who manages the local registry of "mailboxes" of
|
||||
|
|
@ -382,7 +322,7 @@ async def open_root_actor(
|
|||
|
||||
# NOTE that if the current actor IS THE REGISTAR, the
|
||||
# following init steps are taken:
|
||||
# - the tranport layer server is bound to each addr
|
||||
# - the tranport layer server is bound to each (host, port)
|
||||
# pair defined in provided registry_addrs, or the default.
|
||||
trans_bind_addrs = registry_addrs
|
||||
|
||||
|
|
@ -396,8 +336,7 @@ async def open_root_actor(
|
|||
# https://github.com/goodboy/tractor/issues/296
|
||||
|
||||
actor = Arbiter(
|
||||
name=name or 'registrar',
|
||||
uuid=mk_uuid(),
|
||||
name or 'registrar',
|
||||
registry_addrs=registry_addrs,
|
||||
loglevel=loglevel,
|
||||
enable_modules=enable_modules,
|
||||
|
|
@ -475,11 +414,7 @@ async def open_root_actor(
|
|||
err,
|
||||
)
|
||||
):
|
||||
logger.exception(
|
||||
'Root actor crashed\n'
|
||||
f'>x)\n'
|
||||
f' |_{actor}\n'
|
||||
)
|
||||
logger.exception('Root actor crashed\n')
|
||||
|
||||
# ALWAYS re-raise any error bubbled up from the
|
||||
# runtime!
|
||||
|
|
@ -496,19 +431,30 @@ async def open_root_actor(
|
|||
# tempn.start_soon(an.exited.wait)
|
||||
|
||||
logger.info(
|
||||
f'Closing down root actor\n'
|
||||
f'>)\n'
|
||||
f'|_{actor}\n'
|
||||
'Closing down root actor'
|
||||
)
|
||||
await actor.cancel(None) # self cancel
|
||||
finally:
|
||||
_state._current_actor = None
|
||||
_state._last_actor_terminated = actor
|
||||
logger.runtime(
|
||||
f'Root actor terminated\n'
|
||||
f')>\n'
|
||||
f' |_{actor}\n'
|
||||
)
|
||||
|
||||
# restore built-in `breakpoint()` hook state
|
||||
if (
|
||||
debug_mode
|
||||
and
|
||||
maybe_enable_greenback
|
||||
):
|
||||
if builtin_bp_handler is not None:
|
||||
sys.breakpointhook = builtin_bp_handler
|
||||
|
||||
if orig_bp_path is not None:
|
||||
os.environ['PYTHONBREAKPOINT'] = orig_bp_path
|
||||
|
||||
else:
|
||||
# clear env back to having no entry
|
||||
os.environ.pop('PYTHONBREAKPOINT', None)
|
||||
|
||||
logger.runtime("Root actor terminated")
|
||||
|
||||
|
||||
def run_daemon(
|
||||
|
|
@ -516,7 +462,7 @@ def run_daemon(
|
|||
|
||||
# runtime kwargs
|
||||
name: str | None = 'root',
|
||||
registry_addrs: list[UnwrappedAddress]|None = None,
|
||||
registry_addrs: list[tuple[str, int]] = _default_lo_addrs,
|
||||
|
||||
start_method: str | None = None,
|
||||
debug_mode: bool = False,
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ from trio import (
|
|||
TaskStatus,
|
||||
)
|
||||
|
||||
from .ipc import Channel
|
||||
from ._ipc import Channel
|
||||
from ._context import (
|
||||
Context,
|
||||
)
|
||||
|
|
@ -1156,7 +1156,7 @@ async def process_messages(
|
|||
trio.Event(),
|
||||
)
|
||||
|
||||
# XXX RUNTIME-SCOPED! remote (likely internal) error
|
||||
# runtime-scoped remote (internal) error
|
||||
# (^- bc no `Error.cid` -^)
|
||||
#
|
||||
# NOTE: this is the non-rpc error case, that
|
||||
|
|
@ -1219,10 +1219,8 @@ async def process_messages(
|
|||
# -[ ] figure out how this will break with other transports?
|
||||
tc.report_n_maybe_raise(
|
||||
message=(
|
||||
f'peer IPC channel closed abruptly?\n'
|
||||
f'\n'
|
||||
f'<=x[\n'
|
||||
f' {chan}\n'
|
||||
f'peer IPC channel closed abruptly?\n\n'
|
||||
f'<=x {chan}\n'
|
||||
f' |_{chan.raddr}\n\n'
|
||||
)
|
||||
+
|
||||
|
|
|
|||
|
|
@ -52,7 +52,6 @@ import sys
|
|||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Type,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
import uuid
|
||||
|
|
@ -74,14 +73,7 @@ from tractor.msg import (
|
|||
pretty_struct,
|
||||
types as msgtypes,
|
||||
)
|
||||
from .ipc import Channel
|
||||
from ._addr import (
|
||||
UnwrappedAddress,
|
||||
Address,
|
||||
default_lo_addrs,
|
||||
get_address_cls,
|
||||
wrap_address,
|
||||
)
|
||||
from ._ipc import Channel
|
||||
from ._context import (
|
||||
mk_context,
|
||||
Context,
|
||||
|
|
@ -183,15 +175,15 @@ class Actor:
|
|||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
uuid: str,
|
||||
*,
|
||||
enable_modules: list[str] = [],
|
||||
uid: str|None = None,
|
||||
loglevel: str|None = None,
|
||||
registry_addrs: list[UnwrappedAddress]|None = None,
|
||||
registry_addrs: list[tuple[str, int]]|None = None,
|
||||
spawn_method: str|None = None,
|
||||
|
||||
# TODO: remove!
|
||||
arbiter_addr: UnwrappedAddress|None = None,
|
||||
arbiter_addr: tuple[str, int]|None = None,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
|
|
@ -199,14 +191,12 @@ class Actor:
|
|||
phase (aka before a new process is executed).
|
||||
|
||||
'''
|
||||
self._aid = msgtypes.Aid(
|
||||
name=name,
|
||||
uuid=uuid,
|
||||
pid=os.getpid(),
|
||||
self.name = name
|
||||
self.uid = (
|
||||
name,
|
||||
uid or str(uuid.uuid4())
|
||||
)
|
||||
self._task: trio.Task|None = None
|
||||
|
||||
# state
|
||||
self._cancel_complete = trio.Event()
|
||||
self._cancel_called_by_remote: tuple[str, tuple]|None = None
|
||||
self._cancel_called: bool = False
|
||||
|
|
@ -233,7 +223,7 @@ class Actor:
|
|||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
registry_addrs: list[UnwrappedAddress] = [arbiter_addr]
|
||||
registry_addrs: list[tuple[str, int]] = [arbiter_addr]
|
||||
|
||||
# marked by the process spawning backend at startup
|
||||
# will be None for the parent most process started manually
|
||||
|
|
@ -267,7 +257,6 @@ class Actor:
|
|||
] = {}
|
||||
|
||||
self._listeners: list[trio.abc.Listener] = []
|
||||
self._listen_addrs: list[Address] = []
|
||||
self._parent_chan: Channel|None = None
|
||||
self._forkserver_info: tuple|None = None
|
||||
|
||||
|
|
@ -280,97 +269,13 @@ class Actor:
|
|||
|
||||
# when provided, init the registry addresses property from
|
||||
# input via the validator.
|
||||
self._reg_addrs: list[UnwrappedAddress] = []
|
||||
self._reg_addrs: list[tuple[str, int]] = []
|
||||
if registry_addrs:
|
||||
self.reg_addrs: list[UnwrappedAddress] = registry_addrs
|
||||
self.reg_addrs: list[tuple[str, int]] = registry_addrs
|
||||
_state._runtime_vars['_registry_addrs'] = registry_addrs
|
||||
|
||||
@property
|
||||
def aid(self) -> msgtypes.Aid:
|
||||
'''
|
||||
This process-singleton-actor's "unique actor ID" in struct form.
|
||||
|
||||
See the `tractor.msg.Aid` struct for details.
|
||||
|
||||
'''
|
||||
return self._aid
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._aid.name
|
||||
|
||||
@property
|
||||
def uid(self) -> tuple[str, str]:
|
||||
'''
|
||||
This process-singleton's "unique (cross-host) ID".
|
||||
|
||||
Delivered from the `.Aid.name/.uuid` fields as a `tuple` pair
|
||||
and should be multi-host unique despite a large distributed
|
||||
process plane.
|
||||
|
||||
'''
|
||||
msg: str = (
|
||||
f'`{type(self).__name__}.uid` is now deprecated.\n'
|
||||
'Use the new `.aid: tractor.msg.Aid` (struct) instead '
|
||||
'which also provides additional named (optional) fields '
|
||||
'beyond just the `.name` and `.uuid`.'
|
||||
)
|
||||
warnings.warn(
|
||||
msg,
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return (
|
||||
self._aid.name,
|
||||
self._aid.uuid,
|
||||
)
|
||||
|
||||
@property
|
||||
def pid(self) -> int:
|
||||
return self._aid.pid
|
||||
|
||||
def pformat(self) -> str:
|
||||
ds: str = '='
|
||||
parent_uid: tuple|None = None
|
||||
if rent_chan := self._parent_chan:
|
||||
parent_uid = rent_chan.uid
|
||||
peers: list[tuple] = list(self._peer_connected)
|
||||
listen_addrs: str = pformat(self._listen_addrs)
|
||||
fmtstr: str = (
|
||||
f' |_id: {self.aid!r}\n'
|
||||
# f" aid{ds}{self.aid!r}\n"
|
||||
f" parent{ds}{parent_uid}\n"
|
||||
f'\n'
|
||||
f' |_ipc: {len(peers)!r} connected peers\n'
|
||||
f" peers{ds}{peers!r}\n"
|
||||
f" _listen_addrs{ds}'{listen_addrs}'\n"
|
||||
f" _listeners{ds}'{self._listeners}'\n"
|
||||
f'\n'
|
||||
f' |_rpc: {len(self._rpc_tasks)} tasks\n'
|
||||
f" ctxs{ds}{len(self._contexts)}\n"
|
||||
f'\n'
|
||||
f' |_runtime: ._task{ds}{self._task!r}\n'
|
||||
f' _spawn_method{ds}{self._spawn_method}\n'
|
||||
f' _actoruid2nursery{ds}{self._actoruid2nursery}\n'
|
||||
f' _forkserver_info{ds}{self._forkserver_info}\n'
|
||||
f'\n'
|
||||
f' |_state: "TODO: .repr_state()"\n'
|
||||
f' _cancel_complete{ds}{self._cancel_complete}\n'
|
||||
f' _cancel_called_by_remote{ds}{self._cancel_called_by_remote}\n'
|
||||
f' _cancel_called{ds}{self._cancel_called}\n'
|
||||
)
|
||||
return (
|
||||
'<Actor(\n'
|
||||
+
|
||||
fmtstr
|
||||
+
|
||||
')>\n'
|
||||
)
|
||||
|
||||
__repr__ = pformat
|
||||
|
||||
@property
|
||||
def reg_addrs(self) -> list[UnwrappedAddress]:
|
||||
def reg_addrs(self) -> list[tuple[str, int]]:
|
||||
'''
|
||||
List of (socket) addresses for all known (and contactable)
|
||||
registry actors.
|
||||
|
|
@ -381,7 +286,7 @@ class Actor:
|
|||
@reg_addrs.setter
|
||||
def reg_addrs(
|
||||
self,
|
||||
addrs: list[UnwrappedAddress],
|
||||
addrs: list[tuple[str, int]],
|
||||
) -> None:
|
||||
if not addrs:
|
||||
log.warning(
|
||||
|
|
@ -390,6 +295,15 @@ class Actor:
|
|||
)
|
||||
return
|
||||
|
||||
# always sanity check the input list since it's critical
|
||||
# that addrs are correct for discovery sys operation.
|
||||
for addr in addrs:
|
||||
if not isinstance(addr, tuple):
|
||||
raise ValueError(
|
||||
'Expected `Actor.reg_addrs: list[tuple[str, int]]`\n'
|
||||
f'Got {addrs}'
|
||||
)
|
||||
|
||||
self._reg_addrs = addrs
|
||||
|
||||
async def wait_for_peer(
|
||||
|
|
@ -507,23 +421,14 @@ class Actor:
|
|||
|
||||
# send/receive initial handshake response
|
||||
try:
|
||||
peer_aid: msgtypes.Aid = await chan._do_handshake(
|
||||
aid=self.aid,
|
||||
)
|
||||
uid: tuple|None = await self._do_handshake(chan)
|
||||
except (
|
||||
TransportClosed,
|
||||
# ^XXX NOTE, the above wraps `trio` exc types raised
|
||||
# during various `SocketStream.send/receive_xx()` calls
|
||||
# under different fault conditions such as,
|
||||
#
|
||||
# we need this for ``msgspec`` for some reason?
|
||||
# for now, it's been put in the stream backend.
|
||||
# trio.BrokenResourceError,
|
||||
# trio.ClosedResourceError,
|
||||
#
|
||||
# Inside our `.ipc._transport` layer we absorb and
|
||||
# re-raise our own `TransportClosed` exc such that this
|
||||
# higher level runtime code can only worry one
|
||||
# "kinda-error" that we expect to tolerate during
|
||||
# discovery-sys related pings, queires, DoS etc.
|
||||
|
||||
TransportClosed,
|
||||
):
|
||||
# XXX: This may propagate up from `Channel._aiter_recv()`
|
||||
# and `MsgpackStream._inter_packets()` on a read from the
|
||||
|
|
@ -538,12 +443,6 @@ class Actor:
|
|||
)
|
||||
return
|
||||
|
||||
uid: tuple[str, str] = (
|
||||
peer_aid.name,
|
||||
peer_aid.uuid,
|
||||
)
|
||||
# TODO, can we make this downstream peer tracking use the
|
||||
# `peer_aid` instead?
|
||||
familiar: str = 'new-peer'
|
||||
if _pre_chan := self._peers.get(uid):
|
||||
familiar: str = 'pre-existing-peer'
|
||||
|
|
@ -1125,12 +1024,11 @@ class Actor:
|
|||
|
||||
async def _from_parent(
|
||||
self,
|
||||
parent_addr: UnwrappedAddress|None,
|
||||
parent_addr: tuple[str, int]|None,
|
||||
|
||||
) -> tuple[
|
||||
Channel,
|
||||
list[UnwrappedAddress]|None,
|
||||
list[str]|None, # preferred tpts
|
||||
list[tuple[str, int]]|None,
|
||||
]:
|
||||
'''
|
||||
Bootstrap this local actor's runtime config from its parent by
|
||||
|
|
@ -1142,25 +1040,23 @@ class Actor:
|
|||
# Connect back to the parent actor and conduct initial
|
||||
# handshake. From this point on if we error, we
|
||||
# attempt to ship the exception back to the parent.
|
||||
chan = await Channel.from_addr(
|
||||
addr=wrap_address(parent_addr)
|
||||
chan = Channel(
|
||||
destaddr=parent_addr,
|
||||
)
|
||||
assert isinstance(chan, Channel)
|
||||
await chan.connect()
|
||||
|
||||
# TODO: move this into a `Channel.handshake()`?
|
||||
# Initial handshake: swap names.
|
||||
await chan._do_handshake(aid=self.aid)
|
||||
await self._do_handshake(chan)
|
||||
|
||||
accept_addrs: list[UnwrappedAddress]|None = None
|
||||
accept_addrs: list[tuple[str, int]]|None = None
|
||||
|
||||
if self._spawn_method == "trio":
|
||||
|
||||
# Receive post-spawn runtime state from our parent.
|
||||
spawnspec: msgtypes.SpawnSpec = await chan.recv()
|
||||
match spawnspec:
|
||||
case MsgTypeError():
|
||||
raise spawnspec
|
||||
case msgtypes.SpawnSpec():
|
||||
self._spawn_spec = spawnspec
|
||||
|
||||
log.runtime(
|
||||
'Received runtime spec from parent:\n\n'
|
||||
|
||||
|
|
@ -1170,29 +1066,7 @@ class Actor:
|
|||
# if "trace"/"util" mode is enabled?
|
||||
f'{pretty_struct.pformat(spawnspec)}\n'
|
||||
)
|
||||
|
||||
case _:
|
||||
raise InternalError(
|
||||
f'Received invalid non-`SpawnSpec` payload !?\n'
|
||||
f'{spawnspec}\n'
|
||||
)
|
||||
|
||||
# ^^TODO XXX!! when the `SpawnSpec` fails to decode
|
||||
# the above will raise a `MsgTypeError` which if we
|
||||
# do NOT ALSO RAISE it will tried to be pprinted in
|
||||
# the log.runtime() below..
|
||||
#
|
||||
# SO we gotta look at how other `chan.recv()` calls
|
||||
# are wrapped and do the same for this spec receive!
|
||||
# -[ ] see `._rpc` likely has the answer?
|
||||
#
|
||||
# XXX NOTE, can't be called here in subactor
|
||||
# bc we haven't yet received the
|
||||
# `SpawnSpec._runtime_vars: dict` which would
|
||||
# declare whether `debug_mode` is set!
|
||||
# breakpoint()
|
||||
# import pdbp; pdbp.set_trace()
|
||||
accept_addrs: list[UnwrappedAddress] = spawnspec.bind_addrs
|
||||
accept_addrs: list[tuple[str, int]] = spawnspec.bind_addrs
|
||||
|
||||
# TODO: another `Struct` for rtvs..
|
||||
rvs: dict[str, Any] = spawnspec._runtime_vars
|
||||
|
|
@ -1284,19 +1158,11 @@ class Actor:
|
|||
return (
|
||||
chan,
|
||||
accept_addrs,
|
||||
None,
|
||||
# ^TODO, preferred tpts list from rent!
|
||||
# -[ ] need to extend the `SpawnSpec` tho!
|
||||
)
|
||||
|
||||
# failed to connect back?
|
||||
except (
|
||||
OSError,
|
||||
ConnectionError,
|
||||
):
|
||||
except OSError: # failed to connect
|
||||
log.warning(
|
||||
f'Failed to connect to spawning parent actor!?\n'
|
||||
f'\n'
|
||||
f'x=> {parent_addr}\n'
|
||||
f'|_{self}\n\n'
|
||||
)
|
||||
|
|
@ -1307,78 +1173,54 @@ class Actor:
|
|||
self,
|
||||
handler_nursery: Nursery,
|
||||
*,
|
||||
listen_addrs: list[UnwrappedAddress]|None = None,
|
||||
# (host, port) to bind for channel server
|
||||
listen_sockaddrs: list[tuple[str, int]]|None = None,
|
||||
|
||||
task_status: TaskStatus[Nursery] = trio.TASK_STATUS_IGNORED,
|
||||
) -> None:
|
||||
'''
|
||||
Start the IPC transport server, begin listening/accepting new
|
||||
`trio.SocketStream` connections.
|
||||
Start the IPC transport server, begin listening for new connections.
|
||||
|
||||
This will cause an actor to continue living (and thus
|
||||
blocking at the process/OS-thread level) until
|
||||
`.cancel_server()` is called.
|
||||
|
||||
'''
|
||||
if listen_addrs is None:
|
||||
listen_addrs = default_lo_addrs([
|
||||
_state._def_tpt_proto
|
||||
])
|
||||
|
||||
else:
|
||||
listen_addrs: list[Address] = [
|
||||
wrap_address(a) for a in listen_addrs
|
||||
]
|
||||
if listen_sockaddrs is None:
|
||||
listen_sockaddrs = [(None, 0)]
|
||||
|
||||
self._server_down = trio.Event()
|
||||
try:
|
||||
async with trio.open_nursery() as server_n:
|
||||
|
||||
listeners: list[trio.abc.Listener] = []
|
||||
for addr in listen_addrs:
|
||||
try:
|
||||
listener: trio.abc.Listener = await addr.open_listener()
|
||||
except OSError as oserr:
|
||||
if (
|
||||
'[Errno 98] Address already in use'
|
||||
in
|
||||
oserr.args#[0]
|
||||
):
|
||||
log.exception(
|
||||
f'Address already in use?\n'
|
||||
f'{addr}\n'
|
||||
)
|
||||
raise
|
||||
listeners.append(listener)
|
||||
|
||||
await server_n.start(
|
||||
for host, port in listen_sockaddrs:
|
||||
listeners: list[trio.abc.Listener] = await server_n.start(
|
||||
partial(
|
||||
trio.serve_listeners,
|
||||
trio.serve_tcp,
|
||||
|
||||
handler=self._stream_handler,
|
||||
listeners=listeners,
|
||||
port=port,
|
||||
host=host,
|
||||
|
||||
# NOTE: configured such that new
|
||||
# connections will stay alive even if
|
||||
# this server is cancelled!
|
||||
handler_nursery=handler_nursery
|
||||
handler_nursery=handler_nursery,
|
||||
)
|
||||
)
|
||||
# TODO, wow make this message better! XD
|
||||
log.info(
|
||||
'Started server(s)\n'
|
||||
+
|
||||
'\n'.join([f'|_{addr}' for addr in listen_addrs])
|
||||
sockets: list[trio.socket] = [
|
||||
getattr(listener, 'socket', 'unknown socket')
|
||||
for listener in listeners
|
||||
]
|
||||
log.runtime(
|
||||
'Started TCP server(s)\n'
|
||||
f'|_{sockets}\n'
|
||||
)
|
||||
self._listen_addrs.extend(listen_addrs)
|
||||
self._listeners.extend(listeners)
|
||||
|
||||
task_status.started(server_n)
|
||||
|
||||
finally:
|
||||
addr: Address
|
||||
for addr in listen_addrs:
|
||||
addr.close_listener()
|
||||
|
||||
# signal the server is down since nursery above terminated
|
||||
self._server_down.set()
|
||||
|
||||
|
|
@ -1485,13 +1327,8 @@ class Actor:
|
|||
if self._server_down is not None:
|
||||
await self._server_down.wait()
|
||||
else:
|
||||
tpt_protos: list[str] = []
|
||||
addr: Address
|
||||
for addr in self._listen_addrs:
|
||||
tpt_protos.append(addr.proto_key)
|
||||
log.warning(
|
||||
'Transport server(s) may have been cancelled before started?\n'
|
||||
f'protos: {tpt_protos!r}\n'
|
||||
'Transport[TCP] server was cancelled start?'
|
||||
)
|
||||
|
||||
# cancel all rpc tasks permanently
|
||||
|
|
@ -1742,21 +1579,26 @@ class Actor:
|
|||
return False
|
||||
|
||||
@property
|
||||
def accept_addrs(self) -> list[UnwrappedAddress]:
|
||||
def accept_addrs(self) -> list[tuple[str, int]]:
|
||||
'''
|
||||
All addresses to which the transport-channel server binds
|
||||
and listens for new connections.
|
||||
|
||||
'''
|
||||
return [a.unwrap() for a in self._listen_addrs]
|
||||
# throws OSError on failure
|
||||
return [
|
||||
listener.socket.getsockname()
|
||||
for listener in self._listeners
|
||||
] # type: ignore
|
||||
|
||||
@property
|
||||
def accept_addr(self) -> UnwrappedAddress:
|
||||
def accept_addr(self) -> tuple[str, int]:
|
||||
'''
|
||||
Primary address to which the IPC transport server is
|
||||
bound and listening for new connections.
|
||||
|
||||
'''
|
||||
# throws OSError on failure
|
||||
return self.accept_addrs[0]
|
||||
|
||||
def get_parent(self) -> Portal:
|
||||
|
|
@ -1778,6 +1620,43 @@ class Actor:
|
|||
'''
|
||||
return self._peers[uid]
|
||||
|
||||
# TODO: move to `Channel.handshake(uid)`
|
||||
async def _do_handshake(
|
||||
self,
|
||||
chan: Channel
|
||||
|
||||
) -> msgtypes.Aid:
|
||||
'''
|
||||
Exchange `(name, UUIDs)` identifiers as the first
|
||||
communication step with any (peer) remote `Actor`.
|
||||
|
||||
These are essentially the "mailbox addresses" found in
|
||||
"actor model" parlance.
|
||||
|
||||
'''
|
||||
name, uuid = self.uid
|
||||
await chan.send(
|
||||
msgtypes.Aid(
|
||||
name=name,
|
||||
uuid=uuid,
|
||||
)
|
||||
)
|
||||
aid: msgtypes.Aid = await chan.recv()
|
||||
chan.aid = aid
|
||||
|
||||
uid: tuple[str, str] = (
|
||||
# str(value[0]),
|
||||
# str(value[1])
|
||||
aid.name,
|
||||
aid.uuid,
|
||||
)
|
||||
|
||||
if not isinstance(uid, tuple):
|
||||
raise ValueError(f"{uid} is not a valid uid?!")
|
||||
|
||||
chan.uid = uid
|
||||
return uid
|
||||
|
||||
def is_infected_aio(self) -> bool:
|
||||
'''
|
||||
If `True`, this actor is running `trio` in guest mode on
|
||||
|
|
@ -1791,7 +1670,7 @@ class Actor:
|
|||
|
||||
async def async_main(
|
||||
actor: Actor,
|
||||
accept_addrs: UnwrappedAddress|None = None,
|
||||
accept_addrs: tuple[str, int]|None = None,
|
||||
|
||||
# XXX: currently ``parent_addr`` is only needed for the
|
||||
# ``multiprocessing`` backend (which pickles state sent to
|
||||
|
|
@ -1800,7 +1679,7 @@ async def async_main(
|
|||
# change this to a simple ``is_subactor: bool`` which will
|
||||
# be False when running as root actor and True when as
|
||||
# a subactor.
|
||||
parent_addr: UnwrappedAddress|None = None,
|
||||
parent_addr: tuple[str, int]|None = None,
|
||||
task_status: TaskStatus[None] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
|
|
@ -1815,8 +1694,6 @@ async def async_main(
|
|||
the actor's "runtime" and all thus all ongoing RPC tasks.
|
||||
|
||||
'''
|
||||
actor._task: trio.Task = trio.lowlevel.current_task()
|
||||
|
||||
# attempt to retreive ``trio``'s sigint handler and stash it
|
||||
# on our debugger state.
|
||||
_debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
|
||||
|
|
@ -1826,15 +1703,13 @@ async def async_main(
|
|||
|
||||
# establish primary connection with immediate parent
|
||||
actor._parent_chan: Channel|None = None
|
||||
|
||||
if parent_addr is not None:
|
||||
|
||||
(
|
||||
actor._parent_chan,
|
||||
set_accept_addr_says_rent,
|
||||
maybe_preferred_transports_says_rent,
|
||||
) = await actor._from_parent(parent_addr)
|
||||
|
||||
accept_addrs: list[UnwrappedAddress] = []
|
||||
# either it's passed in because we're not a child or
|
||||
# because we're running in mp mode
|
||||
if (
|
||||
|
|
@ -1843,18 +1718,6 @@ async def async_main(
|
|||
set_accept_addr_says_rent is not None
|
||||
):
|
||||
accept_addrs = set_accept_addr_says_rent
|
||||
else:
|
||||
enable_transports: list[str] = (
|
||||
maybe_preferred_transports_says_rent
|
||||
or
|
||||
[_state._def_tpt_proto]
|
||||
)
|
||||
for transport_key in enable_transports:
|
||||
transport_cls: Type[Address] = get_address_cls(
|
||||
transport_key
|
||||
)
|
||||
addr: Address = transport_cls.get_random()
|
||||
accept_addrs.append(addr.unwrap())
|
||||
|
||||
# The "root" nursery ensures the channel with the immediate
|
||||
# parent is kept alive as a resilient service until
|
||||
|
|
@ -1906,7 +1769,7 @@ async def async_main(
|
|||
partial(
|
||||
actor._serve_forever,
|
||||
service_nursery,
|
||||
listen_addrs=accept_addrs,
|
||||
listen_sockaddrs=accept_addrs,
|
||||
)
|
||||
)
|
||||
except OSError as oserr:
|
||||
|
|
@ -1922,7 +1785,7 @@ async def async_main(
|
|||
|
||||
raise
|
||||
|
||||
accept_addrs: list[UnwrappedAddress] = actor.accept_addrs
|
||||
accept_addrs: list[tuple[str, int]] = actor.accept_addrs
|
||||
|
||||
# NOTE: only set the loopback addr for the
|
||||
# process-tree-global "root" mailbox since
|
||||
|
|
@ -1930,8 +1793,9 @@ async def async_main(
|
|||
# their root actor over that channel.
|
||||
if _state._runtime_vars['_is_root']:
|
||||
for addr in accept_addrs:
|
||||
waddr = wrap_address(addr)
|
||||
if waddr == waddr.get_root():
|
||||
host, _ = addr
|
||||
# TODO: generic 'lo' detector predicate
|
||||
if '127.0.0.1' in host:
|
||||
_state._runtime_vars['_root_mailbox'] = addr
|
||||
|
||||
# Register with the arbiter if we're told its addr
|
||||
|
|
@ -1946,21 +1810,24 @@ async def async_main(
|
|||
# only on unique actor uids?
|
||||
for addr in actor.reg_addrs:
|
||||
try:
|
||||
waddr = wrap_address(addr)
|
||||
assert waddr.is_valid
|
||||
assert isinstance(addr, tuple)
|
||||
assert addr[1] # non-zero after bind
|
||||
except AssertionError:
|
||||
await _debug.pause()
|
||||
|
||||
async with get_registry(addr) as reg_portal:
|
||||
async with get_registry(*addr) as reg_portal:
|
||||
for accept_addr in accept_addrs:
|
||||
accept_addr = wrap_address(accept_addr)
|
||||
assert accept_addr.is_valid
|
||||
|
||||
if not accept_addr[1]:
|
||||
await _debug.pause()
|
||||
|
||||
assert accept_addr[1]
|
||||
|
||||
await reg_portal.run_from_ns(
|
||||
'self',
|
||||
'register_actor',
|
||||
uid=actor.uid,
|
||||
addr=accept_addr.unwrap(),
|
||||
sockaddr=accept_addr,
|
||||
)
|
||||
|
||||
is_registered: bool = True
|
||||
|
|
@ -2087,13 +1954,12 @@ async def async_main(
|
|||
):
|
||||
failed: bool = False
|
||||
for addr in actor.reg_addrs:
|
||||
waddr = wrap_address(addr)
|
||||
assert waddr.is_valid
|
||||
assert isinstance(addr, tuple)
|
||||
with trio.move_on_after(0.5) as cs:
|
||||
cs.shield = True
|
||||
try:
|
||||
async with get_registry(
|
||||
addr,
|
||||
*addr,
|
||||
) as reg_portal:
|
||||
await reg_portal.run_from_ns(
|
||||
'self',
|
||||
|
|
@ -2135,15 +2001,15 @@ async def async_main(
|
|||
log.info(teardown_report)
|
||||
|
||||
|
||||
# TODO: rename to `Registry` and move to `.discovery._registry`!
|
||||
# TODO: rename to `Registry` and move to `._discovery`!
|
||||
class Arbiter(Actor):
|
||||
'''
|
||||
A special registrar (and for now..) `Actor` who can contact all
|
||||
other actors within its immediate process tree and possibly keeps
|
||||
a registry of others meant to be discoverable in a distributed
|
||||
application. Normally the registrar is also the "root actor" and
|
||||
thus always has access to the top-most-level actor (process)
|
||||
nursery.
|
||||
A special registrar actor who can contact all other actors
|
||||
within its immediate process tree and possibly keeps a registry
|
||||
of others meant to be discoverable in a distributed
|
||||
application. Normally the registrar is also the "root actor"
|
||||
and thus always has access to the top-most-level actor
|
||||
(process) nursery.
|
||||
|
||||
By default, the registrar is always initialized when and if no
|
||||
other registrar socket addrs have been specified to runtime
|
||||
|
|
@ -2163,12 +2029,6 @@ class Arbiter(Actor):
|
|||
'''
|
||||
is_arbiter = True
|
||||
|
||||
# TODO, implement this as a read on there existing a `._state` of
|
||||
# some sort setup by whenever we impl this all as
|
||||
# a `.discovery._registry.open_registry()` API
|
||||
def is_registry(self) -> bool:
|
||||
return self.is_arbiter
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
|
|
@ -2177,7 +2037,7 @@ class Arbiter(Actor):
|
|||
|
||||
self._registry: dict[
|
||||
tuple[str, str],
|
||||
UnwrappedAddress,
|
||||
tuple[str, int],
|
||||
] = {}
|
||||
self._waiters: dict[
|
||||
str,
|
||||
|
|
@ -2193,18 +2053,18 @@ class Arbiter(Actor):
|
|||
self,
|
||||
name: str,
|
||||
|
||||
) -> UnwrappedAddress|None:
|
||||
) -> tuple[str, int]|None:
|
||||
|
||||
for uid, addr in self._registry.items():
|
||||
for uid, sockaddr in self._registry.items():
|
||||
if name in uid:
|
||||
return addr
|
||||
return sockaddr
|
||||
|
||||
return None
|
||||
|
||||
async def get_registry(
|
||||
self
|
||||
|
||||
) -> dict[str, UnwrappedAddress]:
|
||||
) -> dict[str, tuple[str, int]]:
|
||||
'''
|
||||
Return current name registry.
|
||||
|
||||
|
|
@ -2224,7 +2084,7 @@ class Arbiter(Actor):
|
|||
self,
|
||||
name: str,
|
||||
|
||||
) -> list[UnwrappedAddress]:
|
||||
) -> list[tuple[str, int]]:
|
||||
'''
|
||||
Wait for a particular actor to register.
|
||||
|
||||
|
|
@ -2232,41 +2092,44 @@ class Arbiter(Actor):
|
|||
registered.
|
||||
|
||||
'''
|
||||
addrs: list[UnwrappedAddress] = []
|
||||
addr: UnwrappedAddress
|
||||
sockaddrs: list[tuple[str, int]] = []
|
||||
sockaddr: tuple[str, int]
|
||||
|
||||
mailbox_info: str = 'Actor registry contact infos:\n'
|
||||
for uid, addr in self._registry.items():
|
||||
for uid, sockaddr in self._registry.items():
|
||||
mailbox_info += (
|
||||
f'|_uid: {uid}\n'
|
||||
f'|_addr: {addr}\n\n'
|
||||
f'|_sockaddr: {sockaddr}\n\n'
|
||||
)
|
||||
if name == uid[0]:
|
||||
addrs.append(addr)
|
||||
sockaddrs.append(sockaddr)
|
||||
|
||||
if not addrs:
|
||||
if not sockaddrs:
|
||||
waiter = trio.Event()
|
||||
self._waiters.setdefault(name, []).append(waiter)
|
||||
await waiter.wait()
|
||||
|
||||
for uid in self._waiters[name]:
|
||||
if not isinstance(uid, trio.Event):
|
||||
addrs.append(self._registry[uid])
|
||||
sockaddrs.append(self._registry[uid])
|
||||
|
||||
log.runtime(mailbox_info)
|
||||
return addrs
|
||||
return sockaddrs
|
||||
|
||||
async def register_actor(
|
||||
self,
|
||||
uid: tuple[str, str],
|
||||
addr: UnwrappedAddress
|
||||
sockaddr: tuple[str, int]
|
||||
|
||||
) -> None:
|
||||
uid = name, hash = (str(uid[0]), str(uid[1]))
|
||||
waddr: Address = wrap_address(addr)
|
||||
if not waddr.is_valid:
|
||||
# should never be 0-dynamic-os-alloc
|
||||
addr = (host, port) = (
|
||||
str(sockaddr[0]),
|
||||
int(sockaddr[1]),
|
||||
)
|
||||
if port == 0:
|
||||
await _debug.pause()
|
||||
|
||||
assert port # should never be 0-dynamic-os-alloc
|
||||
self._registry[uid] = addr
|
||||
|
||||
# pop and signal all waiter events
|
||||
|
|
|
|||
|
|
@ -46,13 +46,11 @@ from tractor._state import (
|
|||
_runtime_vars,
|
||||
)
|
||||
from tractor.log import get_logger
|
||||
from tractor._addr import UnwrappedAddress
|
||||
from tractor._portal import Portal
|
||||
from tractor._runtime import Actor
|
||||
from tractor._entry import _mp_main
|
||||
from tractor._exceptions import ActorFailure
|
||||
from tractor.msg.types import (
|
||||
Aid,
|
||||
SpawnSpec,
|
||||
)
|
||||
|
||||
|
|
@ -165,7 +163,7 @@ async def exhaust_portal(
|
|||
# TODO: merge with above?
|
||||
log.warning(
|
||||
'Cancelled portal result waiter task:\n'
|
||||
f'uid: {portal.channel.aid}\n'
|
||||
f'uid: {portal.channel.uid}\n'
|
||||
f'error: {err}\n'
|
||||
)
|
||||
return err
|
||||
|
|
@ -173,7 +171,7 @@ async def exhaust_portal(
|
|||
else:
|
||||
log.debug(
|
||||
f'Returning final result from portal:\n'
|
||||
f'uid: {portal.channel.aid}\n'
|
||||
f'uid: {portal.channel.uid}\n'
|
||||
f'result: {final}\n'
|
||||
)
|
||||
return final
|
||||
|
|
@ -326,12 +324,12 @@ async def soft_kill(
|
|||
see `.hard_kill()`).
|
||||
|
||||
'''
|
||||
peer_aid: Aid = portal.channel.aid
|
||||
uid: tuple[str, str] = portal.channel.uid
|
||||
try:
|
||||
log.cancel(
|
||||
f'Soft killing sub-actor via portal request\n'
|
||||
f'\n'
|
||||
f'(c=> {peer_aid}\n'
|
||||
f'(c=> {portal.chan.uid}\n'
|
||||
f' |_{proc}\n'
|
||||
)
|
||||
# wait on sub-proc to signal termination
|
||||
|
|
@ -380,7 +378,7 @@ async def soft_kill(
|
|||
if proc.poll() is None: # type: ignore
|
||||
log.warning(
|
||||
'Subactor still alive after cancel request?\n\n'
|
||||
f'uid: {peer_aid}\n'
|
||||
f'uid: {uid}\n'
|
||||
f'|_{proc}\n'
|
||||
)
|
||||
n.cancel_scope.cancel()
|
||||
|
|
@ -394,15 +392,14 @@ async def new_proc(
|
|||
errors: dict[tuple[str, str], Exception],
|
||||
|
||||
# passed through to actor main
|
||||
bind_addrs: list[UnwrappedAddress],
|
||||
parent_addr: UnwrappedAddress,
|
||||
bind_addrs: list[tuple[str, int]],
|
||||
parent_addr: tuple[str, int],
|
||||
_runtime_vars: dict[str, Any], # serialized and sent to _child
|
||||
|
||||
*,
|
||||
|
||||
infect_asyncio: bool = False,
|
||||
task_status: TaskStatus[Portal] = trio.TASK_STATUS_IGNORED,
|
||||
proc_kwargs: dict[str, any] = {}
|
||||
task_status: TaskStatus[Portal] = trio.TASK_STATUS_IGNORED
|
||||
|
||||
) -> None:
|
||||
|
||||
|
|
@ -422,7 +419,6 @@ async def new_proc(
|
|||
_runtime_vars, # run time vars
|
||||
infect_asyncio=infect_asyncio,
|
||||
task_status=task_status,
|
||||
proc_kwargs=proc_kwargs
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -433,13 +429,12 @@ async def trio_proc(
|
|||
errors: dict[tuple[str, str], Exception],
|
||||
|
||||
# passed through to actor main
|
||||
bind_addrs: list[UnwrappedAddress],
|
||||
parent_addr: UnwrappedAddress,
|
||||
bind_addrs: list[tuple[str, int]],
|
||||
parent_addr: tuple[str, int],
|
||||
_runtime_vars: dict[str, Any], # serialized and sent to _child
|
||||
*,
|
||||
infect_asyncio: bool = False,
|
||||
task_status: TaskStatus[Portal] = trio.TASK_STATUS_IGNORED,
|
||||
proc_kwargs: dict[str, any] = {}
|
||||
task_status: TaskStatus[Portal] = trio.TASK_STATUS_IGNORED
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
|
|
@ -461,9 +456,6 @@ async def trio_proc(
|
|||
# the OS; it otherwise can be passed via the parent channel if
|
||||
# we prefer in the future (for privacy).
|
||||
"--uid",
|
||||
# TODO, how to pass this over "wire" encodings like
|
||||
# cmdline args?
|
||||
# -[ ] maybe we can add an `Aid.min_tuple()` ?
|
||||
str(subactor.uid),
|
||||
# Address the child must connect to on startup
|
||||
"--parent_addr",
|
||||
|
|
@ -483,7 +475,7 @@ async def trio_proc(
|
|||
proc: trio.Process|None = None
|
||||
try:
|
||||
try:
|
||||
proc: trio.Process = await trio.lowlevel.open_process(spawn_cmd, **proc_kwargs)
|
||||
proc: trio.Process = await trio.lowlevel.open_process(spawn_cmd)
|
||||
log.runtime(
|
||||
'Started new child\n'
|
||||
f'|_{proc}\n'
|
||||
|
|
@ -525,15 +517,15 @@ async def trio_proc(
|
|||
|
||||
# send a "spawning specification" which configures the
|
||||
# initial runtime state of the child.
|
||||
sspec = SpawnSpec(
|
||||
await chan.send(
|
||||
SpawnSpec(
|
||||
_parent_main_data=subactor._parent_main_data,
|
||||
enable_modules=subactor.enable_modules,
|
||||
reg_addrs=subactor.reg_addrs,
|
||||
bind_addrs=bind_addrs,
|
||||
_runtime_vars=_runtime_vars,
|
||||
)
|
||||
log.runtime(f'Sending spawn spec: {str(sspec)}')
|
||||
await chan.send(sspec)
|
||||
)
|
||||
|
||||
# track subactor in current nursery
|
||||
curr_actor: Actor = current_actor()
|
||||
|
|
@ -643,13 +635,12 @@ async def mp_proc(
|
|||
subactor: Actor,
|
||||
errors: dict[tuple[str, str], Exception],
|
||||
# passed through to actor main
|
||||
bind_addrs: list[UnwrappedAddress],
|
||||
parent_addr: UnwrappedAddress,
|
||||
bind_addrs: list[tuple[str, int]],
|
||||
parent_addr: tuple[str, int],
|
||||
_runtime_vars: dict[str, Any], # serialized and sent to _child
|
||||
*,
|
||||
infect_asyncio: bool = False,
|
||||
task_status: TaskStatus[Portal] = trio.TASK_STATUS_IGNORED,
|
||||
proc_kwargs: dict[str, any] = {}
|
||||
task_status: TaskStatus[Portal] = trio.TASK_STATUS_IGNORED
|
||||
|
||||
) -> None:
|
||||
|
||||
|
|
@ -729,8 +720,7 @@ async def mp_proc(
|
|||
# channel should have handshake completed by the
|
||||
# local actor by the time we get a ref to it
|
||||
event, chan = await actor_nursery._actor.wait_for_peer(
|
||||
subactor.uid,
|
||||
)
|
||||
subactor.uid)
|
||||
|
||||
# XXX: monkey patch poll API to match the ``subprocess`` API..
|
||||
# not sure why they don't expose this but kk.
|
||||
|
|
|
|||
|
|
@ -14,19 +14,16 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Per actor-process runtime state mgmt APIs.
|
||||
"""
|
||||
Per process state
|
||||
|
||||
'''
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from contextvars import (
|
||||
ContextVar,
|
||||
)
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
Any,
|
||||
Literal,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
|
|
@ -146,30 +143,3 @@ def current_ipc_ctx(
|
|||
f'|_{current_task()}\n'
|
||||
)
|
||||
return ctx
|
||||
|
||||
|
||||
# std ODE (mutable) app state location
|
||||
_rtdir: Path = Path(os.environ['XDG_RUNTIME_DIR'])
|
||||
|
||||
|
||||
def get_rt_dir(
|
||||
subdir: str = 'tractor'
|
||||
) -> Path:
|
||||
'''
|
||||
Return the user "runtime dir" where most userspace apps stick
|
||||
their IPC and cache related system util-files; we take hold
|
||||
of a `'XDG_RUNTIME_DIR'/tractor/` subdir by default.
|
||||
|
||||
'''
|
||||
rtdir: Path = _rtdir / subdir
|
||||
if not rtdir.is_dir():
|
||||
rtdir.mkdir()
|
||||
return rtdir
|
||||
|
||||
|
||||
# default IPC transport protocol settings
|
||||
TransportProtocolKey = Literal[
|
||||
'tcp',
|
||||
'uds',
|
||||
]
|
||||
_def_tpt_proto: TransportProtocolKey = 'tcp'
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ from tractor.msg import (
|
|||
if TYPE_CHECKING:
|
||||
from ._runtime import Actor
|
||||
from ._context import Context
|
||||
from .ipc import Channel
|
||||
from ._ipc import Channel
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
|
@ -437,23 +437,22 @@ class MsgStream(trio.abc.Channel):
|
|||
message: str = (
|
||||
f'Stream self-closed by {this_side!r}-side before EoC from {peer_side!r}\n'
|
||||
# } bc a stream is a "scope"/msging-phase inside an IPC
|
||||
f'c}}>\n'
|
||||
f'x}}>\n'
|
||||
f' |_{self}\n'
|
||||
)
|
||||
log.cancel(message)
|
||||
self._eoc = trio.EndOfChannel(message)
|
||||
|
||||
if (
|
||||
(rx_chan := self._rx_chan)
|
||||
and
|
||||
(stats := rx_chan.statistics()).tasks_waiting_receive
|
||||
):
|
||||
message += (
|
||||
f'AND there is still reader tasks,\n'
|
||||
f'\n'
|
||||
log.cancel(
|
||||
f'Msg-stream is closing but there is still reader tasks,\n'
|
||||
f'{stats}\n'
|
||||
)
|
||||
|
||||
log.cancel(message)
|
||||
self._eoc = trio.EndOfChannel(message)
|
||||
|
||||
# ?XXX WAIT, why do we not close the local mem chan `._rx_chan` XXX?
|
||||
# => NO, DEFINITELY NOT! <=
|
||||
# if we're a bi-dir `MsgStream` BECAUSE this same
|
||||
|
|
@ -596,17 +595,8 @@ class MsgStream(trio.abc.Channel):
|
|||
trio.ClosedResourceError,
|
||||
trio.BrokenResourceError,
|
||||
BrokenPipeError,
|
||||
) as _trans_err:
|
||||
trans_err = _trans_err
|
||||
if (
|
||||
hide_tb
|
||||
and
|
||||
self._ctx.chan._exc is trans_err
|
||||
# ^XXX, IOW, only if the channel is marked errored
|
||||
# for the same reason as whatever its underlying
|
||||
# transport raised, do we keep the full low-level tb
|
||||
# suppressed from the user.
|
||||
):
|
||||
) as trans_err:
|
||||
if hide_tb:
|
||||
raise type(trans_err)(
|
||||
*trans_err.args
|
||||
) from trans_err
|
||||
|
|
@ -812,12 +802,13 @@ async def open_stream_from_ctx(
|
|||
# sanity, can remove?
|
||||
assert eoc is stream._eoc
|
||||
|
||||
log.runtime(
|
||||
log.warning(
|
||||
'Stream was terminated by EoC\n\n'
|
||||
# NOTE: won't show the error <Type> but
|
||||
# does show txt followed by IPC msg.
|
||||
f'{str(eoc)}\n'
|
||||
)
|
||||
|
||||
finally:
|
||||
if ctx._portal:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -22,20 +22,13 @@ from contextlib import asynccontextmanager as acm
|
|||
from functools import partial
|
||||
import inspect
|
||||
from pprint import pformat
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
from typing import TYPE_CHECKING
|
||||
import typing
|
||||
import warnings
|
||||
|
||||
import trio
|
||||
|
||||
|
||||
from .devx._debug import maybe_wait_for_debugger
|
||||
from ._addr import (
|
||||
UnwrappedAddress,
|
||||
mk_uuid,
|
||||
)
|
||||
from ._state import current_actor, is_main_process
|
||||
from .log import get_logger, get_loglevel
|
||||
from ._runtime import Actor
|
||||
|
|
@ -44,9 +37,7 @@ from ._exceptions import (
|
|||
is_multi_cancelled,
|
||||
ContextCancelled,
|
||||
)
|
||||
from ._root import (
|
||||
open_root_actor,
|
||||
)
|
||||
from ._root import open_root_actor
|
||||
from . import _state
|
||||
from . import _spawn
|
||||
|
||||
|
|
@ -56,6 +47,8 @@ if TYPE_CHECKING:
|
|||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_default_bind_addr: tuple[str, int] = ('127.0.0.1', 0)
|
||||
|
||||
|
||||
class ActorNursery:
|
||||
'''
|
||||
|
|
@ -137,9 +130,8 @@ class ActorNursery:
|
|||
|
||||
*,
|
||||
|
||||
bind_addrs: list[UnwrappedAddress]|None = None,
|
||||
bind_addrs: list[tuple[str, int]] = [_default_bind_addr],
|
||||
rpc_module_paths: list[str]|None = None,
|
||||
enable_transports: list[str] = [_state._def_tpt_proto],
|
||||
enable_modules: list[str]|None = None,
|
||||
loglevel: str|None = None, # set log level per subactor
|
||||
debug_mode: bool|None = None,
|
||||
|
|
@ -149,7 +141,6 @@ class ActorNursery:
|
|||
# a `._ria_nursery` since the dependent APIs have been
|
||||
# removed!
|
||||
nursery: trio.Nursery|None = None,
|
||||
proc_kwargs: dict[str, any] = {}
|
||||
|
||||
) -> Portal:
|
||||
'''
|
||||
|
|
@ -186,9 +177,7 @@ class ActorNursery:
|
|||
enable_modules.extend(rpc_module_paths)
|
||||
|
||||
subactor = Actor(
|
||||
name=name,
|
||||
uuid=mk_uuid(),
|
||||
|
||||
name,
|
||||
# modules allowed to invoked funcs from
|
||||
enable_modules=enable_modules,
|
||||
loglevel=loglevel,
|
||||
|
|
@ -196,7 +185,7 @@ class ActorNursery:
|
|||
# verbatim relay this actor's registrar addresses
|
||||
registry_addrs=current_actor().reg_addrs,
|
||||
)
|
||||
parent_addr: UnwrappedAddress = self._actor.accept_addr
|
||||
parent_addr = self._actor.accept_addr
|
||||
assert parent_addr
|
||||
|
||||
# start a task to spawn a process
|
||||
|
|
@ -215,7 +204,6 @@ class ActorNursery:
|
|||
parent_addr,
|
||||
_rtv, # run time vars
|
||||
infect_asyncio=infect_asyncio,
|
||||
proc_kwargs=proc_kwargs
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -234,12 +222,11 @@ class ActorNursery:
|
|||
*,
|
||||
|
||||
name: str | None = None,
|
||||
bind_addrs: UnwrappedAddress|None = None,
|
||||
bind_addrs: tuple[str, int] = [_default_bind_addr],
|
||||
rpc_module_paths: list[str] | None = None,
|
||||
enable_modules: list[str] | None = None,
|
||||
loglevel: str | None = None, # set log level per subactor
|
||||
infect_asyncio: bool = False,
|
||||
proc_kwargs: dict[str, any] = {},
|
||||
|
||||
**kwargs, # explicit args to ``fn``
|
||||
|
||||
|
|
@ -270,7 +257,6 @@ class ActorNursery:
|
|||
# use the run_in_actor nursery
|
||||
nursery=self._ria_nursery,
|
||||
infect_asyncio=infect_asyncio,
|
||||
proc_kwargs=proc_kwargs
|
||||
)
|
||||
|
||||
# XXX: don't allow stream funcs
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
import os
|
||||
import random
|
||||
|
||||
|
||||
def generate_sample_messages(
|
||||
amount: int,
|
||||
rand_min: int = 0,
|
||||
rand_max: int = 0,
|
||||
silent: bool = False
|
||||
) -> tuple[list[bytes], int]:
|
||||
|
||||
msgs = []
|
||||
size = 0
|
||||
|
||||
if not silent:
|
||||
print(f'\ngenerating {amount} messages...')
|
||||
|
||||
for i in range(amount):
|
||||
msg = f'[{i:08}]'.encode('utf-8')
|
||||
|
||||
if rand_max > 0:
|
||||
msg += os.urandom(
|
||||
random.randint(rand_min, rand_max))
|
||||
|
||||
size += len(msg)
|
||||
|
||||
msgs.append(msg)
|
||||
|
||||
if not silent and i and i % 10_000 == 0:
|
||||
print(f'{i} generated')
|
||||
|
||||
if not silent:
|
||||
print(f'done, {size:,} bytes in total')
|
||||
|
||||
return msgs, size
|
||||
|
|
@ -73,7 +73,6 @@ from tractor.log import get_logger
|
|||
from tractor._context import Context
|
||||
from tractor import _state
|
||||
from tractor._exceptions import (
|
||||
DebugRequestError,
|
||||
InternalError,
|
||||
NoRuntime,
|
||||
is_multi_cancelled,
|
||||
|
|
@ -92,7 +91,7 @@ from tractor._state import (
|
|||
if TYPE_CHECKING:
|
||||
from trio.lowlevel import Task
|
||||
from threading import Thread
|
||||
from tractor.ipc import Channel
|
||||
from tractor._ipc import Channel
|
||||
from tractor._runtime import (
|
||||
Actor,
|
||||
)
|
||||
|
|
@ -1741,6 +1740,13 @@ def sigint_shield(
|
|||
_pause_msg: str = 'Opening a pdb REPL in paused actor'
|
||||
|
||||
|
||||
class DebugRequestError(RuntimeError):
|
||||
'''
|
||||
Failed to request stdio lock from root actor!
|
||||
|
||||
'''
|
||||
|
||||
|
||||
_repl_fail_msg: str|None = (
|
||||
'Failed to REPl via `_pause()` '
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ Pretty formatters for use throughout the code base.
|
|||
Mostly handy for logging and exception message content.
|
||||
|
||||
'''
|
||||
import sys
|
||||
import textwrap
|
||||
import traceback
|
||||
|
||||
|
|
@ -116,85 +115,6 @@ def pformat_boxed_tb(
|
|||
)
|
||||
|
||||
|
||||
def pformat_exc(
|
||||
exc: Exception,
|
||||
header: str = '',
|
||||
message: str = '',
|
||||
body: str = '',
|
||||
with_type_header: bool = True,
|
||||
) -> str:
|
||||
|
||||
# XXX when the currently raised exception is this instance,
|
||||
# we do not ever use the "type header" style repr.
|
||||
is_being_raised: bool = False
|
||||
if (
|
||||
(curr_exc := sys.exception())
|
||||
and
|
||||
curr_exc is exc
|
||||
):
|
||||
is_being_raised: bool = True
|
||||
|
||||
with_type_header: bool = (
|
||||
with_type_header
|
||||
and
|
||||
not is_being_raised
|
||||
)
|
||||
|
||||
# <RemoteActorError( .. )> style
|
||||
if (
|
||||
with_type_header
|
||||
and
|
||||
not header
|
||||
):
|
||||
header: str = f'<{type(exc).__name__}('
|
||||
|
||||
message: str = (
|
||||
message
|
||||
or
|
||||
exc.message
|
||||
)
|
||||
if message:
|
||||
# split off the first line so, if needed, it isn't
|
||||
# indented the same like the "boxed content" which
|
||||
# since there is no `.tb_str` is just the `.message`.
|
||||
lines: list[str] = message.splitlines()
|
||||
first: str = lines[0]
|
||||
message: str = message.removeprefix(first)
|
||||
|
||||
# with a type-style header we,
|
||||
# - have no special message "first line" extraction/handling
|
||||
# - place the message a space in from the header:
|
||||
# `MsgTypeError( <message> ..`
|
||||
# ^-here
|
||||
# - indent the `.message` inside the type body.
|
||||
if with_type_header:
|
||||
first = f' {first} )>'
|
||||
|
||||
message: str = textwrap.indent(
|
||||
message,
|
||||
prefix=' '*2,
|
||||
)
|
||||
message: str = first + message
|
||||
|
||||
tail: str = ''
|
||||
if (
|
||||
with_type_header
|
||||
and
|
||||
not message
|
||||
):
|
||||
tail: str = '>'
|
||||
|
||||
return (
|
||||
header
|
||||
+
|
||||
message
|
||||
+
|
||||
f'{body}'
|
||||
+
|
||||
tail
|
||||
)
|
||||
|
||||
|
||||
def pformat_caller_frame(
|
||||
stack_limit: int = 1,
|
||||
box_tb: bool = True,
|
||||
|
|
|
|||
|
|
@ -45,8 +45,6 @@ __all__ = ['pub']
|
|||
log = get_logger('messaging')
|
||||
|
||||
|
||||
# TODO! this needs to reworked to use the modern
|
||||
# `Context`/`MsgStream` APIs!!
|
||||
async def fan_out_to_ctxs(
|
||||
pub_async_gen_func: typing.Callable, # it's an async gen ... gd mypy
|
||||
topics2ctxs: dict[str, list],
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
import platform
|
||||
|
||||
from ._transport import (
|
||||
MsgTransportKey as MsgTransportKey,
|
||||
MsgType as MsgType,
|
||||
MsgTransport as MsgTransport,
|
||||
MsgpackTransport as MsgpackTransport
|
||||
)
|
||||
|
||||
from ._tcp import MsgpackTCPStream as MsgpackTCPStream
|
||||
from ._uds import MsgpackUDSStream as MsgpackUDSStream
|
||||
|
||||
from ._types import (
|
||||
transport_from_addr as transport_from_addr,
|
||||
transport_from_stream as transport_from_stream,
|
||||
)
|
||||
|
||||
from ._chan import (
|
||||
_connect_chan as _connect_chan,
|
||||
Channel as Channel
|
||||
)
|
||||
|
||||
if platform.system() == 'Linux':
|
||||
from ._linux import (
|
||||
EFD_SEMAPHORE as EFD_SEMAPHORE,
|
||||
EFD_CLOEXEC as EFD_CLOEXEC,
|
||||
EFD_NONBLOCK as EFD_NONBLOCK,
|
||||
open_eventfd as open_eventfd,
|
||||
write_eventfd as write_eventfd,
|
||||
read_eventfd as read_eventfd,
|
||||
close_eventfd as close_eventfd,
|
||||
EventFD as EventFD,
|
||||
)
|
||||
|
||||
from ._ringbuf import (
|
||||
RBToken as RBToken,
|
||||
RingBuffSender as RingBuffSender,
|
||||
RingBuffReceiver as RingBuffReceiver,
|
||||
open_ringbuf as open_ringbuf
|
||||
)
|
||||
|
|
@ -1,444 +0,0 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Inter-process comms abstractions
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
contextmanager as cm,
|
||||
)
|
||||
import platform
|
||||
from pprint import pformat
|
||||
import typing
|
||||
from typing import (
|
||||
Any,
|
||||
)
|
||||
import warnings
|
||||
|
||||
import trio
|
||||
|
||||
from tractor.ipc._transport import MsgTransport
|
||||
from tractor.ipc._types import (
|
||||
transport_from_addr,
|
||||
transport_from_stream,
|
||||
)
|
||||
from tractor._addr import (
|
||||
is_wrapped_addr,
|
||||
wrap_address,
|
||||
Address,
|
||||
UnwrappedAddress,
|
||||
)
|
||||
from tractor.log import get_logger
|
||||
from tractor._exceptions import (
|
||||
MsgTypeError,
|
||||
pack_from_raise,
|
||||
)
|
||||
from tractor.msg import (
|
||||
Aid,
|
||||
MsgCodec,
|
||||
)
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_is_windows = platform.system() == 'Windows'
|
||||
|
||||
|
||||
class Channel:
|
||||
'''
|
||||
An inter-process channel for communication between (remote) actors.
|
||||
|
||||
Wraps a ``MsgStream``: transport + encoding IPC connection.
|
||||
|
||||
Currently we only support ``trio.SocketStream`` for transport
|
||||
(aka TCP) and the ``msgpack`` interchange format via the ``msgspec``
|
||||
codec libary.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
|
||||
self,
|
||||
transport: MsgTransport|None = None,
|
||||
# TODO: optional reconnection support?
|
||||
# auto_reconnect: bool = False,
|
||||
# on_reconnect: typing.Callable[..., typing.Awaitable] = None,
|
||||
|
||||
) -> None:
|
||||
|
||||
# self._recon_seq = on_reconnect
|
||||
# self._autorecon = auto_reconnect
|
||||
|
||||
# Either created in ``.connect()`` or passed in by
|
||||
# user in ``.from_stream()``.
|
||||
self._transport: MsgTransport|None = transport
|
||||
|
||||
# set after handshake - always info from peer end
|
||||
self.aid: Aid|None = None
|
||||
|
||||
self._aiter_msgs = self._iter_msgs()
|
||||
self._exc: Exception|None = None
|
||||
# ^XXX! ONLY set if a remote actor sends an `Error`-msg
|
||||
self._closed: bool = False
|
||||
|
||||
# flag set by ``Portal.cancel_actor()`` indicating remote
|
||||
# (possibly peer) cancellation of the far end actor
|
||||
# runtime.
|
||||
self._cancel_called: bool = False
|
||||
|
||||
@property
|
||||
def uid(self) -> tuple[str, str]:
|
||||
'''
|
||||
Peer actor's unique id.
|
||||
|
||||
'''
|
||||
msg: str = (
|
||||
f'`{type(self).__name__}.uid` is now deprecated.\n'
|
||||
'Use the new `.aid: tractor.msg.Aid` (struct) instead '
|
||||
'which also provides additional named (optional) fields '
|
||||
'beyond just the `.name` and `.uuid`.'
|
||||
)
|
||||
warnings.warn(
|
||||
msg,
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
peer_aid: Aid = self.aid
|
||||
return (
|
||||
peer_aid.name,
|
||||
peer_aid.uuid,
|
||||
)
|
||||
|
||||
@property
|
||||
def stream(self) -> trio.abc.Stream | None:
|
||||
return self._transport.stream if self._transport else None
|
||||
|
||||
@property
|
||||
def msgstream(self) -> MsgTransport:
|
||||
log.info(
|
||||
'`Channel.msgstream` is an old name, use `._transport`'
|
||||
)
|
||||
return self._transport
|
||||
|
||||
@property
|
||||
def transport(self) -> MsgTransport:
|
||||
return self._transport
|
||||
|
||||
@classmethod
|
||||
def from_stream(
|
||||
cls,
|
||||
stream: trio.abc.Stream,
|
||||
) -> Channel:
|
||||
transport_cls = transport_from_stream(stream)
|
||||
return Channel(
|
||||
transport=transport_cls(stream)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def from_addr(
|
||||
cls,
|
||||
addr: UnwrappedAddress,
|
||||
**kwargs
|
||||
) -> Channel:
|
||||
|
||||
if not is_wrapped_addr(addr):
|
||||
addr: Address = wrap_address(addr)
|
||||
|
||||
transport_cls = transport_from_addr(addr)
|
||||
transport = await transport_cls.connect_to(
|
||||
addr,
|
||||
**kwargs,
|
||||
)
|
||||
assert transport.raddr == addr
|
||||
chan = Channel(transport=transport)
|
||||
log.runtime(
|
||||
f'Connected channel IPC transport\n'
|
||||
f'[>\n'
|
||||
f' |_{chan}\n'
|
||||
)
|
||||
return chan
|
||||
|
||||
@cm
|
||||
def apply_codec(
|
||||
self,
|
||||
codec: MsgCodec,
|
||||
) -> None:
|
||||
'''
|
||||
Temporarily override the underlying IPC msg codec for
|
||||
dynamic enforcement of messaging schema.
|
||||
|
||||
'''
|
||||
orig: MsgCodec = self._transport.codec
|
||||
try:
|
||||
self._transport.codec = codec
|
||||
yield
|
||||
finally:
|
||||
self._transport.codec = orig
|
||||
|
||||
# TODO: do a .src/.dst: str for maddrs?
|
||||
def pformat(self) -> str:
|
||||
if not self._transport:
|
||||
return '<Channel with inactive transport?>'
|
||||
|
||||
tpt: MsgTransport = self._transport
|
||||
tpt_name: str = type(tpt).__name__
|
||||
tpt_status: str = (
|
||||
'connected' if self.connected()
|
||||
else 'closed'
|
||||
)
|
||||
return (
|
||||
f'<Channel(\n'
|
||||
f' |_status: {tpt_status!r}\n'
|
||||
f' _closed={self._closed}\n'
|
||||
f' _cancel_called={self._cancel_called}\n'
|
||||
f'\n'
|
||||
f' |_peer: {self.aid}\n'
|
||||
f'\n'
|
||||
f' |_msgstream: {tpt_name}\n'
|
||||
f' proto={tpt.laddr.proto_key!r}\n'
|
||||
f' layer={tpt.layer_key!r}\n'
|
||||
f' laddr={tpt.laddr}\n'
|
||||
f' raddr={tpt.raddr}\n'
|
||||
f' codec={tpt.codec_key!r}\n'
|
||||
f' stream={tpt.stream}\n'
|
||||
f' maddr={tpt.maddr!r}\n'
|
||||
f' drained={tpt.drained}\n'
|
||||
f' _send_lock={tpt._send_lock.statistics()}\n'
|
||||
f')>\n'
|
||||
)
|
||||
|
||||
# NOTE: making this return a value that can be passed to
|
||||
# `eval()` is entirely **optional** FYI!
|
||||
# https://docs.python.org/3/library/functions.html#repr
|
||||
# https://docs.python.org/3/reference/datamodel.html#object.__repr__
|
||||
#
|
||||
# Currently we target **readability** from a (console)
|
||||
# logging perspective over `eval()`-ability since we do NOT
|
||||
# target serializing non-struct instances!
|
||||
# def __repr__(self) -> str:
|
||||
__str__ = pformat
|
||||
__repr__ = pformat
|
||||
|
||||
@property
|
||||
def laddr(self) -> Address|None:
|
||||
return self._transport.laddr if self._transport else None
|
||||
|
||||
@property
|
||||
def raddr(self) -> Address|None:
|
||||
return self._transport.raddr if self._transport else None
|
||||
|
||||
# TODO: something like,
|
||||
# `pdbp.hideframe_on(errors=[MsgTypeError])`
|
||||
# instead of the `try/except` hack we have rn..
|
||||
# seems like a pretty useful thing to have in general
|
||||
# along with being able to filter certain stack frame(s / sets)
|
||||
# possibly based on the current log-level?
|
||||
async def send(
|
||||
self,
|
||||
payload: Any,
|
||||
|
||||
hide_tb: bool = False,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Send a coded msg-blob over the transport.
|
||||
|
||||
'''
|
||||
__tracebackhide__: bool = hide_tb
|
||||
try:
|
||||
log.transport(
|
||||
'=> send IPC msg:\n\n'
|
||||
f'{pformat(payload)}\n'
|
||||
)
|
||||
# assert self._transport # but why typing?
|
||||
await self._transport.send(
|
||||
payload,
|
||||
hide_tb=hide_tb,
|
||||
)
|
||||
except BaseException as _err:
|
||||
err = _err # bind for introspection
|
||||
if not isinstance(_err, MsgTypeError):
|
||||
# assert err
|
||||
__tracebackhide__: bool = False
|
||||
else:
|
||||
try:
|
||||
assert err.cid
|
||||
|
||||
except KeyError:
|
||||
raise err
|
||||
|
||||
raise
|
||||
|
||||
async def recv(self) -> Any:
|
||||
assert self._transport
|
||||
return await self._transport.recv()
|
||||
|
||||
# TODO: auto-reconnect features like 0mq/nanomsg?
|
||||
# -[ ] implement it manually with nods to SC prot
|
||||
# possibly on multiple transport backends?
|
||||
# -> seems like that might be re-inventing scalability
|
||||
# prots tho no?
|
||||
# try:
|
||||
# return await self._transport.recv()
|
||||
# except trio.BrokenResourceError:
|
||||
# if self._autorecon:
|
||||
# await self._reconnect()
|
||||
# return await self.recv()
|
||||
# raise
|
||||
|
||||
async def aclose(self) -> None:
|
||||
|
||||
log.transport(
|
||||
f'Closing channel to {self.aid} '
|
||||
f'{self.laddr} -> {self.raddr}'
|
||||
)
|
||||
assert self._transport
|
||||
await self._transport.stream.aclose()
|
||||
self._closed = True
|
||||
|
||||
async def __aenter__(self):
|
||||
await self.connect()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
await self.aclose(*args)
|
||||
|
||||
def __aiter__(self):
|
||||
return self._aiter_msgs
|
||||
|
||||
# ?TODO? run any reconnection sequence?
|
||||
# -[ ] prolly should be impl-ed as deco-API?
|
||||
#
|
||||
# async def _reconnect(self) -> None:
|
||||
# """Handle connection failures by polling until a reconnect can be
|
||||
# established.
|
||||
# """
|
||||
# down = False
|
||||
# while True:
|
||||
# try:
|
||||
# with trio.move_on_after(3) as cancel_scope:
|
||||
# await self.connect()
|
||||
# cancelled = cancel_scope.cancelled_caught
|
||||
# if cancelled:
|
||||
# log.transport(
|
||||
# "Reconnect timed out after 3 seconds, retrying...")
|
||||
# continue
|
||||
# else:
|
||||
# log.transport("Stream connection re-established!")
|
||||
|
||||
# # on_recon = self._recon_seq
|
||||
# # if on_recon:
|
||||
# # await on_recon(self)
|
||||
|
||||
# break
|
||||
# except (OSError, ConnectionRefusedError):
|
||||
# if not down:
|
||||
# down = True
|
||||
# log.transport(
|
||||
# f"Connection to {self.raddr} went down, waiting"
|
||||
# " for re-establishment")
|
||||
# await trio.sleep(1)
|
||||
|
||||
async def _iter_msgs(
|
||||
self
|
||||
) -> AsyncGenerator[Any, None]:
|
||||
'''
|
||||
Yield `MsgType` IPC msgs decoded and deliverd from
|
||||
an underlying `MsgTransport` protocol.
|
||||
|
||||
This is a streaming routine alo implemented as an async-gen
|
||||
func (same a `MsgTransport._iter_pkts()`) gets allocated by
|
||||
a `.__call__()` inside `.__init__()` where it is assigned to
|
||||
the `._aiter_msgs` attr.
|
||||
|
||||
'''
|
||||
assert self._transport
|
||||
while True:
|
||||
try:
|
||||
async for msg in self._transport:
|
||||
match msg:
|
||||
# NOTE: if transport/interchange delivers
|
||||
# a type error, we pack it with the far
|
||||
# end peer `Actor.uid` and relay the
|
||||
# `Error`-msg upward to the `._rpc` stack
|
||||
# for normal RAE handling.
|
||||
case MsgTypeError():
|
||||
yield pack_from_raise(
|
||||
local_err=msg,
|
||||
cid=msg.cid,
|
||||
|
||||
# XXX we pack it here bc lower
|
||||
# layers have no notion of an
|
||||
# actor-id ;)
|
||||
src_uid=self.uid,
|
||||
)
|
||||
case _:
|
||||
yield msg
|
||||
|
||||
except trio.BrokenResourceError:
|
||||
|
||||
# if not self._autorecon:
|
||||
raise
|
||||
|
||||
await self.aclose()
|
||||
|
||||
# if self._autorecon: # attempt reconnect
|
||||
# await self._reconnect()
|
||||
# continue
|
||||
|
||||
def connected(self) -> bool:
|
||||
return self._transport.connected() if self._transport else False
|
||||
|
||||
async def _do_handshake(
|
||||
self,
|
||||
aid: Aid,
|
||||
|
||||
) -> Aid:
|
||||
'''
|
||||
Exchange `(name, UUIDs)` identifiers as the first
|
||||
communication step with any (peer) remote `Actor`.
|
||||
|
||||
These are essentially the "mailbox addresses" found in
|
||||
"actor model" parlance.
|
||||
|
||||
'''
|
||||
await self.send(aid)
|
||||
peer_aid: Aid = await self.recv()
|
||||
log.runtime(
|
||||
f'Received hanshake with peer actor,\n'
|
||||
f'{peer_aid}\n'
|
||||
)
|
||||
# NOTE, we always are referencing the remote peer!
|
||||
self.aid = peer_aid
|
||||
return peer_aid
|
||||
|
||||
|
||||
@acm
|
||||
async def _connect_chan(
|
||||
addr: UnwrappedAddress
|
||||
) -> typing.AsyncGenerator[Channel, None]:
|
||||
'''
|
||||
Create and connect a channel with disconnect on context manager
|
||||
teardown.
|
||||
|
||||
'''
|
||||
chan = await Channel.from_addr(addr)
|
||||
yield chan
|
||||
with trio.CancelScope(shield=True):
|
||||
await chan.aclose()
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
Linux specifics, for now we are only exposing EventFD
|
||||
|
||||
'''
|
||||
import os
|
||||
import errno
|
||||
|
||||
import cffi
|
||||
import trio
|
||||
|
||||
ffi = cffi.FFI()
|
||||
|
||||
# Declare the C functions and types we plan to use.
|
||||
# - eventfd: for creating the event file descriptor
|
||||
# - write: for writing to the file descriptor
|
||||
# - read: for reading from the file descriptor
|
||||
# - close: for closing the file descriptor
|
||||
ffi.cdef(
|
||||
'''
|
||||
int eventfd(unsigned int initval, int flags);
|
||||
|
||||
ssize_t write(int fd, const void *buf, size_t count);
|
||||
ssize_t read(int fd, void *buf, size_t count);
|
||||
|
||||
int close(int fd);
|
||||
'''
|
||||
)
|
||||
|
||||
|
||||
# Open the default dynamic library (essentially 'libc' in most cases)
|
||||
C = ffi.dlopen(None)
|
||||
|
||||
|
||||
# Constants from <sys/eventfd.h>, if needed.
|
||||
EFD_SEMAPHORE = 1
|
||||
EFD_CLOEXEC = 0o2000000
|
||||
EFD_NONBLOCK = 0o4000
|
||||
|
||||
|
||||
def open_eventfd(initval: int = 0, flags: int = 0) -> int:
|
||||
'''
|
||||
Open an eventfd with the given initial value and flags.
|
||||
Returns the file descriptor on success, otherwise raises OSError.
|
||||
|
||||
'''
|
||||
fd = C.eventfd(initval, flags)
|
||||
if fd < 0:
|
||||
raise OSError(errno.errorcode[ffi.errno], 'eventfd failed')
|
||||
return fd
|
||||
|
||||
|
||||
def write_eventfd(fd: int, value: int) -> int:
|
||||
'''
|
||||
Write a 64-bit integer (uint64_t) to the eventfd's counter.
|
||||
|
||||
'''
|
||||
# Create a uint64_t* in C, store `value`
|
||||
data_ptr = ffi.new('uint64_t *', value)
|
||||
|
||||
# Call write(fd, data_ptr, 8)
|
||||
# We expect to write exactly 8 bytes (sizeof(uint64_t))
|
||||
ret = C.write(fd, data_ptr, 8)
|
||||
if ret < 0:
|
||||
raise OSError(errno.errorcode[ffi.errno], 'write to eventfd failed')
|
||||
return ret
|
||||
|
||||
|
||||
def read_eventfd(fd: int) -> int:
|
||||
'''
|
||||
Read a 64-bit integer (uint64_t) from the eventfd, returning the value.
|
||||
Reading resets the counter to 0 (unless using EFD_SEMAPHORE).
|
||||
|
||||
'''
|
||||
# Allocate an 8-byte buffer in C for reading
|
||||
buf = ffi.new('char[]', 8)
|
||||
|
||||
ret = C.read(fd, buf, 8)
|
||||
if ret < 0:
|
||||
raise OSError(errno.errorcode[ffi.errno], 'read from eventfd failed')
|
||||
# Convert the 8 bytes we read into a Python integer
|
||||
data_bytes = ffi.unpack(buf, 8) # returns a Python bytes object of length 8
|
||||
value = int.from_bytes(data_bytes, byteorder='little', signed=False)
|
||||
return value
|
||||
|
||||
|
||||
def close_eventfd(fd: int) -> int:
|
||||
'''
|
||||
Close the eventfd.
|
||||
|
||||
'''
|
||||
ret = C.close(fd)
|
||||
if ret < 0:
|
||||
raise OSError(errno.errorcode[ffi.errno], 'close failed')
|
||||
|
||||
|
||||
class EventFD:
|
||||
'''
|
||||
Use a previously opened eventfd(2), meant to be used in
|
||||
sub-actors after root actor opens the eventfds then passes
|
||||
them through pass_fds
|
||||
|
||||
'''
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fd: int,
|
||||
omode: str
|
||||
):
|
||||
self._fd: int = fd
|
||||
self._omode: str = omode
|
||||
self._fobj = None
|
||||
|
||||
@property
|
||||
def fd(self) -> int | None:
|
||||
return self._fd
|
||||
|
||||
def write(self, value: int) -> int:
|
||||
return write_eventfd(self._fd, value)
|
||||
|
||||
async def read(self) -> int:
|
||||
return await trio.to_thread.run_sync(
|
||||
read_eventfd, self._fd,
|
||||
abandon_on_cancel=True
|
||||
)
|
||||
|
||||
def open(self):
|
||||
self._fobj = os.fdopen(self._fd, self._omode)
|
||||
|
||||
def close(self):
|
||||
if self._fobj:
|
||||
self._fobj.close()
|
||||
|
||||
def __enter__(self):
|
||||
self.open()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.close()
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
Utils to tame mp non-SC madeness
|
||||
|
||||
'''
|
||||
def disable_mantracker():
|
||||
'''
|
||||
Disable all ``multiprocessing``` "resource tracking" machinery since
|
||||
it's an absolute multi-threaded mess of non-SC madness.
|
||||
|
||||
'''
|
||||
from multiprocessing import resource_tracker as mantracker
|
||||
|
||||
# Tell the "resource tracker" thing to fuck off.
|
||||
class ManTracker(mantracker.ResourceTracker):
|
||||
def register(self, name, rtype):
|
||||
pass
|
||||
|
||||
def unregister(self, name, rtype):
|
||||
pass
|
||||
|
||||
def ensure_running(self):
|
||||
pass
|
||||
|
||||
# "know your land and know your prey"
|
||||
# https://www.dailymotion.com/video/x6ozzco
|
||||
mantracker._resource_tracker = ManTracker()
|
||||
mantracker.register = mantracker._resource_tracker.register
|
||||
mantracker.ensure_running = mantracker._resource_tracker.ensure_running
|
||||
mantracker.unregister = mantracker._resource_tracker.unregister
|
||||
mantracker.getfd = mantracker._resource_tracker.getfd
|
||||
|
|
@ -1,253 +0,0 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
IPC Reliable RingBuffer implementation
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from contextlib import contextmanager as cm
|
||||
from multiprocessing.shared_memory import SharedMemory
|
||||
|
||||
import trio
|
||||
from msgspec import (
|
||||
Struct,
|
||||
to_builtins
|
||||
)
|
||||
|
||||
from ._linux import (
|
||||
EFD_NONBLOCK,
|
||||
open_eventfd,
|
||||
EventFD
|
||||
)
|
||||
from ._mp_bs import disable_mantracker
|
||||
|
||||
|
||||
disable_mantracker()
|
||||
|
||||
|
||||
class RBToken(Struct, frozen=True):
|
||||
'''
|
||||
RingBuffer token contains necesary info to open the two
|
||||
eventfds and the shared memory
|
||||
|
||||
'''
|
||||
shm_name: str
|
||||
write_eventfd: int
|
||||
wrap_eventfd: int
|
||||
buf_size: int
|
||||
|
||||
def as_msg(self):
|
||||
return to_builtins(self)
|
||||
|
||||
@classmethod
|
||||
def from_msg(cls, msg: dict) -> RBToken:
|
||||
if isinstance(msg, RBToken):
|
||||
return msg
|
||||
|
||||
return RBToken(**msg)
|
||||
|
||||
|
||||
@cm
|
||||
def open_ringbuf(
|
||||
shm_name: str,
|
||||
buf_size: int = 10 * 1024,
|
||||
write_efd_flags: int = 0,
|
||||
wrap_efd_flags: int = 0
|
||||
) -> RBToken:
|
||||
shm = SharedMemory(
|
||||
name=shm_name,
|
||||
size=buf_size,
|
||||
create=True
|
||||
)
|
||||
try:
|
||||
token = RBToken(
|
||||
shm_name=shm_name,
|
||||
write_eventfd=open_eventfd(flags=write_efd_flags),
|
||||
wrap_eventfd=open_eventfd(flags=wrap_efd_flags),
|
||||
buf_size=buf_size
|
||||
)
|
||||
yield token
|
||||
|
||||
finally:
|
||||
shm.unlink()
|
||||
|
||||
|
||||
class RingBuffSender(trio.abc.SendStream):
|
||||
'''
|
||||
IPC Reliable Ring Buffer sender side implementation
|
||||
|
||||
`eventfd(2)` is used for wrap around sync, and also to signal
|
||||
writes to the reader.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
token: RBToken,
|
||||
start_ptr: int = 0,
|
||||
):
|
||||
token = RBToken.from_msg(token)
|
||||
self._shm = SharedMemory(
|
||||
name=token.shm_name,
|
||||
size=token.buf_size,
|
||||
create=False
|
||||
)
|
||||
self._write_event = EventFD(token.write_eventfd, 'w')
|
||||
self._wrap_event = EventFD(token.wrap_eventfd, 'r')
|
||||
self._ptr = start_ptr
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return self._shm.name
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self._shm.size
|
||||
|
||||
@property
|
||||
def ptr(self) -> int:
|
||||
return self._ptr
|
||||
|
||||
@property
|
||||
def write_fd(self) -> int:
|
||||
return self._write_event.fd
|
||||
|
||||
@property
|
||||
def wrap_fd(self) -> int:
|
||||
return self._wrap_event.fd
|
||||
|
||||
async def send_all(self, data: bytes | bytearray | memoryview):
|
||||
# while data is larger than the remaining buf
|
||||
target_ptr = self.ptr + len(data)
|
||||
while target_ptr > self.size:
|
||||
# write all bytes that fit
|
||||
remaining = self.size - self.ptr
|
||||
self._shm.buf[self.ptr:] = data[:remaining]
|
||||
# signal write and wait for reader wrap around
|
||||
self._write_event.write(remaining)
|
||||
await self._wrap_event.read()
|
||||
|
||||
# wrap around and trim already written bytes
|
||||
self._ptr = 0
|
||||
data = data[remaining:]
|
||||
target_ptr = self._ptr + len(data)
|
||||
|
||||
# remaining data fits on buffer
|
||||
self._shm.buf[self.ptr:target_ptr] = data
|
||||
self._write_event.write(len(data))
|
||||
self._ptr = target_ptr
|
||||
|
||||
async def wait_send_all_might_not_block(self):
|
||||
raise NotImplementedError
|
||||
|
||||
async def aclose(self):
|
||||
self._write_event.close()
|
||||
self._wrap_event.close()
|
||||
self._shm.close()
|
||||
|
||||
async def __aenter__(self):
|
||||
self._write_event.open()
|
||||
self._wrap_event.open()
|
||||
return self
|
||||
|
||||
|
||||
class RingBuffReceiver(trio.abc.ReceiveStream):
|
||||
'''
|
||||
IPC Reliable Ring Buffer receiver side implementation
|
||||
|
||||
`eventfd(2)` is used for wrap around sync, and also to signal
|
||||
writes to the reader.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
token: RBToken,
|
||||
start_ptr: int = 0,
|
||||
flags: int = 0
|
||||
):
|
||||
token = RBToken.from_msg(token)
|
||||
self._shm = SharedMemory(
|
||||
name=token.shm_name,
|
||||
size=token.buf_size,
|
||||
create=False
|
||||
)
|
||||
self._write_event = EventFD(token.write_eventfd, 'w')
|
||||
self._wrap_event = EventFD(token.wrap_eventfd, 'r')
|
||||
self._ptr = start_ptr
|
||||
self._flags = flags
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return self._shm.name
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self._shm.size
|
||||
|
||||
@property
|
||||
def ptr(self) -> int:
|
||||
return self._ptr
|
||||
|
||||
@property
|
||||
def write_fd(self) -> int:
|
||||
return self._write_event.fd
|
||||
|
||||
@property
|
||||
def wrap_fd(self) -> int:
|
||||
return self._wrap_event.fd
|
||||
|
||||
async def receive_some(
|
||||
self,
|
||||
max_bytes: int | None = None,
|
||||
nb_timeout: float = 0.1
|
||||
) -> memoryview:
|
||||
# if non blocking eventfd enabled, do polling
|
||||
# until next write, this allows signal handling
|
||||
if self._flags | EFD_NONBLOCK:
|
||||
delta = None
|
||||
while delta is None:
|
||||
try:
|
||||
delta = await self._write_event.read()
|
||||
|
||||
except OSError as e:
|
||||
if e.errno == 'EAGAIN':
|
||||
continue
|
||||
|
||||
raise e
|
||||
|
||||
else:
|
||||
delta = await self._write_event.read()
|
||||
|
||||
# fetch next segment and advance ptr
|
||||
next_ptr = self._ptr + delta
|
||||
segment = self._shm.buf[self._ptr:next_ptr]
|
||||
self._ptr = next_ptr
|
||||
|
||||
if self.ptr == self.size:
|
||||
# reached the end, signal wrap around
|
||||
self._ptr = 0
|
||||
self._wrap_event.write(1)
|
||||
|
||||
return segment
|
||||
|
||||
async def aclose(self):
|
||||
self._write_event.close()
|
||||
self._wrap_event.close()
|
||||
self._shm.close()
|
||||
|
||||
async def __aenter__(self):
|
||||
self._write_event.open()
|
||||
self._wrap_event.open()
|
||||
return self
|
||||
|
|
@ -1,812 +0,0 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
SC friendly shared memory management geared at real-time
|
||||
processing.
|
||||
|
||||
Support for ``numpy`` compatible array-buffers is provided but is
|
||||
considered optional within the context of this runtime-library.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from sys import byteorder
|
||||
import time
|
||||
from typing import Optional
|
||||
from multiprocessing import shared_memory as shm
|
||||
from multiprocessing.shared_memory import (
|
||||
SharedMemory,
|
||||
ShareableList,
|
||||
)
|
||||
|
||||
from msgspec import (
|
||||
Struct,
|
||||
to_builtins
|
||||
)
|
||||
import tractor
|
||||
|
||||
from tractor.ipc._mp_bs import disable_mantracker
|
||||
from tractor.log import get_logger
|
||||
|
||||
|
||||
_USE_POSIX = getattr(shm, '_USE_POSIX', False)
|
||||
if _USE_POSIX:
|
||||
from _posixshmem import shm_unlink
|
||||
|
||||
|
||||
try:
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
# TODO ruff complains with,
|
||||
# warning| F401: `nptyping` imported but unused; consider using
|
||||
# `importlib.util.find_spec` to test for availability
|
||||
import nptyping # noqa
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
disable_mantracker()
|
||||
|
||||
|
||||
class SharedInt:
|
||||
'''
|
||||
Wrapper around a single entry shared memory array which
|
||||
holds an ``int`` value used as an index counter.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
shm: SharedMemory,
|
||||
) -> None:
|
||||
self._shm = shm
|
||||
|
||||
@property
|
||||
def value(self) -> int:
|
||||
return int.from_bytes(self._shm.buf, byteorder)
|
||||
|
||||
@value.setter
|
||||
def value(self, value) -> None:
|
||||
self._shm.buf[:] = value.to_bytes(self._shm.size, byteorder)
|
||||
|
||||
def destroy(self) -> None:
|
||||
if _USE_POSIX:
|
||||
# We manually unlink to bypass all the "resource tracker"
|
||||
# nonsense meant for non-SC systems.
|
||||
name = self._shm.name
|
||||
try:
|
||||
shm_unlink(name)
|
||||
except FileNotFoundError:
|
||||
# might be a teardown race here?
|
||||
log.warning(f'Shm for {name} already unlinked?')
|
||||
|
||||
|
||||
class NDToken(Struct, frozen=True):
|
||||
'''
|
||||
Internal represenation of a shared memory ``numpy`` array "token"
|
||||
which can be used to key and load a system (OS) wide shm entry
|
||||
and correctly read the array by type signature.
|
||||
|
||||
This type is msg safe.
|
||||
|
||||
'''
|
||||
shm_name: str # this servers as a "key" value
|
||||
shm_first_index_name: str
|
||||
shm_last_index_name: str
|
||||
dtype_descr: tuple
|
||||
size: int # in struct-array index / row terms
|
||||
|
||||
# TODO: use nptyping here on dtypes
|
||||
@property
|
||||
def dtype(self) -> list[tuple[str, str, tuple[int, ...]]]:
|
||||
return np.dtype(
|
||||
list(
|
||||
map(tuple, self.dtype_descr)
|
||||
)
|
||||
).descr
|
||||
|
||||
def as_msg(self):
|
||||
return to_builtins(self)
|
||||
|
||||
@classmethod
|
||||
def from_msg(cls, msg: dict) -> NDToken:
|
||||
if isinstance(msg, NDToken):
|
||||
return msg
|
||||
|
||||
# TODO: native struct decoding
|
||||
# return _token_dec.decode(msg)
|
||||
|
||||
msg['dtype_descr'] = tuple(map(tuple, msg['dtype_descr']))
|
||||
return NDToken(**msg)
|
||||
|
||||
|
||||
# _token_dec = msgspec.msgpack.Decoder(NDToken)
|
||||
|
||||
# TODO: this api?
|
||||
# _known_tokens = tractor.ActorVar('_shm_tokens', {})
|
||||
# _known_tokens = tractor.ContextStack('_known_tokens', )
|
||||
# _known_tokens = trio.RunVar('shms', {})
|
||||
|
||||
# TODO: this should maybe be provided via
|
||||
# a `.trionics.maybe_open_context()` wrapper factory?
|
||||
# process-local store of keys to tokens
|
||||
_known_tokens: dict[str, NDToken] = {}
|
||||
|
||||
|
||||
def get_shm_token(key: str) -> NDToken | None:
|
||||
'''
|
||||
Convenience func to check if a token
|
||||
for the provided key is known by this process.
|
||||
|
||||
Returns either the ``numpy`` token or a string for a shared list.
|
||||
|
||||
'''
|
||||
return _known_tokens.get(key)
|
||||
|
||||
|
||||
def _make_token(
|
||||
key: str,
|
||||
size: int,
|
||||
dtype: np.dtype,
|
||||
|
||||
) -> NDToken:
|
||||
'''
|
||||
Create a serializable token that can be used
|
||||
to access a shared array.
|
||||
|
||||
'''
|
||||
return NDToken(
|
||||
shm_name=key,
|
||||
shm_first_index_name=key + "_first",
|
||||
shm_last_index_name=key + "_last",
|
||||
dtype_descr=tuple(np.dtype(dtype).descr),
|
||||
size=size,
|
||||
)
|
||||
|
||||
|
||||
class ShmArray:
|
||||
'''
|
||||
A shared memory ``numpy.ndarray`` API.
|
||||
|
||||
An underlying shared memory buffer is allocated based on
|
||||
a user specified ``numpy.ndarray``. This fixed size array
|
||||
can be read and written to by pushing data both onto the "front"
|
||||
or "back" of a set index range. The indexes for the "first" and
|
||||
"last" index are themselves stored in shared memory (accessed via
|
||||
``SharedInt`` interfaces) values such that multiple processes can
|
||||
interact with the same array using a synchronized-index.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
shmarr: np.ndarray,
|
||||
first: SharedInt,
|
||||
last: SharedInt,
|
||||
shm: SharedMemory,
|
||||
# readonly: bool = True,
|
||||
) -> None:
|
||||
self._array = shmarr
|
||||
|
||||
# indexes for first and last indices corresponding
|
||||
# to fille data
|
||||
self._first = first
|
||||
self._last = last
|
||||
|
||||
self._len = len(shmarr)
|
||||
self._shm = shm
|
||||
self._post_init: bool = False
|
||||
|
||||
# pushing data does not write the index (aka primary key)
|
||||
self._write_fields: list[str] | None = None
|
||||
dtype = shmarr.dtype
|
||||
if dtype.fields:
|
||||
self._write_fields = list(shmarr.dtype.fields.keys())[1:]
|
||||
|
||||
# TODO: ringbuf api?
|
||||
|
||||
@property
|
||||
def _token(self) -> NDToken:
|
||||
return NDToken(
|
||||
shm_name=self._shm.name,
|
||||
shm_first_index_name=self._first._shm.name,
|
||||
shm_last_index_name=self._last._shm.name,
|
||||
dtype_descr=tuple(self._array.dtype.descr),
|
||||
size=self._len,
|
||||
)
|
||||
|
||||
@property
|
||||
def token(self) -> dict:
|
||||
"""Shared memory token that can be serialized and used by
|
||||
another process to attach to this array.
|
||||
"""
|
||||
return self._token.as_msg()
|
||||
|
||||
@property
|
||||
def index(self) -> int:
|
||||
return self._last.value % self._len
|
||||
|
||||
@property
|
||||
def array(self) -> np.ndarray:
|
||||
'''
|
||||
Return an up-to-date ``np.ndarray`` view of the
|
||||
so-far-written data to the underlying shm buffer.
|
||||
|
||||
'''
|
||||
a = self._array[self._first.value:self._last.value]
|
||||
|
||||
# first, last = self._first.value, self._last.value
|
||||
# a = self._array[first:last]
|
||||
|
||||
# TODO: eventually comment this once we've not seen it in the
|
||||
# wild in a long time..
|
||||
# XXX: race where first/last indexes cause a reader
|
||||
# to load an empty array..
|
||||
if len(a) == 0 and self._post_init:
|
||||
raise RuntimeError('Empty array race condition hit!?')
|
||||
# breakpoint()
|
||||
|
||||
return a
|
||||
|
||||
def ustruct(
|
||||
self,
|
||||
fields: Optional[list[str]] = None,
|
||||
|
||||
# type that all field values will be cast to
|
||||
# in the returned view.
|
||||
common_dtype: np.dtype = float,
|
||||
|
||||
) -> np.ndarray:
|
||||
|
||||
array = self._array
|
||||
|
||||
if fields:
|
||||
selection = array[fields]
|
||||
# fcount = len(fields)
|
||||
else:
|
||||
selection = array
|
||||
# fcount = len(array.dtype.fields)
|
||||
|
||||
# XXX: manual ``.view()`` attempt that also doesn't work.
|
||||
# uview = selection.view(
|
||||
# dtype='<f16',
|
||||
# ).reshape(-1, 4, order='A')
|
||||
|
||||
# assert len(selection) == len(uview)
|
||||
|
||||
u = rfn.structured_to_unstructured(
|
||||
selection,
|
||||
# dtype=float,
|
||||
copy=True,
|
||||
)
|
||||
|
||||
# unstruct = np.ndarray(u.shape, dtype=a.dtype, buffer=shm.buf)
|
||||
# array[:] = a[:]
|
||||
return u
|
||||
# return ShmArray(
|
||||
# shmarr=u,
|
||||
# first=self._first,
|
||||
# last=self._last,
|
||||
# shm=self._shm
|
||||
# )
|
||||
|
||||
def last(
|
||||
self,
|
||||
length: int = 1,
|
||||
|
||||
) -> np.ndarray:
|
||||
'''
|
||||
Return the last ``length``'s worth of ("row") entries from the
|
||||
array.
|
||||
|
||||
'''
|
||||
return self.array[-length:]
|
||||
|
||||
def push(
|
||||
self,
|
||||
data: np.ndarray,
|
||||
|
||||
field_map: Optional[dict[str, str]] = None,
|
||||
prepend: bool = False,
|
||||
update_first: bool = True,
|
||||
start: int | None = None,
|
||||
|
||||
) -> int:
|
||||
'''
|
||||
Ring buffer like "push" to append data
|
||||
into the buffer and return updated "last" index.
|
||||
|
||||
NB: no actual ring logic yet to give a "loop around" on overflow
|
||||
condition, lel.
|
||||
|
||||
'''
|
||||
length = len(data)
|
||||
|
||||
if prepend:
|
||||
index = (start or self._first.value) - length
|
||||
|
||||
if index < 0:
|
||||
raise ValueError(
|
||||
f'Array size of {self._len} was overrun during prepend.\n'
|
||||
f'You have passed {abs(index)} too many datums.'
|
||||
)
|
||||
|
||||
else:
|
||||
index = start if start is not None else self._last.value
|
||||
|
||||
end = index + length
|
||||
|
||||
if field_map:
|
||||
src_names, dst_names = zip(*field_map.items())
|
||||
else:
|
||||
dst_names = src_names = self._write_fields
|
||||
|
||||
try:
|
||||
self._array[
|
||||
list(dst_names)
|
||||
][index:end] = data[list(src_names)][:]
|
||||
|
||||
# NOTE: there was a race here between updating
|
||||
# the first and last indices and when the next reader
|
||||
# tries to access ``.array`` (which due to the index
|
||||
# overlap will be empty). Pretty sure we've fixed it now
|
||||
# but leaving this here as a reminder.
|
||||
if (
|
||||
prepend
|
||||
and update_first
|
||||
and length
|
||||
):
|
||||
assert index < self._first.value
|
||||
|
||||
if (
|
||||
index < self._first.value
|
||||
and update_first
|
||||
):
|
||||
assert prepend, 'prepend=True not passed but index decreased?'
|
||||
self._first.value = index
|
||||
|
||||
elif not prepend:
|
||||
self._last.value = end
|
||||
|
||||
self._post_init = True
|
||||
return end
|
||||
|
||||
except ValueError as err:
|
||||
if field_map:
|
||||
raise
|
||||
|
||||
# should raise if diff detected
|
||||
self.diff_err_fields(data)
|
||||
raise err
|
||||
|
||||
def diff_err_fields(
|
||||
self,
|
||||
data: np.ndarray,
|
||||
) -> None:
|
||||
# reraise with any field discrepancy
|
||||
our_fields, their_fields = (
|
||||
set(self._array.dtype.fields),
|
||||
set(data.dtype.fields),
|
||||
)
|
||||
|
||||
only_in_ours = our_fields - their_fields
|
||||
only_in_theirs = their_fields - our_fields
|
||||
|
||||
if only_in_ours:
|
||||
raise TypeError(
|
||||
f"Input array is missing field(s): {only_in_ours}"
|
||||
)
|
||||
elif only_in_theirs:
|
||||
raise TypeError(
|
||||
f"Input array has unknown field(s): {only_in_theirs}"
|
||||
)
|
||||
|
||||
# TODO: support "silent" prepends that don't update ._first.value?
|
||||
def prepend(
|
||||
self,
|
||||
data: np.ndarray,
|
||||
) -> int:
|
||||
end = self.push(data, prepend=True)
|
||||
assert end
|
||||
|
||||
def close(self) -> None:
|
||||
self._first._shm.close()
|
||||
self._last._shm.close()
|
||||
self._shm.close()
|
||||
|
||||
def destroy(self) -> None:
|
||||
if _USE_POSIX:
|
||||
# We manually unlink to bypass all the "resource tracker"
|
||||
# nonsense meant for non-SC systems.
|
||||
shm_unlink(self._shm.name)
|
||||
|
||||
self._first.destroy()
|
||||
self._last.destroy()
|
||||
|
||||
def flush(self) -> None:
|
||||
# TODO: flush to storage backend like markestore?
|
||||
...
|
||||
|
||||
|
||||
def open_shm_ndarray(
|
||||
size: int,
|
||||
key: str | None = None,
|
||||
dtype: np.dtype | None = None,
|
||||
append_start_index: int | None = None,
|
||||
readonly: bool = False,
|
||||
|
||||
) -> ShmArray:
|
||||
'''
|
||||
Open a memory shared ``numpy`` using the standard library.
|
||||
|
||||
This call unlinks (aka permanently destroys) the buffer on teardown
|
||||
and thus should be used from the parent-most accessor (process).
|
||||
|
||||
'''
|
||||
# create new shared mem segment for which we
|
||||
# have write permission
|
||||
a = np.zeros(size, dtype=dtype)
|
||||
a['index'] = np.arange(len(a))
|
||||
|
||||
shm = SharedMemory(
|
||||
name=key,
|
||||
create=True,
|
||||
size=a.nbytes
|
||||
)
|
||||
array = np.ndarray(
|
||||
a.shape,
|
||||
dtype=a.dtype,
|
||||
buffer=shm.buf
|
||||
)
|
||||
array[:] = a[:]
|
||||
array.setflags(write=int(not readonly))
|
||||
|
||||
token = _make_token(
|
||||
key=key,
|
||||
size=size,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
# create single entry arrays for storing an first and last indices
|
||||
first = SharedInt(
|
||||
shm=SharedMemory(
|
||||
name=token.shm_first_index_name,
|
||||
create=True,
|
||||
size=4, # std int
|
||||
)
|
||||
)
|
||||
|
||||
last = SharedInt(
|
||||
shm=SharedMemory(
|
||||
name=token.shm_last_index_name,
|
||||
create=True,
|
||||
size=4, # std int
|
||||
)
|
||||
)
|
||||
|
||||
# Start the "real-time" append-updated (or "pushed-to") section
|
||||
# after some start index: ``append_start_index``. This allows appending
|
||||
# from a start point in the array which isn't the 0 index and looks
|
||||
# something like,
|
||||
# -------------------------
|
||||
# | | i
|
||||
# _________________________
|
||||
# <-------------> <------->
|
||||
# history real-time
|
||||
#
|
||||
# Once fully "prepended", the history section will leave the
|
||||
# ``ShmArray._start.value: int = 0`` and the yet-to-be written
|
||||
# real-time section will start at ``ShmArray.index: int``.
|
||||
|
||||
# this sets the index to nearly 2/3rds into the the length of
|
||||
# the buffer leaving at least a "days worth of second samples"
|
||||
# for the real-time section.
|
||||
if append_start_index is None:
|
||||
append_start_index = round(size * 0.616)
|
||||
|
||||
last.value = first.value = append_start_index
|
||||
|
||||
shmarr = ShmArray(
|
||||
array,
|
||||
first,
|
||||
last,
|
||||
shm,
|
||||
)
|
||||
|
||||
assert shmarr._token == token
|
||||
_known_tokens[key] = shmarr.token
|
||||
|
||||
# "unlink" created shm on process teardown by
|
||||
# pushing teardown calls onto actor context stack
|
||||
stack = tractor.current_actor().lifetime_stack
|
||||
stack.callback(shmarr.close)
|
||||
stack.callback(shmarr.destroy)
|
||||
|
||||
return shmarr
|
||||
|
||||
|
||||
def attach_shm_ndarray(
|
||||
token: tuple[str, str, tuple[str, str]],
|
||||
readonly: bool = True,
|
||||
|
||||
) -> ShmArray:
|
||||
'''
|
||||
Attach to an existing shared memory array previously
|
||||
created by another process using ``open_shared_array``.
|
||||
|
||||
No new shared mem is allocated but wrapper types for read/write
|
||||
access are constructed.
|
||||
|
||||
'''
|
||||
token = NDToken.from_msg(token)
|
||||
key = token.shm_name
|
||||
|
||||
if key in _known_tokens:
|
||||
assert NDToken.from_msg(_known_tokens[key]) == token, "WTF"
|
||||
|
||||
# XXX: ugh, looks like due to the ``shm_open()`` C api we can't
|
||||
# actually place files in a subdir, see discussion here:
|
||||
# https://stackoverflow.com/a/11103289
|
||||
|
||||
# attach to array buffer and view as per dtype
|
||||
_err: Optional[Exception] = None
|
||||
for _ in range(3):
|
||||
try:
|
||||
shm = SharedMemory(
|
||||
name=key,
|
||||
create=False,
|
||||
)
|
||||
break
|
||||
except OSError as oserr:
|
||||
_err = oserr
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
if _err:
|
||||
raise _err
|
||||
|
||||
shmarr = np.ndarray(
|
||||
(token.size,),
|
||||
dtype=token.dtype,
|
||||
buffer=shm.buf
|
||||
)
|
||||
shmarr.setflags(write=int(not readonly))
|
||||
|
||||
first = SharedInt(
|
||||
shm=SharedMemory(
|
||||
name=token.shm_first_index_name,
|
||||
create=False,
|
||||
size=4, # std int
|
||||
),
|
||||
)
|
||||
last = SharedInt(
|
||||
shm=SharedMemory(
|
||||
name=token.shm_last_index_name,
|
||||
create=False,
|
||||
size=4, # std int
|
||||
),
|
||||
)
|
||||
|
||||
# make sure we can read
|
||||
first.value
|
||||
|
||||
sha = ShmArray(
|
||||
shmarr,
|
||||
first,
|
||||
last,
|
||||
shm,
|
||||
)
|
||||
# read test
|
||||
sha.array
|
||||
|
||||
# Stash key -> token knowledge for future queries
|
||||
# via `maybe_opepn_shm_array()` but only after we know
|
||||
# we can attach.
|
||||
if key not in _known_tokens:
|
||||
_known_tokens[key] = token
|
||||
|
||||
# "close" attached shm on actor teardown
|
||||
tractor.current_actor().lifetime_stack.callback(sha.close)
|
||||
|
||||
return sha
|
||||
|
||||
|
||||
def maybe_open_shm_ndarray(
|
||||
key: str, # unique identifier for segment
|
||||
size: int,
|
||||
dtype: np.dtype | None = None,
|
||||
append_start_index: int = 0,
|
||||
readonly: bool = True,
|
||||
|
||||
) -> tuple[ShmArray, bool]:
|
||||
'''
|
||||
Attempt to attach to a shared memory block using a "key" lookup
|
||||
to registered blocks in the users overall "system" registry
|
||||
(presumes you don't have the block's explicit token).
|
||||
|
||||
This function is meant to solve the problem of discovering whether
|
||||
a shared array token has been allocated or discovered by the actor
|
||||
running in **this** process. Systems where multiple actors may seek
|
||||
to access a common block can use this function to attempt to acquire
|
||||
a token as discovered by the actors who have previously stored
|
||||
a "key" -> ``NDToken`` map in an actor local (aka python global)
|
||||
variable.
|
||||
|
||||
If you know the explicit ``NDToken`` for your memory segment instead
|
||||
use ``attach_shm_array``.
|
||||
|
||||
'''
|
||||
try:
|
||||
# see if we already know this key
|
||||
token = _known_tokens[key]
|
||||
return (
|
||||
attach_shm_ndarray(
|
||||
token=token,
|
||||
readonly=readonly,
|
||||
),
|
||||
False, # not newly opened
|
||||
)
|
||||
except KeyError:
|
||||
log.warning(f"Could not find {key} in shms cache")
|
||||
if dtype:
|
||||
token = _make_token(
|
||||
key,
|
||||
size=size,
|
||||
dtype=dtype,
|
||||
)
|
||||
else:
|
||||
|
||||
try:
|
||||
return (
|
||||
attach_shm_ndarray(
|
||||
token=token,
|
||||
readonly=readonly,
|
||||
),
|
||||
False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
log.warning(f"Could not attach to shm with token {token}")
|
||||
|
||||
# This actor does not know about memory
|
||||
# associated with the provided "key".
|
||||
# Attempt to open a block and expect
|
||||
# to fail if a block has been allocated
|
||||
# on the OS by someone else.
|
||||
return (
|
||||
open_shm_ndarray(
|
||||
key=key,
|
||||
size=size,
|
||||
dtype=dtype,
|
||||
append_start_index=append_start_index,
|
||||
readonly=readonly,
|
||||
),
|
||||
True,
|
||||
)
|
||||
|
||||
|
||||
class ShmList(ShareableList):
|
||||
'''
|
||||
Carbon copy of ``.shared_memory.ShareableList`` with a few
|
||||
enhancements:
|
||||
|
||||
- readonly mode via instance var flag `._readonly: bool`
|
||||
- ``.__getitem__()`` accepts ``slice`` inputs
|
||||
- exposes the underlying buffer "name" as a ``.key: str``
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
sequence: list | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
readonly: bool = True
|
||||
|
||||
) -> None:
|
||||
self._readonly = readonly
|
||||
self._key = name
|
||||
return super().__init__(
|
||||
sequence=sequence,
|
||||
name=name,
|
||||
)
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return self._key
|
||||
|
||||
@property
|
||||
def readonly(self) -> bool:
|
||||
return self._readonly
|
||||
|
||||
def __setitem__(
|
||||
self,
|
||||
position,
|
||||
value,
|
||||
|
||||
) -> None:
|
||||
|
||||
# mimick ``numpy`` error
|
||||
if self._readonly:
|
||||
raise ValueError('assignment destination is read-only')
|
||||
|
||||
return super().__setitem__(position, value)
|
||||
|
||||
def __getitem__(
|
||||
self,
|
||||
indexish,
|
||||
) -> list:
|
||||
|
||||
# NOTE: this is a non-writeable view (copy?) of the buffer
|
||||
# in a new list instance.
|
||||
if isinstance(indexish, slice):
|
||||
return list(self)[indexish]
|
||||
|
||||
return super().__getitem__(indexish)
|
||||
|
||||
# TODO: should we offer a `.array` and `.push()` equivalent
|
||||
# to the `ShmArray`?
|
||||
# currently we have the following limitations:
|
||||
# - can't write slices of input using traditional slice-assign
|
||||
# syntax due to the ``ShareableList.__setitem__()`` implementation.
|
||||
# - ``list(shmlist)`` returns a non-mutable copy instead of
|
||||
# a writeable view which would be handier numpy-style ops.
|
||||
|
||||
|
||||
def open_shm_list(
|
||||
key: str,
|
||||
sequence: list | None = None,
|
||||
size: int = int(2 ** 10),
|
||||
dtype: float | int | bool | str | bytes | None = float,
|
||||
readonly: bool = True,
|
||||
|
||||
) -> ShmList:
|
||||
|
||||
if sequence is None:
|
||||
default = {
|
||||
float: 0.,
|
||||
int: 0,
|
||||
bool: True,
|
||||
str: 'doggy',
|
||||
None: None,
|
||||
}[dtype]
|
||||
sequence = [default] * size
|
||||
|
||||
shml = ShmList(
|
||||
sequence=sequence,
|
||||
name=key,
|
||||
readonly=readonly,
|
||||
)
|
||||
|
||||
# "close" attached shm on actor teardown
|
||||
try:
|
||||
actor = tractor.current_actor()
|
||||
actor.lifetime_stack.callback(shml.shm.close)
|
||||
actor.lifetime_stack.callback(shml.shm.unlink)
|
||||
except RuntimeError:
|
||||
log.warning('tractor runtime not active, skipping teardown steps')
|
||||
|
||||
return shml
|
||||
|
||||
|
||||
def attach_shm_list(
|
||||
key: str,
|
||||
readonly: bool = False,
|
||||
|
||||
) -> ShmList:
|
||||
|
||||
return ShmList(
|
||||
name=key,
|
||||
readonly=readonly,
|
||||
)
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
TCP implementation of tractor.ipc._transport.MsgTransport protocol
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
|
||||
import trio
|
||||
|
||||
from tractor.msg import MsgCodec
|
||||
from tractor.log import get_logger
|
||||
from tractor._addr import TCPAddress
|
||||
from tractor.ipc._transport import MsgpackTransport
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# TODO: typing oddity.. not sure why we have to inherit here, but it
|
||||
# seems to be an issue with `get_msg_transport()` returning
|
||||
# a `Type[Protocol]`; probably should make a `mypy` issue?
|
||||
class MsgpackTCPStream(MsgpackTransport):
|
||||
'''
|
||||
A ``trio.SocketStream`` delivering ``msgpack`` formatted data
|
||||
using the ``msgspec`` codec lib.
|
||||
|
||||
'''
|
||||
address_type = TCPAddress
|
||||
layer_key: int = 4
|
||||
|
||||
@property
|
||||
def maddr(self) -> str:
|
||||
host, port = self.raddr.unwrap()
|
||||
return (
|
||||
# TODO, use `ipaddress` from stdlib to handle
|
||||
# first detecting which of `ipv4/6` before
|
||||
# choosing the routing prefix part.
|
||||
f'/ipv4/{host}'
|
||||
|
||||
f'/{self.address_type.proto_key}/{port}'
|
||||
# f'/{self.chan.uid[0]}'
|
||||
# f'/{self.cid}'
|
||||
|
||||
# f'/cid={cid_head}..{cid_tail}'
|
||||
# TODO: ? not use this ^ right ?
|
||||
)
|
||||
|
||||
def connected(self) -> bool:
|
||||
return self.stream.socket.fileno() != -1
|
||||
|
||||
@classmethod
|
||||
async def connect_to(
|
||||
cls,
|
||||
destaddr: TCPAddress,
|
||||
prefix_size: int = 4,
|
||||
codec: MsgCodec|None = None,
|
||||
**kwargs
|
||||
) -> MsgpackTCPStream:
|
||||
stream = await trio.open_tcp_stream(
|
||||
*destaddr.unwrap(),
|
||||
**kwargs
|
||||
)
|
||||
return MsgpackTCPStream(
|
||||
stream,
|
||||
prefix_size=prefix_size,
|
||||
codec=codec
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_stream_addrs(
|
||||
cls,
|
||||
stream: trio.SocketStream
|
||||
) -> tuple[
|
||||
TCPAddress,
|
||||
TCPAddress,
|
||||
]:
|
||||
# TODO, what types are these?
|
||||
lsockname = stream.socket.getsockname()
|
||||
l_sockaddr: tuple[str, int] = tuple(lsockname[:2])
|
||||
rsockname = stream.socket.getpeername()
|
||||
r_sockaddr: tuple[str, int] = tuple(rsockname[:2])
|
||||
return (
|
||||
TCPAddress.from_addr(l_sockaddr),
|
||||
TCPAddress.from_addr(r_sockaddr),
|
||||
)
|
||||
|
|
@ -1,504 +0,0 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
typing.Protocol based generic msg API, implement this class to add backends for
|
||||
tractor.ipc.Channel
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
runtime_checkable,
|
||||
Type,
|
||||
Protocol,
|
||||
TypeVar,
|
||||
ClassVar
|
||||
)
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
AsyncIterator,
|
||||
)
|
||||
import struct
|
||||
|
||||
import trio
|
||||
import msgspec
|
||||
from tricycle import BufferedReceiveStream
|
||||
|
||||
from tractor.log import get_logger
|
||||
from tractor._exceptions import (
|
||||
MsgTypeError,
|
||||
TransportClosed,
|
||||
_mk_send_mte,
|
||||
_mk_recv_mte,
|
||||
)
|
||||
from tractor.msg import (
|
||||
_ctxvar_MsgCodec,
|
||||
# _codec, XXX see `self._codec` sanity/debug checks
|
||||
MsgCodec,
|
||||
types as msgtypes,
|
||||
pretty_struct,
|
||||
)
|
||||
from tractor._addr import Address
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# (codec, transport)
|
||||
MsgTransportKey = tuple[str, str]
|
||||
|
||||
|
||||
# from tractor.msg.types import MsgType
|
||||
# ?TODO? this should be our `Union[*msgtypes.__spec__]` alias now right..?
|
||||
# => BLEH, except can't bc prots must inherit typevar or param-spec
|
||||
# vars..
|
||||
MsgType = TypeVar('MsgType')
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class MsgTransport(Protocol[MsgType]):
|
||||
#
|
||||
# ^-TODO-^ consider using a generic def and indexing with our
|
||||
# eventual msg definition/types?
|
||||
# - https://docs.python.org/3/library/typing.html#typing.Protocol
|
||||
|
||||
stream: trio.SocketStream
|
||||
drained: list[MsgType]
|
||||
|
||||
address_type: ClassVar[Type[Address]]
|
||||
codec_key: ClassVar[str]
|
||||
|
||||
# XXX: should this instead be called `.sendall()`?
|
||||
async def send(self, msg: MsgType) -> None:
|
||||
...
|
||||
|
||||
async def recv(self) -> MsgType:
|
||||
...
|
||||
|
||||
def __aiter__(self) -> MsgType:
|
||||
...
|
||||
|
||||
def connected(self) -> bool:
|
||||
...
|
||||
|
||||
# defining this sync otherwise it causes a mypy error because it
|
||||
# can't figure out it's a generator i guess?..?
|
||||
def drain(self) -> AsyncIterator[dict]:
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def key(cls) -> MsgTransportKey:
|
||||
return cls.codec_key, cls.address_type.proto_key
|
||||
|
||||
@property
|
||||
def laddr(self) -> Address:
|
||||
...
|
||||
|
||||
@property
|
||||
def raddr(self) -> Address:
|
||||
...
|
||||
|
||||
@property
|
||||
def maddr(self) -> str:
|
||||
...
|
||||
|
||||
@classmethod
|
||||
async def connect_to(
|
||||
cls,
|
||||
addr: Address,
|
||||
**kwargs
|
||||
) -> MsgTransport:
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_stream_addrs(
|
||||
cls,
|
||||
stream: trio.abc.Stream
|
||||
) -> tuple[
|
||||
Address, # local
|
||||
Address # remote
|
||||
]:
|
||||
'''
|
||||
Return the `trio` streaming transport prot's addrs for both
|
||||
the local and remote sides as a pair.
|
||||
|
||||
'''
|
||||
...
|
||||
|
||||
# TODO, such that all `.raddr`s for each `SocketStream` are
|
||||
# delivered?
|
||||
# -[ ] move `.open_listener()` here and internally track the
|
||||
# listener set, per address?
|
||||
# def get_peers(
|
||||
# self,
|
||||
# ) -> list[Address]:
|
||||
# ...
|
||||
|
||||
|
||||
|
||||
class MsgpackTransport(MsgTransport):
|
||||
|
||||
# TODO: better naming for this?
|
||||
# -[ ] check how libp2p does naming for such things?
|
||||
codec_key: str = 'msgpack'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
stream: trio.abc.Stream,
|
||||
prefix_size: int = 4,
|
||||
|
||||
# XXX optionally provided codec pair for `msgspec`:
|
||||
# https://jcristharif.com/msgspec/extending.html#mapping-to-from-native-types
|
||||
#
|
||||
# TODO: define this as a `Codec` struct which can be
|
||||
# overriden dynamically by the application/runtime?
|
||||
codec: MsgCodec = None,
|
||||
|
||||
) -> None:
|
||||
self.stream = stream
|
||||
(
|
||||
self._laddr,
|
||||
self._raddr,
|
||||
) = self.get_stream_addrs(stream)
|
||||
|
||||
# create read loop instance
|
||||
self._aiter_pkts = self._iter_packets()
|
||||
self._send_lock = trio.StrictFIFOLock()
|
||||
|
||||
# public i guess?
|
||||
self.drained: list[dict] = []
|
||||
|
||||
self.recv_stream = BufferedReceiveStream(
|
||||
transport_stream=stream
|
||||
)
|
||||
self.prefix_size = prefix_size
|
||||
|
||||
# allow for custom IPC msg interchange format
|
||||
# dynamic override Bo
|
||||
self._task = trio.lowlevel.current_task()
|
||||
|
||||
# XXX for ctxvar debug only!
|
||||
# self._codec: MsgCodec = (
|
||||
# codec
|
||||
# or
|
||||
# _codec._ctxvar_MsgCodec.get()
|
||||
# )
|
||||
|
||||
async def _iter_packets(self) -> AsyncGenerator[dict, None]:
|
||||
'''
|
||||
Yield `bytes`-blob decoded packets from the underlying TCP
|
||||
stream using the current task's `MsgCodec`.
|
||||
|
||||
This is a streaming routine implemented as an async generator
|
||||
func (which was the original design, but could be changed?)
|
||||
and is allocated by a `.__call__()` inside `.__init__()` where
|
||||
it is assigned to the `._aiter_pkts` attr.
|
||||
|
||||
'''
|
||||
decodes_failed: int = 0
|
||||
|
||||
tpt_name: str = f'{type(self).__name__!r}'
|
||||
while True:
|
||||
try:
|
||||
header: bytes = await self.recv_stream.receive_exactly(4)
|
||||
except (
|
||||
ValueError,
|
||||
ConnectionResetError,
|
||||
|
||||
# not sure entirely why we need this but without it we
|
||||
# seem to be getting racy failures here on
|
||||
# arbiter/registry name subs..
|
||||
trio.BrokenResourceError,
|
||||
|
||||
) as trans_err:
|
||||
|
||||
loglevel = 'transport'
|
||||
match trans_err:
|
||||
# case (
|
||||
# ConnectionResetError()
|
||||
# ):
|
||||
# loglevel = 'transport'
|
||||
|
||||
# peer actor (graceful??) TCP EOF but `tricycle`
|
||||
# seems to raise a 0-bytes-read?
|
||||
case ValueError() if (
|
||||
'unclean EOF' in trans_err.args[0]
|
||||
):
|
||||
pass
|
||||
|
||||
# peer actor (task) prolly shutdown quickly due
|
||||
# to cancellation
|
||||
case trio.BrokenResourceError() if (
|
||||
'Connection reset by peer' in trans_err.args[0]
|
||||
):
|
||||
pass
|
||||
|
||||
# unless the disconnect condition falls under "a
|
||||
# normal operation breakage" we usualy console warn
|
||||
# about it.
|
||||
case _:
|
||||
loglevel: str = 'warning'
|
||||
|
||||
|
||||
raise TransportClosed(
|
||||
message=(
|
||||
f'{tpt_name} already closed by peer\n'
|
||||
),
|
||||
src_exc=trans_err,
|
||||
loglevel=loglevel,
|
||||
) from trans_err
|
||||
|
||||
# XXX definitely can happen if transport is closed
|
||||
# manually by another `trio.lowlevel.Task` in the
|
||||
# same actor; we use this in some simulated fault
|
||||
# testing for ex, but generally should never happen
|
||||
# under normal operation!
|
||||
#
|
||||
# NOTE: as such we always re-raise this error from the
|
||||
# RPC msg loop!
|
||||
except trio.ClosedResourceError as cre:
|
||||
closure_err = cre
|
||||
|
||||
raise TransportClosed(
|
||||
message=(
|
||||
f'{tpt_name} was already closed locally ?\n'
|
||||
),
|
||||
src_exc=closure_err,
|
||||
loglevel='error',
|
||||
raise_on_report=(
|
||||
'another task closed this fd' in closure_err.args
|
||||
),
|
||||
) from closure_err
|
||||
|
||||
# graceful TCP EOF disconnect
|
||||
if header == b'':
|
||||
raise TransportClosed(
|
||||
message=(
|
||||
f'{tpt_name} already gracefully closed\n'
|
||||
),
|
||||
loglevel='transport',
|
||||
)
|
||||
|
||||
size: int
|
||||
size, = struct.unpack("<I", header)
|
||||
|
||||
log.transport(f'received header {size}') # type: ignore
|
||||
msg_bytes: bytes = await self.recv_stream.receive_exactly(size)
|
||||
|
||||
log.transport(f"received {msg_bytes}") # type: ignore
|
||||
try:
|
||||
# NOTE: lookup the `trio.Task.context`'s var for
|
||||
# the current `MsgCodec`.
|
||||
codec: MsgCodec = _ctxvar_MsgCodec.get()
|
||||
|
||||
# XXX for ctxvar debug only!
|
||||
# if self._codec.pld_spec != codec.pld_spec:
|
||||
# assert (
|
||||
# task := trio.lowlevel.current_task()
|
||||
# ) is not self._task
|
||||
# self._task = task
|
||||
# self._codec = codec
|
||||
# log.runtime(
|
||||
# f'Using new codec in {self}.recv()\n'
|
||||
# f'codec: {self._codec}\n\n'
|
||||
# f'msg_bytes: {msg_bytes}\n'
|
||||
# )
|
||||
yield codec.decode(msg_bytes)
|
||||
|
||||
# XXX NOTE: since the below error derives from
|
||||
# `DecodeError` we need to catch is specially
|
||||
# and always raise such that spec violations
|
||||
# are never allowed to be caught silently!
|
||||
except msgspec.ValidationError as verr:
|
||||
msgtyperr: MsgTypeError = _mk_recv_mte(
|
||||
msg=msg_bytes,
|
||||
codec=codec,
|
||||
src_validation_error=verr,
|
||||
)
|
||||
# XXX deliver up to `Channel.recv()` where
|
||||
# a re-raise and `Error`-pack can inject the far
|
||||
# end actor `.uid`.
|
||||
yield msgtyperr
|
||||
|
||||
except (
|
||||
msgspec.DecodeError,
|
||||
UnicodeDecodeError,
|
||||
):
|
||||
if decodes_failed < 4:
|
||||
# ignore decoding errors for now and assume they have to
|
||||
# do with a channel drop - hope that receiving from the
|
||||
# channel will raise an expected error and bubble up.
|
||||
try:
|
||||
msg_str: str|bytes = msg_bytes.decode()
|
||||
except UnicodeDecodeError:
|
||||
msg_str = msg_bytes
|
||||
|
||||
log.exception(
|
||||
'Failed to decode msg?\n'
|
||||
f'{codec}\n\n'
|
||||
'Rxed bytes from wire:\n\n'
|
||||
f'{msg_str!r}\n'
|
||||
)
|
||||
decodes_failed += 1
|
||||
else:
|
||||
raise
|
||||
|
||||
async def send(
|
||||
self,
|
||||
msg: msgtypes.MsgType,
|
||||
|
||||
strict_types: bool = True,
|
||||
hide_tb: bool = False,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Send a msgpack encoded py-object-blob-as-msg over TCP.
|
||||
|
||||
If `strict_types == True` then a `MsgTypeError` will be raised on any
|
||||
invalid msg type
|
||||
|
||||
'''
|
||||
__tracebackhide__: bool = hide_tb
|
||||
|
||||
# XXX see `trio._sync.AsyncContextManagerMixin` for details
|
||||
# on the `.acquire()`/`.release()` sequencing..
|
||||
async with self._send_lock:
|
||||
|
||||
# NOTE: lookup the `trio.Task.context`'s var for
|
||||
# the current `MsgCodec`.
|
||||
codec: MsgCodec = _ctxvar_MsgCodec.get()
|
||||
|
||||
# XXX for ctxvar debug only!
|
||||
# if self._codec.pld_spec != codec.pld_spec:
|
||||
# self._codec = codec
|
||||
# log.runtime(
|
||||
# f'Using new codec in {self}.send()\n'
|
||||
# f'codec: {self._codec}\n\n'
|
||||
# f'msg: {msg}\n'
|
||||
# )
|
||||
|
||||
if type(msg) not in msgtypes.__msg_types__:
|
||||
if strict_types:
|
||||
raise _mk_send_mte(
|
||||
msg,
|
||||
codec=codec,
|
||||
)
|
||||
else:
|
||||
log.warning(
|
||||
'Sending non-`Msg`-spec msg?\n\n'
|
||||
f'{msg}\n'
|
||||
)
|
||||
|
||||
try:
|
||||
bytes_data: bytes = codec.encode(msg)
|
||||
except TypeError as _err:
|
||||
typerr = _err
|
||||
msgtyperr: MsgTypeError = _mk_send_mte(
|
||||
msg,
|
||||
codec=codec,
|
||||
message=(
|
||||
f'IPC-msg-spec violation in\n\n'
|
||||
f'{pretty_struct.Struct.pformat(msg)}'
|
||||
),
|
||||
src_type_error=typerr,
|
||||
)
|
||||
raise msgtyperr from typerr
|
||||
|
||||
# supposedly the fastest says,
|
||||
# https://stackoverflow.com/a/54027962
|
||||
size: bytes = struct.pack("<I", len(bytes_data))
|
||||
try:
|
||||
return await self.stream.send_all(size + bytes_data)
|
||||
except (
|
||||
trio.BrokenResourceError,
|
||||
) as trans_err:
|
||||
loglevel = 'transport'
|
||||
match trans_err:
|
||||
case trio.BrokenResourceError() if (
|
||||
'[Errno 32] Broken pipe' in trans_err.args[0]
|
||||
# ^XXX, specifc to UDS transport and its,
|
||||
# well, "speediness".. XD
|
||||
# |_ likely todo with races related to how fast
|
||||
# the socket is setup/torn-down on linux
|
||||
# as it pertains to rando pings from the
|
||||
# `.discovery` subsys and protos.
|
||||
):
|
||||
raise TransportClosed(
|
||||
message=(
|
||||
f'IPC transport already closed by peer\n'
|
||||
f'x)> {type(trans_err)}\n'
|
||||
f' |_{self}\n'
|
||||
),
|
||||
loglevel=loglevel,
|
||||
) from trans_err
|
||||
|
||||
# unless the disconnect condition falls under "a
|
||||
# normal operation breakage" we usualy console warn
|
||||
# about it.
|
||||
case _:
|
||||
log.exception(
|
||||
'Transport layer failed for {self.transport!r} ?\n'
|
||||
)
|
||||
raise trans_err
|
||||
|
||||
# ?TODO? does it help ever to dynamically show this
|
||||
# frame?
|
||||
# try:
|
||||
# <the-above_code>
|
||||
# except BaseException as _err:
|
||||
# err = _err
|
||||
# if not isinstance(err, MsgTypeError):
|
||||
# __tracebackhide__: bool = False
|
||||
# raise
|
||||
|
||||
async def recv(self) -> msgtypes.MsgType:
|
||||
return await self._aiter_pkts.asend(None)
|
||||
|
||||
async def drain(self) -> AsyncIterator[dict]:
|
||||
'''
|
||||
Drain the stream's remaining messages sent from
|
||||
the far end until the connection is closed by
|
||||
the peer.
|
||||
|
||||
'''
|
||||
try:
|
||||
async for msg in self._iter_packets():
|
||||
self.drained.append(msg)
|
||||
except TransportClosed:
|
||||
for msg in self.drained:
|
||||
yield msg
|
||||
|
||||
def __aiter__(self):
|
||||
return self._aiter_pkts
|
||||
|
||||
@property
|
||||
def laddr(self) -> Address:
|
||||
return self._laddr
|
||||
|
||||
@property
|
||||
def raddr(self) -> Address:
|
||||
return self._raddr
|
||||
|
||||
def pformat(self) -> str:
|
||||
return (
|
||||
f'<{type(self).__name__}(\n'
|
||||
f' |_task: {self._task}\n'
|
||||
f'\n'
|
||||
f' |_peers: 2\n'
|
||||
f' laddr: {self._laddr}\n'
|
||||
f' raddr: {self._raddr}\n'
|
||||
f')>\n'
|
||||
)
|
||||
|
||||
__repr__ = __str__ = pformat
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
from typing import Type
|
||||
|
||||
import trio
|
||||
import socket
|
||||
|
||||
from tractor._addr import Address
|
||||
from tractor.ipc._transport import (
|
||||
MsgTransportKey,
|
||||
MsgTransport
|
||||
)
|
||||
from tractor.ipc._tcp import MsgpackTCPStream
|
||||
from tractor.ipc._uds import MsgpackUDSStream
|
||||
|
||||
|
||||
# manually updated list of all supported msg transport types
|
||||
_msg_transports = [
|
||||
MsgpackTCPStream,
|
||||
MsgpackUDSStream
|
||||
]
|
||||
|
||||
|
||||
# convert a MsgTransportKey to the corresponding transport type
|
||||
_key_to_transport: dict[MsgTransportKey, Type[MsgTransport]] = {
|
||||
cls.key(): cls
|
||||
for cls in _msg_transports
|
||||
}
|
||||
|
||||
# convert an Address wrapper to its corresponding transport type
|
||||
_addr_to_transport: dict[Type[Address], Type[MsgTransport]] = {
|
||||
cls.address_type: cls
|
||||
for cls in _msg_transports
|
||||
}
|
||||
|
||||
|
||||
def transport_from_addr(
|
||||
addr: Address,
|
||||
codec_key: str = 'msgpack',
|
||||
) -> Type[MsgTransport]:
|
||||
'''
|
||||
Given a destination address and a desired codec, find the
|
||||
corresponding `MsgTransport` type.
|
||||
|
||||
'''
|
||||
try:
|
||||
return _addr_to_transport[type(addr)]
|
||||
|
||||
except KeyError:
|
||||
raise NotImplementedError(
|
||||
f'No known transport for address {repr(addr)}'
|
||||
)
|
||||
|
||||
|
||||
def transport_from_stream(
|
||||
stream: trio.abc.Stream,
|
||||
codec_key: str = 'msgpack'
|
||||
) -> Type[MsgTransport]:
|
||||
'''
|
||||
Given an arbitrary `trio.abc.Stream` and a desired codec,
|
||||
find the corresponding `MsgTransport` type.
|
||||
|
||||
'''
|
||||
transport = None
|
||||
if isinstance(stream, trio.SocketStream):
|
||||
sock: socket.socket = stream.socket
|
||||
match sock.family:
|
||||
case socket.AF_INET | socket.AF_INET6:
|
||||
transport = 'tcp'
|
||||
|
||||
case socket.AF_UNIX:
|
||||
transport = 'uds'
|
||||
|
||||
case _:
|
||||
raise NotImplementedError(
|
||||
f'Unsupported socket family: {sock.family}'
|
||||
)
|
||||
|
||||
if not transport:
|
||||
raise NotImplementedError(
|
||||
f'Could not figure out transport type for stream type {type(stream)}'
|
||||
)
|
||||
|
||||
key = (codec_key, transport)
|
||||
|
||||
return _key_to_transport[key]
|
||||
|
|
@ -1,216 +0,0 @@
|
|||
# tractor: structured concurrent "actors".
|
||||
# Copyright 2018-eternity Tyler Goodlet.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
Unix Domain Socket implementation of tractor.ipc._transport.MsgTransport protocol
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
import os
|
||||
from socket import (
|
||||
# socket,
|
||||
AF_UNIX,
|
||||
SOCK_STREAM,
|
||||
SO_PASSCRED,
|
||||
SO_PEERCRED,
|
||||
SOL_SOCKET,
|
||||
)
|
||||
import struct
|
||||
|
||||
import trio
|
||||
from trio._highlevel_open_unix_stream import (
|
||||
close_on_error,
|
||||
has_unix,
|
||||
)
|
||||
|
||||
from tractor.msg import MsgCodec
|
||||
from tractor.log import get_logger
|
||||
from tractor._addr import (
|
||||
UDSAddress,
|
||||
unwrap_sockpath,
|
||||
)
|
||||
from tractor.ipc._transport import MsgpackTransport
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
async def open_unix_socket_w_passcred(
|
||||
filename: str|bytes|os.PathLike[str]|os.PathLike[bytes],
|
||||
) -> trio.SocketStream:
|
||||
'''
|
||||
Literally the exact same as `trio.open_unix_socket()` except we set the additiona
|
||||
`socket.SO_PASSCRED` option to ensure the server side (the process calling `accept()`)
|
||||
can extract the connecting peer's credentials, namely OS specific process
|
||||
related IDs.
|
||||
|
||||
See this SO for "why" the extra opts,
|
||||
- https://stackoverflow.com/a/7982749
|
||||
|
||||
'''
|
||||
if not has_unix:
|
||||
raise RuntimeError("Unix sockets are not supported on this platform")
|
||||
|
||||
# much more simplified logic vs tcp sockets - one socket type and only one
|
||||
# possible location to connect to
|
||||
sock = trio.socket.socket(AF_UNIX, SOCK_STREAM)
|
||||
sock.setsockopt(SOL_SOCKET, SO_PASSCRED, 1)
|
||||
with close_on_error(sock):
|
||||
await sock.connect(os.fspath(filename))
|
||||
|
||||
return trio.SocketStream(sock)
|
||||
|
||||
|
||||
def get_peer_info(sock: trio.socket.socket) -> tuple[
|
||||
int, # pid
|
||||
int, # uid
|
||||
int, # guid
|
||||
]:
|
||||
'''
|
||||
Deliver the connecting peer's "credentials"-info as defined in
|
||||
a very Linux specific way..
|
||||
|
||||
For more deats see,
|
||||
- `man accept`,
|
||||
- `man unix`,
|
||||
|
||||
this great online guide to all things sockets,
|
||||
- https://beej.us/guide/bgnet/html/split-wide/man-pages.html#setsockoptman
|
||||
|
||||
AND this **wonderful SO answer**
|
||||
- https://stackoverflow.com/a/7982749
|
||||
|
||||
'''
|
||||
creds: bytes = sock.getsockopt(
|
||||
SOL_SOCKET,
|
||||
SO_PEERCRED,
|
||||
struct.calcsize('3i')
|
||||
)
|
||||
# i.e a tuple of the fields,
|
||||
# pid: int, "process"
|
||||
# uid: int, "user"
|
||||
# gid: int, "group"
|
||||
return struct.unpack('3i', creds)
|
||||
|
||||
|
||||
class MsgpackUDSStream(MsgpackTransport):
|
||||
'''
|
||||
A `trio.SocketStream` around a Unix-Domain-Socket transport
|
||||
delivering `msgpack` encoded msgs using the `msgspec` codec lib.
|
||||
|
||||
'''
|
||||
address_type = UDSAddress
|
||||
layer_key: int = 4
|
||||
|
||||
@property
|
||||
def maddr(self) -> str:
|
||||
if not self.raddr:
|
||||
return '<unknown-peer>'
|
||||
|
||||
filepath: Path = Path(self.raddr.unwrap()[0])
|
||||
return (
|
||||
f'/{self.address_type.proto_key}/{filepath}'
|
||||
# f'/{self.chan.uid[0]}'
|
||||
# f'/{self.cid}'
|
||||
|
||||
# f'/cid={cid_head}..{cid_tail}'
|
||||
# TODO: ? not use this ^ right ?
|
||||
)
|
||||
|
||||
def connected(self) -> bool:
|
||||
return self.stream.socket.fileno() != -1
|
||||
|
||||
@classmethod
|
||||
async def connect_to(
|
||||
cls,
|
||||
addr: UDSAddress,
|
||||
prefix_size: int = 4,
|
||||
codec: MsgCodec|None = None,
|
||||
**kwargs
|
||||
) -> MsgpackUDSStream:
|
||||
|
||||
|
||||
sockpath: Path = addr.sockpath
|
||||
#
|
||||
# ^XXX NOTE, we don't provide any out-of-band `.pid` info
|
||||
# (like, over the socket as extra msgs) since the (augmented)
|
||||
# `.setsockopt()` call tells the OS provide it; the client
|
||||
# pid can then be read on server/listen() side via
|
||||
# `get_peer_info()` above.
|
||||
try:
|
||||
stream = await open_unix_socket_w_passcred(
|
||||
str(sockpath),
|
||||
**kwargs
|
||||
)
|
||||
except (
|
||||
FileNotFoundError,
|
||||
) as fdne:
|
||||
raise ConnectionError(
|
||||
f'Bad UDS socket-filepath-as-address ??\n'
|
||||
f'{addr}\n'
|
||||
f' |_sockpath: {sockpath}\n'
|
||||
) from fdne
|
||||
|
||||
stream = MsgpackUDSStream(
|
||||
stream,
|
||||
prefix_size=prefix_size,
|
||||
codec=codec
|
||||
)
|
||||
stream._raddr = addr
|
||||
return stream
|
||||
|
||||
@classmethod
|
||||
def get_stream_addrs(
|
||||
cls,
|
||||
stream: trio.SocketStream
|
||||
) -> tuple[
|
||||
Path,
|
||||
int,
|
||||
]:
|
||||
sock: trio.socket.socket = stream.socket
|
||||
|
||||
# NOTE XXX, it's unclear why one or the other ends up being
|
||||
# `bytes` versus the socket-file-path, i presume it's
|
||||
# something to do with who is the server (called `.listen()`)?
|
||||
# maybe could be better implemented using another info-query
|
||||
# on the socket like,
|
||||
# https://beej.us/guide/bgnet/html/split-wide/system-calls-or-bust.html#gethostnamewho-am-i
|
||||
sockname: str|bytes = sock.getsockname()
|
||||
# https://beej.us/guide/bgnet/html/split-wide/system-calls-or-bust.html#getpeernamewho-are-you
|
||||
peername: str|bytes = sock.getpeername()
|
||||
match (peername, sockname):
|
||||
case (str(), bytes()):
|
||||
sock_path: Path = Path(peername)
|
||||
case (bytes(), str()):
|
||||
sock_path: Path = Path(sockname)
|
||||
(
|
||||
peer_pid,
|
||||
_,
|
||||
_,
|
||||
) = get_peer_info(sock)
|
||||
|
||||
filedir, filename = unwrap_sockpath(sock_path)
|
||||
laddr = UDSAddress(
|
||||
filedir=filedir,
|
||||
filename=filename,
|
||||
maybe_pid=os.getpid(),
|
||||
)
|
||||
raddr = UDSAddress(
|
||||
filedir=filedir,
|
||||
filename=filename,
|
||||
maybe_pid=peer_pid
|
||||
)
|
||||
return (laddr, raddr)
|
||||
|
|
@ -92,7 +92,7 @@ class StackLevelAdapter(LoggerAdapter):
|
|||
) -> None:
|
||||
'''
|
||||
IPC transport level msg IO; generally anything below
|
||||
`.ipc.Channel` and friends.
|
||||
`._ipc.Channel` and friends.
|
||||
|
||||
'''
|
||||
return self.log(5, msg)
|
||||
|
|
@ -285,7 +285,7 @@ def get_logger(
|
|||
# NOTE: for handling for modules that use ``get_logger(__name__)``
|
||||
# we make the following stylistic choice:
|
||||
# - always avoid duplicate project-package token
|
||||
# in msg output: i.e. tractor.tractor.ipc._chan.py in header
|
||||
# in msg output: i.e. tractor.tractor _ipc.py in header
|
||||
# looks ridiculous XD
|
||||
# - never show the leaf module name in the {name} part
|
||||
# since in python the {filename} is always this same
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ from typing import (
|
|||
Type,
|
||||
TypeVar,
|
||||
TypeAlias,
|
||||
# TYPE_CHECKING,
|
||||
Union,
|
||||
)
|
||||
|
||||
|
|
@ -48,7 +47,6 @@ from tractor.msg import (
|
|||
pretty_struct,
|
||||
)
|
||||
from tractor.log import get_logger
|
||||
from tractor._addr import UnwrappedAddress
|
||||
|
||||
|
||||
log = get_logger('tractor.msgspec')
|
||||
|
|
@ -143,16 +141,9 @@ class Aid(
|
|||
'''
|
||||
name: str
|
||||
uuid: str
|
||||
pid: int|None = None
|
||||
|
||||
# TODO? can/should we extend this field set?
|
||||
# -[ ] use built-in support for UUIDs? `uuid.UUID` which has
|
||||
# multi-protocol support
|
||||
# TODO: use built-in support for UUIDs?
|
||||
# -[ ] `uuid.UUID` which has multi-protocol support
|
||||
# https://jcristharif.com/msgspec/supported-types.html#uuid
|
||||
#
|
||||
# -[ ] as per the `.ipc._uds` / `._addr` comments, maybe we
|
||||
# should also include at least `.pid` (equiv to port for tcp)
|
||||
# and/or host-part always?
|
||||
|
||||
|
||||
class SpawnSpec(
|
||||
|
|
@ -176,8 +167,8 @@ class SpawnSpec(
|
|||
|
||||
# TODO: not just sockaddr pairs?
|
||||
# -[ ] abstract into a `TransportAddr` type?
|
||||
reg_addrs: list[UnwrappedAddress]
|
||||
bind_addrs: list[UnwrappedAddress]|None
|
||||
reg_addrs: list[tuple[str, int]]
|
||||
bind_addrs: list[tuple[str, int]]
|
||||
|
||||
|
||||
# TODO: caps based RPC support in the payload?
|
||||
|
|
|
|||
58
uv.lock
58
uv.lock
|
|
@ -11,15 +11,6 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/89/aa/ab0f7891a01eeb2d2e338ae8fecbe57fcebea1a24dbb64d45801bfab481d/attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308", size = 63397 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bidict"
|
||||
version = "0.23.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9a/6e/026678aa5a830e07cd9498a05d3e7e650a4f56a42f267a53d22bcda1bdc9/bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71", size = 29093 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/99/37/e8730c3587a65eb5645d4aba2d27aae48e8003614d6aaf15dda67f702f1f/bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5", size = 32764 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cffi"
|
||||
version = "1.17.1"
|
||||
|
|
@ -29,38 +20,10 @@ dependencies = [
|
|||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264 },
|
||||
{ url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651 },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259 },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200 },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235 },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721 },
|
||||
{ url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242 },
|
||||
{ url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999 },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242 },
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604 },
|
||||
{ url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727 },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400 },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178 },
|
||||
{ url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840 },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803 },
|
||||
{ url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850 },
|
||||
{ url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729 },
|
||||
{ url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256 },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424 },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568 },
|
||||
{ url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736 },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448 },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976 },
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989 },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802 },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792 },
|
||||
{ url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893 },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810 },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200 },
|
||||
{ url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447 },
|
||||
{ url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358 },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469 },
|
||||
{ url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475 },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009 },
|
||||
]
|
||||
|
|
@ -257,21 +220,6 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/e4/ea/d836f008d33151c7a1f62caf3d8dd782e4d15f6a43897f64480c2b8de2ad/prompt_toolkit-3.0.50-py3-none-any.whl", hash = "sha256:9b6427eb19e479d98acff65196a307c555eb567989e6d88ebbb1b509d9779198", size = 387816 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psutil"
|
||||
version = "7.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456", size = 497003 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25", size = 238051 },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da", size = 239535 },
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91", size = 275004 },
|
||||
{ url = "https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34", size = 277986 },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993", size = 279544 },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99", size = 241053 },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ptyprocess"
|
||||
version = "0.7.0"
|
||||
|
|
@ -373,8 +321,6 @@ name = "tractor"
|
|||
version = "0.1.0a6.dev0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "bidict" },
|
||||
{ name = "cffi" },
|
||||
{ name = "colorlog" },
|
||||
{ name = "msgspec" },
|
||||
{ name = "pdbp" },
|
||||
|
|
@ -388,7 +334,6 @@ dev = [
|
|||
{ name = "greenback" },
|
||||
{ name = "pexpect" },
|
||||
{ name = "prompt-toolkit" },
|
||||
{ name = "psutil" },
|
||||
{ name = "pyperclip" },
|
||||
{ name = "pytest" },
|
||||
{ name = "stackscope" },
|
||||
|
|
@ -397,8 +342,6 @@ dev = [
|
|||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "bidict", specifier = ">=0.23.1" },
|
||||
{ name = "cffi", specifier = ">=1.17.1" },
|
||||
{ name = "colorlog", specifier = ">=6.8.2,<7" },
|
||||
{ name = "msgspec", specifier = ">=0.19.0" },
|
||||
{ name = "pdbp", specifier = ">=1.6,<2" },
|
||||
|
|
@ -412,7 +355,6 @@ dev = [
|
|||
{ name = "greenback", specifier = ">=1.2.1,<2" },
|
||||
{ name = "pexpect", specifier = ">=4.9.0,<5" },
|
||||
{ name = "prompt-toolkit", specifier = ">=3.0.50" },
|
||||
{ name = "psutil", specifier = ">=7.0.0" },
|
||||
{ name = "pyperclip", specifier = ">=1.9.0" },
|
||||
{ name = "pytest", specifier = ">=8.3.5" },
|
||||
{ name = "stackscope", specifier = ">=0.2.2,<0.3" },
|
||||
|
|
|
|||
Loading…
Reference in New Issue