Add "spawn" start method support
Add full support for using the "spawn" process starting method as per: https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods Add a `spawn_method` argument to `tractor.run()` for specifying the desired method explicitly. By default use the "fastest" method available. On *nix systems this is the original "forkserver" method. This should be the solution to getting windows support! Resolves #60spawn_method_support
parent
d75739e9c7
commit
7014a07986
|
@ -19,6 +19,7 @@ from ._trionics import open_nursery
|
||||||
from ._state import current_actor
|
from ._state import current_actor
|
||||||
from ._exceptions import RemoteActorError, ModuleNotExposed
|
from ._exceptions import RemoteActorError, ModuleNotExposed
|
||||||
from . import msg
|
from . import msg
|
||||||
|
from . import _spawn
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
@ -92,12 +93,14 @@ def run(
|
||||||
name: str = None,
|
name: str = None,
|
||||||
arbiter_addr: Tuple[str, int] = (
|
arbiter_addr: Tuple[str, int] = (
|
||||||
_default_arbiter_host, _default_arbiter_port),
|
_default_arbiter_host, _default_arbiter_port),
|
||||||
|
spawn_method: str = 'forkserver',
|
||||||
**kwargs: typing.Dict[str, typing.Any],
|
**kwargs: typing.Dict[str, typing.Any],
|
||||||
) -> Any:
|
) -> Any:
|
||||||
"""Run a trio-actor async function in process.
|
"""Run a trio-actor async function in process.
|
||||||
|
|
||||||
This is tractor's main entry and the start point for any async actor.
|
This is tractor's main entry and the start point for any async actor.
|
||||||
"""
|
"""
|
||||||
|
_spawn.try_set_start_method(spawn_method)
|
||||||
return trio.run(_main, async_fn, args, kwargs, name, arbiter_addr)
|
return trio.run(_main, async_fn, args, kwargs, name, arbiter_addr)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -391,7 +391,8 @@ class Actor:
|
||||||
f" {chan} from {chan.uid}")
|
f" {chan} from {chan.uid}")
|
||||||
break
|
break
|
||||||
|
|
||||||
log.trace(f"Received msg {msg} from {chan.uid}") # type: ignore
|
log.trace( # type: ignore
|
||||||
|
f"Received msg {msg} from {chan.uid}")
|
||||||
if msg.get('cid'):
|
if msg.get('cid'):
|
||||||
# deliver response to local caller/waiter
|
# deliver response to local caller/waiter
|
||||||
await self._push_result(chan, msg)
|
await self._push_result(chan, msg)
|
||||||
|
@ -478,18 +479,20 @@ class Actor:
|
||||||
self,
|
self,
|
||||||
accept_addr: Tuple[str, int],
|
accept_addr: Tuple[str, int],
|
||||||
forkserver_info: Tuple[Any, Any, Any, Any, Any],
|
forkserver_info: Tuple[Any, Any, Any, Any, Any],
|
||||||
|
start_method: str,
|
||||||
parent_addr: Tuple[str, int] = None
|
parent_addr: Tuple[str, int] = None
|
||||||
) -> None:
|
) -> None:
|
||||||
"""The routine called *after fork* which invokes a fresh ``trio.run``
|
"""The routine called *after fork* which invokes a fresh ``trio.run``
|
||||||
"""
|
"""
|
||||||
self._forkserver_info = forkserver_info
|
self._forkserver_info = forkserver_info
|
||||||
from ._spawn import ctx
|
from ._spawn import try_set_start_method
|
||||||
|
spawn_ctx = try_set_start_method(start_method)
|
||||||
if self.loglevel is not None:
|
if self.loglevel is not None:
|
||||||
log.info(
|
log.info(
|
||||||
f"Setting loglevel for {self.uid} to {self.loglevel}")
|
f"Setting loglevel for {self.uid} to {self.loglevel}")
|
||||||
get_console_log(self.loglevel)
|
get_console_log(self.loglevel)
|
||||||
log.info(
|
log.info(
|
||||||
f"Started new {ctx.current_process()} for {self.uid}")
|
f"Started new {spawn_ctx.current_process()} for {self.uid}")
|
||||||
_state._current_actor = self
|
_state._current_actor = self
|
||||||
log.debug(f"parent_addr is {parent_addr}")
|
log.debug(f"parent_addr is {parent_addr}")
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -5,15 +5,35 @@ Mostly just wrapping around ``multiprocessing``.
|
||||||
"""
|
"""
|
||||||
import multiprocessing as mp
|
import multiprocessing as mp
|
||||||
from multiprocessing import forkserver, semaphore_tracker # type: ignore
|
from multiprocessing import forkserver, semaphore_tracker # type: ignore
|
||||||
from typing import Tuple
|
from typing import Tuple, Optional
|
||||||
|
|
||||||
from . import _forkserver_hackzorz
|
from . import _forkserver_hackzorz
|
||||||
from ._state import current_actor
|
from ._state import current_actor
|
||||||
from ._actor import Actor
|
from ._actor import Actor
|
||||||
|
|
||||||
|
|
||||||
|
_ctx: mp.context.BaseContext = mp.get_context("spawn")
|
||||||
|
|
||||||
|
|
||||||
|
def try_set_start_method(name: str) -> mp.context.BaseContext:
|
||||||
|
"""Attempt to set the start method for ``multiprocess.Process`` spawning.
|
||||||
|
|
||||||
|
If the desired method is not supported the sub-interpreter (aka "spawn"
|
||||||
|
method) is used.
|
||||||
|
"""
|
||||||
|
global _ctx
|
||||||
|
|
||||||
|
allowed = mp.get_all_start_methods()
|
||||||
|
if name not in allowed:
|
||||||
|
name == 'spawn'
|
||||||
|
|
||||||
|
assert name in allowed
|
||||||
|
|
||||||
|
if name == 'forkserver':
|
||||||
_forkserver_hackzorz.override_stdlib()
|
_forkserver_hackzorz.override_stdlib()
|
||||||
ctx = mp.get_context("forkserver")
|
|
||||||
|
_ctx = mp.get_context(name)
|
||||||
|
return _ctx
|
||||||
|
|
||||||
|
|
||||||
def is_main_process() -> bool:
|
def is_main_process() -> bool:
|
||||||
|
@ -29,6 +49,13 @@ def new_proc(
|
||||||
bind_addr: Tuple[str, int],
|
bind_addr: Tuple[str, int],
|
||||||
parent_addr: Tuple[str, int],
|
parent_addr: Tuple[str, int],
|
||||||
) -> mp.Process:
|
) -> mp.Process:
|
||||||
|
"""Create a new ``multiprocessing.Process`` using the
|
||||||
|
spawn method as configured using ``try_set_start_method()``.
|
||||||
|
"""
|
||||||
|
start_method = _ctx.get_start_method()
|
||||||
|
if start_method == 'forkserver':
|
||||||
|
# XXX do our hackery on the stdlib to avoid multiple
|
||||||
|
# forkservers (one at each subproc layer).
|
||||||
fs = forkserver._forkserver
|
fs = forkserver._forkserver
|
||||||
curr_actor = current_actor()
|
curr_actor = current_actor()
|
||||||
if is_main_process() and not curr_actor._forkserver_info:
|
if is_main_process() and not curr_actor._forkserver_info:
|
||||||
|
@ -52,10 +79,17 @@ def new_proc(
|
||||||
semaphore_tracker._semaphore_tracker._pid,
|
semaphore_tracker._semaphore_tracker._pid,
|
||||||
semaphore_tracker._semaphore_tracker._fd,
|
semaphore_tracker._semaphore_tracker._fd,
|
||||||
) = curr_actor._forkserver_info
|
) = curr_actor._forkserver_info
|
||||||
|
else:
|
||||||
|
fs_info = (None, None, None, None, None)
|
||||||
|
|
||||||
return ctx.Process(
|
return _ctx.Process(
|
||||||
target=actor._fork_main,
|
target=actor._fork_main,
|
||||||
args=(bind_addr, fs_info, parent_addr),
|
args=(
|
||||||
|
bind_addr,
|
||||||
|
fs_info,
|
||||||
|
start_method,
|
||||||
|
parent_addr
|
||||||
|
),
|
||||||
# daemon=True,
|
# daemon=True,
|
||||||
name=name,
|
name=name,
|
||||||
)
|
)
|
||||||
|
|
Loading…
Reference in New Issue