Disable shm resource tracker via flag on 3.13+
As per the newly added support, https://docs.python.org/3/library/multiprocessing.shared_memory.htmlmain
parent
79f502034f
commit
5021514a6a
|
@ -17,36 +17,59 @@
|
||||||
Utils to tame mp non-SC madeness
|
Utils to tame mp non-SC madeness
|
||||||
|
|
||||||
'''
|
'''
|
||||||
|
import platform
|
||||||
|
|
||||||
|
|
||||||
# !TODO! in 3.13 this can be disabled (the-same/similarly) using
|
|
||||||
# a flag,
|
|
||||||
# - [ ] soo if it works like this, drop this module entirely for
|
|
||||||
# 3.13+ B)
|
|
||||||
# |_https://docs.python.org/3/library/multiprocessing.shared_memory.html
|
|
||||||
#
|
|
||||||
def disable_mantracker():
|
def disable_mantracker():
|
||||||
'''
|
'''
|
||||||
Disable all `multiprocessing` "resource tracking" machinery since
|
Disable all `multiprocessing` "resource tracking" machinery since
|
||||||
it's an absolute multi-threaded mess of non-SC madness.
|
it's an absolute multi-threaded mess of non-SC madness.
|
||||||
|
|
||||||
'''
|
'''
|
||||||
from multiprocessing import resource_tracker as mantracker
|
from multiprocessing.shared_memory import SharedMemory
|
||||||
|
|
||||||
# Tell the "resource tracker" thing to fuck off.
|
|
||||||
class ManTracker(mantracker.ResourceTracker):
|
|
||||||
def register(self, name, rtype):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def unregister(self, name, rtype):
|
# 3.13+ only.. can pass `track=False` to disable
|
||||||
pass
|
# all the resource tracker bs.
|
||||||
|
# https://docs.python.org/3/library/multiprocessing.shared_memory.html
|
||||||
|
if (_py_313 := (
|
||||||
|
platform.python_version_tuple()[:-1]
|
||||||
|
>=
|
||||||
|
('3', '13')
|
||||||
|
)
|
||||||
|
):
|
||||||
|
from functools import partial
|
||||||
|
return partial(
|
||||||
|
SharedMemory,
|
||||||
|
track=False,
|
||||||
|
)
|
||||||
|
|
||||||
def ensure_running(self):
|
# !TODO, once we drop 3.12- we can obvi remove all this!
|
||||||
pass
|
else:
|
||||||
|
from multiprocessing import (
|
||||||
|
resource_tracker as mantracker,
|
||||||
|
)
|
||||||
|
|
||||||
# "know your land and know your prey"
|
# Tell the "resource tracker" thing to fuck off.
|
||||||
# https://www.dailymotion.com/video/x6ozzco
|
class ManTracker(mantracker.ResourceTracker):
|
||||||
mantracker._resource_tracker = ManTracker()
|
def register(self, name, rtype):
|
||||||
mantracker.register = mantracker._resource_tracker.register
|
pass
|
||||||
mantracker.ensure_running = mantracker._resource_tracker.ensure_running
|
|
||||||
mantracker.unregister = mantracker._resource_tracker.unregister
|
def unregister(self, name, rtype):
|
||||||
mantracker.getfd = mantracker._resource_tracker.getfd
|
pass
|
||||||
|
|
||||||
|
def ensure_running(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# "know your land and know your prey"
|
||||||
|
# https://www.dailymotion.com/video/x6ozzco
|
||||||
|
mantracker._resource_tracker = ManTracker()
|
||||||
|
mantracker.register = mantracker._resource_tracker.register
|
||||||
|
mantracker.ensure_running = mantracker._resource_tracker.ensure_running
|
||||||
|
mantracker.unregister = mantracker._resource_tracker.unregister
|
||||||
|
mantracker.getfd = mantracker._resource_tracker.getfd
|
||||||
|
|
||||||
|
# use std type verbatim
|
||||||
|
shmT = SharedMemory
|
||||||
|
|
||||||
|
return shmT
|
||||||
|
|
|
@ -23,14 +23,15 @@ considered optional within the context of this runtime-library.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
from multiprocessing import shared_memory as shm
|
||||||
|
from multiprocessing.shared_memory import (
|
||||||
|
# SharedMemory,
|
||||||
|
ShareableList,
|
||||||
|
)
|
||||||
|
import platform
|
||||||
from sys import byteorder
|
from sys import byteorder
|
||||||
import time
|
import time
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from multiprocessing import shared_memory as shm
|
|
||||||
from multiprocessing.shared_memory import (
|
|
||||||
SharedMemory,
|
|
||||||
ShareableList,
|
|
||||||
)
|
|
||||||
|
|
||||||
from msgspec import (
|
from msgspec import (
|
||||||
Struct,
|
Struct,
|
||||||
|
@ -61,7 +62,7 @@ except ImportError:
|
||||||
log = get_logger(__name__)
|
log = get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
disable_mantracker()
|
SharedMemory = disable_mantracker()
|
||||||
|
|
||||||
|
|
||||||
class SharedInt:
|
class SharedInt:
|
||||||
|
@ -797,8 +798,15 @@ def open_shm_list(
|
||||||
# "close" attached shm on actor teardown
|
# "close" attached shm on actor teardown
|
||||||
try:
|
try:
|
||||||
actor = tractor.current_actor()
|
actor = tractor.current_actor()
|
||||||
|
|
||||||
actor.lifetime_stack.callback(shml.shm.close)
|
actor.lifetime_stack.callback(shml.shm.close)
|
||||||
actor.lifetime_stack.callback(shml.shm.unlink)
|
|
||||||
|
# XXX on 3.13+ we don't need to call this?
|
||||||
|
# -> bc we pass `track=False` for `SharedMemeory` orr?
|
||||||
|
if (
|
||||||
|
platform.python_version_tuple()[:-1] < ('3', '13')
|
||||||
|
):
|
||||||
|
actor.lifetime_stack.callback(shml.shm.unlink)
|
||||||
except RuntimeError:
|
except RuntimeError:
|
||||||
log.warning('tractor runtime not active, skipping teardown steps')
|
log.warning('tractor runtime not active, skipping teardown steps')
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue