Also shorten shm-key for `ShmList` on macos
Same problem as for the `ShmArray` tokens, so tweak and reuse the `_shorten_key_for_macos()` helper and call it from `open_shm_list()` similarly. Some tweaks/updates to the various helpers, - support `prefix/suffix` inputs and if provided take their lengths and subtract them from the known *macOS shm_open() has a 31 char limit (PSHMNAMLEN)* when generating and using the `hashlib.sha256()` value which overrides (for now..) wtv `key` is passed by the caller. - pass the appropriate `suffix='_first/_last'` values for the `ShmArray` token generators case. - add a `prefix: str = 'shml_'` param to `open_shm_list()`. - better log formatting with `!r` to report any key shortening.ns_aware
parent
01c0db651a
commit
d8a3969048
|
|
@ -197,28 +197,46 @@ def get_shm_token(key: str) -> NDToken | None:
|
||||||
return _known_tokens.get(key)
|
return _known_tokens.get(key)
|
||||||
|
|
||||||
|
|
||||||
def _shorten_key_for_macos(key: str) -> str:
|
def _shorten_key_for_macos(
|
||||||
|
key: str,
|
||||||
|
prefix: str = '',
|
||||||
|
suffix: str = '',
|
||||||
|
) -> str:
|
||||||
'''
|
'''
|
||||||
macOS has a 31 character limit for POSIX shared
|
MacOS has a (hillarious) 31 character limit for POSIX shared
|
||||||
memory names. Hash long keys to fit within this
|
memory names. Hash long keys to fit within this limit while
|
||||||
limit while maintaining uniqueness.
|
maintaining uniqueness.
|
||||||
|
|
||||||
'''
|
'''
|
||||||
# macOS shm_open() has a 31 char limit (PSHMNAMLEN)
|
# macOS shm_open() has a 31 char limit (PSHMNAMLEN)
|
||||||
# format: /t_<hash16> = 19 chars, well under limit
|
# format: /t_<hash16> = 19 chars, well under limit
|
||||||
if len(key) <= 31:
|
max_len: int = 31
|
||||||
|
if len(key) <= max_len:
|
||||||
return key
|
return key
|
||||||
|
|
||||||
key_hash: str = hashlib.sha256(
|
_hash: str = hashlib.sha256(
|
||||||
key.encode()
|
key.encode()
|
||||||
).hexdigest()[:16]
|
).hexdigest()
|
||||||
short_key = f't_{key_hash}'
|
|
||||||
|
hash_len: int = (
|
||||||
|
(max_len - 1)
|
||||||
|
- len(prefix)
|
||||||
|
- len(suffix)
|
||||||
|
)
|
||||||
|
key_hash: str = _hash[:hash_len]
|
||||||
|
short_key = (
|
||||||
|
prefix
|
||||||
|
+
|
||||||
|
f'{key_hash}'
|
||||||
|
+
|
||||||
|
suffix
|
||||||
|
)
|
||||||
|
|
||||||
log.debug(
|
log.debug(
|
||||||
f'Shortened shm key for macOS:\n'
|
f'Shortened shm key for macOS:\n'
|
||||||
f' original: {key} ({len(key)} chars)\n'
|
f' original: {key!r} ({len(key)!r} chars)\n'
|
||||||
f' shortened: {short_key}'
|
f' shortened: {short_key!r}'
|
||||||
f' ({len(short_key)} chars)'
|
f' ({len(short_key)!r} chars)'
|
||||||
)
|
)
|
||||||
return short_key
|
return short_key
|
||||||
|
|
||||||
|
|
@ -237,12 +255,16 @@ def _make_token(
|
||||||
# On macOS, shorten keys that exceed the
|
# On macOS, shorten keys that exceed the
|
||||||
# 31 character limit
|
# 31 character limit
|
||||||
if platform.system() == 'Darwin':
|
if platform.system() == 'Darwin':
|
||||||
shm_name = _shorten_key_for_macos(key)
|
shm_name = _shorten_key_for_macos(
|
||||||
|
key=key,
|
||||||
|
)
|
||||||
shm_first = _shorten_key_for_macos(
|
shm_first = _shorten_key_for_macos(
|
||||||
key + '_first'
|
key=key,
|
||||||
|
suffix='_first',
|
||||||
)
|
)
|
||||||
shm_last = _shorten_key_for_macos(
|
shm_last = _shorten_key_for_macos(
|
||||||
key + '_last'
|
key=key,
|
||||||
|
suffix='_last',
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
shm_name = key
|
shm_name = key
|
||||||
|
|
@ -760,7 +782,10 @@ def maybe_open_shm_ndarray(
|
||||||
False, # not newly opened
|
False, # not newly opened
|
||||||
)
|
)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
log.warning(f"Could not find {key} in shms cache")
|
log.warning(
|
||||||
|
f'Could not find key in shms cache,\n'
|
||||||
|
f'key: {key!r}\n'
|
||||||
|
)
|
||||||
if dtype:
|
if dtype:
|
||||||
token = _make_token(
|
token = _make_token(
|
||||||
key,
|
key,
|
||||||
|
|
@ -870,6 +895,7 @@ def open_shm_list(
|
||||||
size: int = int(2 ** 10),
|
size: int = int(2 ** 10),
|
||||||
dtype: float | int | bool | str | bytes | None = float,
|
dtype: float | int | bool | str | bytes | None = float,
|
||||||
readonly: bool = True,
|
readonly: bool = True,
|
||||||
|
prefix: str = 'shml_',
|
||||||
|
|
||||||
) -> ShmList:
|
) -> ShmList:
|
||||||
|
|
||||||
|
|
@ -883,6 +909,12 @@ def open_shm_list(
|
||||||
}[dtype]
|
}[dtype]
|
||||||
sequence = [default] * size
|
sequence = [default] * size
|
||||||
|
|
||||||
|
if platform.system() == 'Darwin':
|
||||||
|
key: str = _shorten_key_for_macos(
|
||||||
|
key=key,
|
||||||
|
prefix=prefix,
|
||||||
|
)
|
||||||
|
|
||||||
shml = ShmList(
|
shml = ShmList(
|
||||||
sequence=sequence,
|
sequence=sequence,
|
||||||
name=key,
|
name=key,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue