Also shorten shm-key for `ShmList` on macos

Same problem as for the `ShmArray` tokens, so tweak and reuse
the `_shorten_key_for_macos()` helper and call it from
`open_shm_list()` similarly.

Some tweaks/updates to the various helpers,
- support `prefix/suffix` inputs and if provided take their lengths and
  subtract them from the known *macOS shm_open() has a 31 char limit
  (PSHMNAMLEN)* when generating and using the `hashlib.sha256()` value
  which overrides (for now..) wtv `key` is passed by the caller.
- pass the appropriate `suffix='_first/_last'` values for the `ShmArray`
  token generators case.
- add a `prefix: str = 'shml_'` param to `open_shm_list()`.
- better log formatting with `!r` to report any key shortening.
ns_aware
Gud Boi 2026-03-01 18:41:02 -05:00
parent 01c0db651a
commit d8a3969048
1 changed files with 47 additions and 15 deletions

View File

@ -197,28 +197,46 @@ def get_shm_token(key: str) -> NDToken | None:
return _known_tokens.get(key) return _known_tokens.get(key)
def _shorten_key_for_macos(key: str) -> str: def _shorten_key_for_macos(
key: str,
prefix: str = '',
suffix: str = '',
) -> str:
''' '''
macOS has a 31 character limit for POSIX shared MacOS has a (hillarious) 31 character limit for POSIX shared
memory names. Hash long keys to fit within this memory names. Hash long keys to fit within this limit while
limit while maintaining uniqueness. maintaining uniqueness.
''' '''
# macOS shm_open() has a 31 char limit (PSHMNAMLEN) # macOS shm_open() has a 31 char limit (PSHMNAMLEN)
# format: /t_<hash16> = 19 chars, well under limit # format: /t_<hash16> = 19 chars, well under limit
if len(key) <= 31: max_len: int = 31
if len(key) <= max_len:
return key return key
key_hash: str = hashlib.sha256( _hash: str = hashlib.sha256(
key.encode() key.encode()
).hexdigest()[:16] ).hexdigest()
short_key = f't_{key_hash}'
hash_len: int = (
(max_len - 1)
- len(prefix)
- len(suffix)
)
key_hash: str = _hash[:hash_len]
short_key = (
prefix
+
f'{key_hash}'
+
suffix
)
log.debug( log.debug(
f'Shortened shm key for macOS:\n' f'Shortened shm key for macOS:\n'
f' original: {key} ({len(key)} chars)\n' f' original: {key!r} ({len(key)!r} chars)\n'
f' shortened: {short_key}' f' shortened: {short_key!r}'
f' ({len(short_key)} chars)' f' ({len(short_key)!r} chars)'
) )
return short_key return short_key
@ -237,12 +255,16 @@ def _make_token(
# On macOS, shorten keys that exceed the # On macOS, shorten keys that exceed the
# 31 character limit # 31 character limit
if platform.system() == 'Darwin': if platform.system() == 'Darwin':
shm_name = _shorten_key_for_macos(key) shm_name = _shorten_key_for_macos(
key=key,
)
shm_first = _shorten_key_for_macos( shm_first = _shorten_key_for_macos(
key + '_first' key=key,
suffix='_first',
) )
shm_last = _shorten_key_for_macos( shm_last = _shorten_key_for_macos(
key + '_last' key=key,
suffix='_last',
) )
else: else:
shm_name = key shm_name = key
@ -760,7 +782,10 @@ def maybe_open_shm_ndarray(
False, # not newly opened False, # not newly opened
) )
except KeyError: except KeyError:
log.warning(f"Could not find {key} in shms cache") log.warning(
f'Could not find key in shms cache,\n'
f'key: {key!r}\n'
)
if dtype: if dtype:
token = _make_token( token = _make_token(
key, key,
@ -870,6 +895,7 @@ def open_shm_list(
size: int = int(2 ** 10), size: int = int(2 ** 10),
dtype: float | int | bool | str | bytes | None = float, dtype: float | int | bool | str | bytes | None = float,
readonly: bool = True, readonly: bool = True,
prefix: str = 'shml_',
) -> ShmList: ) -> ShmList:
@ -883,6 +909,12 @@ def open_shm_list(
}[dtype] }[dtype]
sequence = [default] * size sequence = [default] * size
if platform.system() == 'Darwin':
key: str = _shorten_key_for_macos(
key=key,
prefix=prefix,
)
shml = ShmList( shml = ShmList(
sequence=sequence, sequence=sequence,
name=key, name=key,