Actually do per-actor-caching in `open_symcache()`
Use `dict.get()` instead of `try/except KeyError` for the actor-level
`_caches` lookup; actually store the loaded `symcache` back into
`_caches[provider]` so subsequent opens retrieve any in-mem cache
instance.
Deats,
- swap `_caches[provider]` `KeyError` catch to
`.get()` with `if symcache:` guard.
- assign result back to `_caches[provider]` before
yielding so the cache is persistent across calls.
- rename local `cache` -> `symcache` throughout.
- add `loglevel` param and init `get_console_log()`
at function scope.
Also,
- improve `log.info()` msgs with `{provider!r}`,
load-latency, and cachefile path details.
- demote "no cache exists" from `.warning()` to
`.info()`.
- track `from_msg` to distinguish "NEW request to
provider" vs loaded-from-file in final log line.
- reformat `or` conditions to multiline style.
- move `import time` to module-level.
(this commit msg was generated in some part by [`claude-code`][claude-code-gh])
[claude-code-gh]: https://github.com/anthropics/claude-code
symcache_polishin
parent
f7244b89f8
commit
6647651229
|
|
@ -29,6 +29,7 @@ from contextlib import (
|
|||
)
|
||||
from pathlib import Path
|
||||
from pprint import pformat
|
||||
import time
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
|
|
@ -48,7 +49,10 @@ except ModuleNotFoundError:
|
|||
import tomli as tomllib
|
||||
from msgspec import field
|
||||
|
||||
from piker.log import get_logger
|
||||
from piker.log import (
|
||||
get_console_log,
|
||||
get_logger,
|
||||
)
|
||||
from piker import config
|
||||
from piker.types import Struct
|
||||
from piker.brokers import (
|
||||
|
|
@ -356,24 +360,35 @@ def mk_cachefile(
|
|||
) -> Path:
|
||||
cachedir: Path = config.get_conf_dir() / '_cache'
|
||||
if not cachedir.is_dir():
|
||||
log.info(f'Creating `nativedb` director: {cachedir}')
|
||||
log.info(
|
||||
f'Creating symbology-cache subdir\n'
|
||||
f'{cachedir}'
|
||||
)
|
||||
cachedir.mkdir()
|
||||
|
||||
cachefile: Path = cachedir / f'{str(provider)}.symcache.toml'
|
||||
cachefile.touch()
|
||||
|
||||
return cachefile
|
||||
|
||||
|
||||
@acm
|
||||
async def open_symcache(
|
||||
mod_or_name: ModuleType | str,
|
||||
mod_or_name: ModuleType|str,
|
||||
|
||||
reload: bool = False,
|
||||
only_from_memcache: bool = False, # no API req
|
||||
_no_symcache: bool = False, # no backend support
|
||||
|
||||
loglevel: str = 'info',
|
||||
|
||||
) -> SymbologyCache:
|
||||
|
||||
log = get_console_log(
|
||||
level=loglevel,
|
||||
name=__name__,
|
||||
)
|
||||
|
||||
if isinstance(mod_or_name, str):
|
||||
mod = get_brokermod(mod_or_name)
|
||||
else:
|
||||
|
|
@ -388,7 +403,8 @@ async def open_symcache(
|
|||
# the backend pkg-module is annotated appropriately.
|
||||
if (
|
||||
getattr(mod, '_no_symcache', False)
|
||||
or _no_symcache
|
||||
or
|
||||
_no_symcache
|
||||
):
|
||||
yield SymbologyCache(
|
||||
mod=mod,
|
||||
|
|
@ -400,60 +416,75 @@ async def open_symcache(
|
|||
# actor-level cache-cache XD
|
||||
global _caches
|
||||
if not reload:
|
||||
try:
|
||||
yield _caches[provider]
|
||||
except KeyError:
|
||||
symcache: SymbologyCache|None = _caches.get(provider)
|
||||
if symcache:
|
||||
yield symcache
|
||||
else:
|
||||
msg: str = (
|
||||
f'No asset info cache exists yet for `{provider}`'
|
||||
f'No in-mem symcache found for {provider!r}\n'
|
||||
f'Loading..'
|
||||
)
|
||||
if only_from_memcache:
|
||||
raise RuntimeError(msg)
|
||||
else:
|
||||
log.warning(msg)
|
||||
log.info(msg)
|
||||
|
||||
# if no cache exists or an explicit reload is requested, load
|
||||
# the provider API and call appropriate endpoints to populate
|
||||
# the mkt and asset tables.
|
||||
if (
|
||||
reload
|
||||
or not cachefile.is_file()
|
||||
or
|
||||
not cachefile.is_file()
|
||||
):
|
||||
cache = await SymbologyCache.from_scratch(
|
||||
log.info(
|
||||
f'Generating NEW symbology-cache for {provider!r}..\n'
|
||||
f'>[{cachefile}'
|
||||
)
|
||||
symcache: SymbologyCache = await SymbologyCache.from_scratch(
|
||||
mod=mod,
|
||||
fp=cachefile,
|
||||
)
|
||||
|
||||
else:
|
||||
log.info(
|
||||
f'Loading EXISTING `{mod.name}` symbology cache:\n'
|
||||
f'> {cachefile}'
|
||||
f'Loading EXISTING symbology-cache for {provider!r}..\n'
|
||||
f'[>{cachefile}'
|
||||
)
|
||||
import time
|
||||
now = time.time()
|
||||
now: float = time.time()
|
||||
with cachefile.open('rb') as existing_fp:
|
||||
data: dict[str, dict] = tomllib.load(existing_fp)
|
||||
log.runtime(f'SYMCACHE TOML LOAD TIME: {time.time() - now}')
|
||||
log.runtime(
|
||||
f'Symcache loaded!\n'
|
||||
f'load-latency: {time.time() - now}\n'
|
||||
)
|
||||
|
||||
# if there's an empty file for some reason we need
|
||||
# to do a full reload as well!
|
||||
if not data:
|
||||
cache = await SymbologyCache.from_scratch(
|
||||
from_msg: str = 'NEW request to provider'
|
||||
symcache = await SymbologyCache.from_scratch(
|
||||
mod=mod,
|
||||
fp=cachefile,
|
||||
)
|
||||
else:
|
||||
cache = SymbologyCache.from_dict(
|
||||
from_msg: str = f'{cachefile}'
|
||||
symcache = SymbologyCache.from_dict(
|
||||
data,
|
||||
mod=mod,
|
||||
fp=cachefile,
|
||||
)
|
||||
|
||||
|
||||
# TODO: use a real profiling sys..
|
||||
# https://github.com/pikers/piker/issues/337
|
||||
log.info(f'SYMCACHE LOAD TIME: {time.time() - now}')
|
||||
log.info(
|
||||
f'Symcache data loaded from {from_msg} !\n'
|
||||
f'load-latency: {time.time() - now}s\n'
|
||||
f'cachefile: {cachefile}\n'
|
||||
)
|
||||
|
||||
yield cache
|
||||
_caches[provider] = symcache
|
||||
yield symcache
|
||||
|
||||
# TODO: write only when changes detected? but that should
|
||||
# never happen right except on reload?
|
||||
|
|
@ -476,7 +507,6 @@ def get_symcache(
|
|||
async with (
|
||||
# only for runtime's debug mode
|
||||
tractor.open_nursery(debug_mode=True),
|
||||
|
||||
open_symcache(
|
||||
get_brokermod(provider),
|
||||
reload=force_reload,
|
||||
|
|
|
|||
Loading…
Reference in New Issue