Actually do per-actor-caching in `open_symcache()`
Use `dict.get()` instead of `try/except KeyError` for the actor-level
`_caches` lookup; actually store the loaded `symcache` back into
`_caches[provider]` so subsequent opens retrieve any in-mem cache
instance.
Deats,
- swap `_caches[provider]` `KeyError` catch to
`.get()` with `if symcache:` guard.
- assign result back to `_caches[provider]` before
yielding so the cache is persistent across calls.
- rename local `cache` -> `symcache` throughout.
- add `loglevel` param and init `get_console_log()`
at function scope.
Also,
- improve `log.info()` msgs with `{provider!r}`,
load-latency, and cachefile path details.
- demote "no cache exists" from `.warning()` to
`.info()`.
- track `from_msg` to distinguish "NEW request to
provider" vs loaded-from-file in final log line.
- reformat `or` conditions to multiline style.
- move `import time` to module-level.
(this commit msg was generated in some part by [`claude-code`][claude-code-gh])
[claude-code-gh]: https://github.com/anthropics/claude-code
symcache_polishin
parent
f7244b89f8
commit
6647651229
|
|
@ -29,6 +29,7 @@ from contextlib import (
|
||||||
)
|
)
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from pprint import pformat
|
from pprint import pformat
|
||||||
|
import time
|
||||||
from typing import (
|
from typing import (
|
||||||
Any,
|
Any,
|
||||||
Callable,
|
Callable,
|
||||||
|
|
@ -48,7 +49,10 @@ except ModuleNotFoundError:
|
||||||
import tomli as tomllib
|
import tomli as tomllib
|
||||||
from msgspec import field
|
from msgspec import field
|
||||||
|
|
||||||
from piker.log import get_logger
|
from piker.log import (
|
||||||
|
get_console_log,
|
||||||
|
get_logger,
|
||||||
|
)
|
||||||
from piker import config
|
from piker import config
|
||||||
from piker.types import Struct
|
from piker.types import Struct
|
||||||
from piker.brokers import (
|
from piker.brokers import (
|
||||||
|
|
@ -356,11 +360,15 @@ def mk_cachefile(
|
||||||
) -> Path:
|
) -> Path:
|
||||||
cachedir: Path = config.get_conf_dir() / '_cache'
|
cachedir: Path = config.get_conf_dir() / '_cache'
|
||||||
if not cachedir.is_dir():
|
if not cachedir.is_dir():
|
||||||
log.info(f'Creating `nativedb` director: {cachedir}')
|
log.info(
|
||||||
|
f'Creating symbology-cache subdir\n'
|
||||||
|
f'{cachedir}'
|
||||||
|
)
|
||||||
cachedir.mkdir()
|
cachedir.mkdir()
|
||||||
|
|
||||||
cachefile: Path = cachedir / f'{str(provider)}.symcache.toml'
|
cachefile: Path = cachedir / f'{str(provider)}.symcache.toml'
|
||||||
cachefile.touch()
|
cachefile.touch()
|
||||||
|
|
||||||
return cachefile
|
return cachefile
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -372,8 +380,15 @@ async def open_symcache(
|
||||||
only_from_memcache: bool = False, # no API req
|
only_from_memcache: bool = False, # no API req
|
||||||
_no_symcache: bool = False, # no backend support
|
_no_symcache: bool = False, # no backend support
|
||||||
|
|
||||||
|
loglevel: str = 'info',
|
||||||
|
|
||||||
) -> SymbologyCache:
|
) -> SymbologyCache:
|
||||||
|
|
||||||
|
log = get_console_log(
|
||||||
|
level=loglevel,
|
||||||
|
name=__name__,
|
||||||
|
)
|
||||||
|
|
||||||
if isinstance(mod_or_name, str):
|
if isinstance(mod_or_name, str):
|
||||||
mod = get_brokermod(mod_or_name)
|
mod = get_brokermod(mod_or_name)
|
||||||
else:
|
else:
|
||||||
|
|
@ -388,7 +403,8 @@ async def open_symcache(
|
||||||
# the backend pkg-module is annotated appropriately.
|
# the backend pkg-module is annotated appropriately.
|
||||||
if (
|
if (
|
||||||
getattr(mod, '_no_symcache', False)
|
getattr(mod, '_no_symcache', False)
|
||||||
or _no_symcache
|
or
|
||||||
|
_no_symcache
|
||||||
):
|
):
|
||||||
yield SymbologyCache(
|
yield SymbologyCache(
|
||||||
mod=mod,
|
mod=mod,
|
||||||
|
|
@ -400,60 +416,75 @@ async def open_symcache(
|
||||||
# actor-level cache-cache XD
|
# actor-level cache-cache XD
|
||||||
global _caches
|
global _caches
|
||||||
if not reload:
|
if not reload:
|
||||||
try:
|
symcache: SymbologyCache|None = _caches.get(provider)
|
||||||
yield _caches[provider]
|
if symcache:
|
||||||
except KeyError:
|
yield symcache
|
||||||
|
else:
|
||||||
msg: str = (
|
msg: str = (
|
||||||
f'No asset info cache exists yet for `{provider}`'
|
f'No in-mem symcache found for {provider!r}\n'
|
||||||
|
f'Loading..'
|
||||||
)
|
)
|
||||||
if only_from_memcache:
|
if only_from_memcache:
|
||||||
raise RuntimeError(msg)
|
raise RuntimeError(msg)
|
||||||
else:
|
else:
|
||||||
log.warning(msg)
|
log.info(msg)
|
||||||
|
|
||||||
# if no cache exists or an explicit reload is requested, load
|
# if no cache exists or an explicit reload is requested, load
|
||||||
# the provider API and call appropriate endpoints to populate
|
# the provider API and call appropriate endpoints to populate
|
||||||
# the mkt and asset tables.
|
# the mkt and asset tables.
|
||||||
if (
|
if (
|
||||||
reload
|
reload
|
||||||
or not cachefile.is_file()
|
or
|
||||||
|
not cachefile.is_file()
|
||||||
):
|
):
|
||||||
cache = await SymbologyCache.from_scratch(
|
log.info(
|
||||||
|
f'Generating NEW symbology-cache for {provider!r}..\n'
|
||||||
|
f'>[{cachefile}'
|
||||||
|
)
|
||||||
|
symcache: SymbologyCache = await SymbologyCache.from_scratch(
|
||||||
mod=mod,
|
mod=mod,
|
||||||
fp=cachefile,
|
fp=cachefile,
|
||||||
)
|
)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
log.info(
|
log.info(
|
||||||
f'Loading EXISTING `{mod.name}` symbology cache:\n'
|
f'Loading EXISTING symbology-cache for {provider!r}..\n'
|
||||||
f'> {cachefile}'
|
f'[>{cachefile}'
|
||||||
)
|
)
|
||||||
import time
|
now: float = time.time()
|
||||||
now = time.time()
|
|
||||||
with cachefile.open('rb') as existing_fp:
|
with cachefile.open('rb') as existing_fp:
|
||||||
data: dict[str, dict] = tomllib.load(existing_fp)
|
data: dict[str, dict] = tomllib.load(existing_fp)
|
||||||
log.runtime(f'SYMCACHE TOML LOAD TIME: {time.time() - now}')
|
log.runtime(
|
||||||
|
f'Symcache loaded!\n'
|
||||||
|
f'load-latency: {time.time() - now}\n'
|
||||||
|
)
|
||||||
|
|
||||||
# if there's an empty file for some reason we need
|
# if there's an empty file for some reason we need
|
||||||
# to do a full reload as well!
|
# to do a full reload as well!
|
||||||
if not data:
|
if not data:
|
||||||
cache = await SymbologyCache.from_scratch(
|
from_msg: str = 'NEW request to provider'
|
||||||
|
symcache = await SymbologyCache.from_scratch(
|
||||||
mod=mod,
|
mod=mod,
|
||||||
fp=cachefile,
|
fp=cachefile,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
cache = SymbologyCache.from_dict(
|
from_msg: str = f'{cachefile}'
|
||||||
|
symcache = SymbologyCache.from_dict(
|
||||||
data,
|
data,
|
||||||
mod=mod,
|
mod=mod,
|
||||||
fp=cachefile,
|
fp=cachefile,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# TODO: use a real profiling sys..
|
# TODO: use a real profiling sys..
|
||||||
# https://github.com/pikers/piker/issues/337
|
# https://github.com/pikers/piker/issues/337
|
||||||
log.info(f'SYMCACHE LOAD TIME: {time.time() - now}')
|
log.info(
|
||||||
|
f'Symcache data loaded from {from_msg} !\n'
|
||||||
|
f'load-latency: {time.time() - now}s\n'
|
||||||
|
f'cachefile: {cachefile}\n'
|
||||||
|
)
|
||||||
|
|
||||||
yield cache
|
_caches[provider] = symcache
|
||||||
|
yield symcache
|
||||||
|
|
||||||
# TODO: write only when changes detected? but that should
|
# TODO: write only when changes detected? but that should
|
||||||
# never happen right except on reload?
|
# never happen right except on reload?
|
||||||
|
|
@ -476,7 +507,6 @@ def get_symcache(
|
||||||
async with (
|
async with (
|
||||||
# only for runtime's debug mode
|
# only for runtime's debug mode
|
||||||
tractor.open_nursery(debug_mode=True),
|
tractor.open_nursery(debug_mode=True),
|
||||||
|
|
||||||
open_symcache(
|
open_symcache(
|
||||||
get_brokermod(provider),
|
get_brokermod(provider),
|
||||||
reload=force_reload,
|
reload=force_reload,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue