diff --git a/piker/data/_symcache.py b/piker/data/_symcache.py index 1f1cb9ec..22baedd7 100644 --- a/piker/data/_symcache.py +++ b/piker/data/_symcache.py @@ -29,6 +29,7 @@ from contextlib import ( ) from pathlib import Path from pprint import pformat +import time from typing import ( Any, Callable, @@ -48,7 +49,10 @@ except ModuleNotFoundError: import tomli as tomllib from msgspec import field -from piker.log import get_logger +from piker.log import ( + get_console_log, + get_logger, +) from piker import config from piker.types import Struct from piker.brokers import ( @@ -356,24 +360,35 @@ def mk_cachefile( ) -> Path: cachedir: Path = config.get_conf_dir() / '_cache' if not cachedir.is_dir(): - log.info(f'Creating `nativedb` director: {cachedir}') + log.info( + f'Creating symbology-cache subdir\n' + f'{cachedir}' + ) cachedir.mkdir() cachefile: Path = cachedir / f'{str(provider)}.symcache.toml' cachefile.touch() + return cachefile @acm async def open_symcache( - mod_or_name: ModuleType | str, + mod_or_name: ModuleType|str, reload: bool = False, only_from_memcache: bool = False, # no API req _no_symcache: bool = False, # no backend support + loglevel: str = 'info', + ) -> SymbologyCache: + log = get_console_log( + level=loglevel, + name=__name__, + ) + if isinstance(mod_or_name, str): mod = get_brokermod(mod_or_name) else: @@ -388,7 +403,8 @@ async def open_symcache( # the backend pkg-module is annotated appropriately. if ( getattr(mod, '_no_symcache', False) - or _no_symcache + or + _no_symcache ): yield SymbologyCache( mod=mod, @@ -400,60 +416,75 @@ async def open_symcache( # actor-level cache-cache XD global _caches if not reload: - try: - yield _caches[provider] - except KeyError: + symcache: SymbologyCache|None = _caches.get(provider) + if symcache: + yield symcache + else: msg: str = ( - f'No asset info cache exists yet for `{provider}`' + f'No in-mem symcache found for {provider!r}\n' + f'Loading..' ) if only_from_memcache: raise RuntimeError(msg) else: - log.warning(msg) + log.info(msg) # if no cache exists or an explicit reload is requested, load # the provider API and call appropriate endpoints to populate # the mkt and asset tables. if ( reload - or not cachefile.is_file() + or + not cachefile.is_file() ): - cache = await SymbologyCache.from_scratch( + log.info( + f'Generating NEW symbology-cache for {provider!r}..\n' + f'>[{cachefile}' + ) + symcache: SymbologyCache = await SymbologyCache.from_scratch( mod=mod, fp=cachefile, ) else: log.info( - f'Loading EXISTING `{mod.name}` symbology cache:\n' - f'> {cachefile}' + f'Loading EXISTING symbology-cache for {provider!r}..\n' + f'[>{cachefile}' ) - import time - now = time.time() + now: float = time.time() with cachefile.open('rb') as existing_fp: data: dict[str, dict] = tomllib.load(existing_fp) - log.runtime(f'SYMCACHE TOML LOAD TIME: {time.time() - now}') + log.runtime( + f'Symcache loaded!\n' + f'load-latency: {time.time() - now}\n' + ) # if there's an empty file for some reason we need # to do a full reload as well! if not data: - cache = await SymbologyCache.from_scratch( + from_msg: str = 'NEW request to provider' + symcache = await SymbologyCache.from_scratch( mod=mod, fp=cachefile, ) else: - cache = SymbologyCache.from_dict( + from_msg: str = f'{cachefile}' + symcache = SymbologyCache.from_dict( data, mod=mod, fp=cachefile, ) - # TODO: use a real profiling sys.. # https://github.com/pikers/piker/issues/337 - log.info(f'SYMCACHE LOAD TIME: {time.time() - now}') + log.info( + f'Symcache data loaded from {from_msg} !\n' + f'load-latency: {time.time() - now}s\n' + f'cachefile: {cachefile}\n' + ) - yield cache + _caches[provider] = symcache + yield symcache # TODO: write only when changes detected? but that should # never happen right except on reload? @@ -476,7 +507,6 @@ def get_symcache( async with ( # only for runtime's debug mode tractor.open_nursery(debug_mode=True), - open_symcache( get_brokermod(provider), reload=force_reload,