Compare commits

..

No commits in common. "symcache_polishin" and "main" have entirely different histories.

1 changed files with 22 additions and 52 deletions

View File

@ -29,7 +29,6 @@ from contextlib import (
)
from pathlib import Path
from pprint import pformat
import time
from typing import (
Any,
Callable,
@ -49,10 +48,7 @@ except ModuleNotFoundError:
import tomli as tomllib
from msgspec import field
from piker.log import (
get_console_log,
get_logger,
)
from piker.log import get_logger
from piker import config
from piker.types import Struct
from piker.brokers import (
@ -360,15 +356,11 @@ def mk_cachefile(
) -> Path:
cachedir: Path = config.get_conf_dir() / '_cache'
if not cachedir.is_dir():
log.info(
f'Creating symbology-cache subdir\n'
f'{cachedir}'
)
log.info(f'Creating `nativedb` director: {cachedir}')
cachedir.mkdir()
cachefile: Path = cachedir / f'{str(provider)}.symcache.toml'
cachefile.touch()
return cachefile
@ -380,15 +372,8 @@ async def open_symcache(
only_from_memcache: bool = False, # no API req
_no_symcache: bool = False, # no backend support
loglevel: str = 'info',
) -> SymbologyCache:
log = get_console_log(
level=loglevel,
name=__name__,
)
if isinstance(mod_or_name, str):
mod = get_brokermod(mod_or_name)
else:
@ -403,8 +388,7 @@ async def open_symcache(
# the backend pkg-module is annotated appropriately.
if (
getattr(mod, '_no_symcache', False)
or
_no_symcache
or _no_symcache
):
yield SymbologyCache(
mod=mod,
@ -416,75 +400,60 @@ async def open_symcache(
# actor-level cache-cache XD
global _caches
if not reload:
symcache: SymbologyCache|None = _caches.get(provider)
if symcache:
yield symcache
else:
try:
yield _caches[provider]
except KeyError:
msg: str = (
f'No in-mem symcache found for {provider!r}\n'
f'Loading..'
f'No asset info cache exists yet for `{provider}`'
)
if only_from_memcache:
raise RuntimeError(msg)
else:
log.info(msg)
log.warning(msg)
# if no cache exists or an explicit reload is requested, load
# the provider API and call appropriate endpoints to populate
# the mkt and asset tables.
if (
reload
or
not cachefile.is_file()
or not cachefile.is_file()
):
log.info(
f'Generating NEW symbology-cache for {provider!r}..\n'
f'>[{cachefile}'
)
symcache: SymbologyCache = await SymbologyCache.from_scratch(
cache = await SymbologyCache.from_scratch(
mod=mod,
fp=cachefile,
)
else:
log.info(
f'Loading EXISTING symbology-cache for {provider!r}..\n'
f'[>{cachefile}'
f'Loading EXISTING `{mod.name}` symbology cache:\n'
f'> {cachefile}'
)
now: float = time.time()
import time
now = time.time()
with cachefile.open('rb') as existing_fp:
data: dict[str, dict] = tomllib.load(existing_fp)
log.runtime(
f'Symcache loaded!\n'
f'load-latency: {time.time() - now}\n'
)
log.runtime(f'SYMCACHE TOML LOAD TIME: {time.time() - now}')
# if there's an empty file for some reason we need
# to do a full reload as well!
if not data:
from_msg: str = 'NEW request to provider'
symcache = await SymbologyCache.from_scratch(
cache = await SymbologyCache.from_scratch(
mod=mod,
fp=cachefile,
)
else:
from_msg: str = f'{cachefile}'
symcache = SymbologyCache.from_dict(
cache = SymbologyCache.from_dict(
data,
mod=mod,
fp=cachefile,
)
# TODO: use a real profiling sys..
# https://github.com/pikers/piker/issues/337
log.info(
f'Symcache data loaded from {from_msg} !\n'
f'load-latency: {time.time() - now}s\n'
f'cachefile: {cachefile}\n'
)
log.info(f'SYMCACHE LOAD TIME: {time.time() - now}')
_caches[provider] = symcache
yield symcache
yield cache
# TODO: write only when changes detected? but that should
# never happen right except on reload?
@ -507,6 +476,7 @@ def get_symcache(
async with (
# only for runtime's debug mode
tractor.open_nursery(debug_mode=True),
open_symcache(
get_brokermod(provider),
reload=force_reload,