Decode cached mkts and assets back to structs B)
As part of loading the cache we can now fill the asset sub-tables:
`.mktmaps` and `.assets` with their deserialized struct instances!
In theory this might be possible for the backend defined `Pair` structs
as well but we need to figure out probably an endpoint to offer
the conversion?
Also, add a `SymbologyCache.search()` which allows sync code to scan the
existing (known via cache) symbol set just like how async code can use the
(much slower) `open_symbol_search()` ctx endpoint 💥
account_tests
parent
309b91676d
commit
13f231b926
|
@ -29,9 +29,14 @@ from contextlib import (
|
||||||
contextmanager as cm,
|
contextmanager as cm,
|
||||||
)
|
)
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from pprint import pformat
|
||||||
|
from typing import (
|
||||||
|
Any,
|
||||||
|
TYPE_CHECKING,
|
||||||
|
)
|
||||||
from types import ModuleType
|
from types import ModuleType
|
||||||
|
|
||||||
|
from fuzzywuzzy import process as fuzzy
|
||||||
import tomli_w # for fast symbol cache writing
|
import tomli_w # for fast symbol cache writing
|
||||||
try:
|
try:
|
||||||
import tomllib
|
import tomllib
|
||||||
|
@ -43,10 +48,12 @@ from ..log import get_logger
|
||||||
from .. import config
|
from .. import config
|
||||||
from ..brokers import open_cached_client
|
from ..brokers import open_cached_client
|
||||||
from .types import Struct
|
from .types import Struct
|
||||||
from ..accounting import (
|
|
||||||
Asset,
|
if TYPE_CHECKING:
|
||||||
MktPair,
|
from ..accounting import (
|
||||||
)
|
Asset,
|
||||||
|
MktPair,
|
||||||
|
)
|
||||||
|
|
||||||
log = get_logger('data.cache')
|
log = get_logger('data.cache')
|
||||||
|
|
||||||
|
@ -79,7 +86,6 @@ class SymbologyCache(Struct):
|
||||||
for key, attr in {
|
for key, attr in {
|
||||||
'assets': self.assets,
|
'assets': self.assets,
|
||||||
'pairs': self.pairs,
|
'pairs': self.pairs,
|
||||||
# 'mktmaps': self.mktmaps,
|
|
||||||
}.items():
|
}.items():
|
||||||
if not attr:
|
if not attr:
|
||||||
log.warning(
|
log.warning(
|
||||||
|
@ -89,6 +95,11 @@ class SymbologyCache(Struct):
|
||||||
|
|
||||||
cachedict[key] = attr
|
cachedict[key] = attr
|
||||||
|
|
||||||
|
# serialize mkts
|
||||||
|
mktmapsdict = cachedict['mktmaps'] = {}
|
||||||
|
for fqme, mkt in self.mktmaps.items():
|
||||||
|
mktmapsdict[fqme] = mkt.to_dict()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with self.fp.open(mode='wb') as fp:
|
with self.fp.open(mode='wb') as fp:
|
||||||
tomli_w.dump(cachedict, fp)
|
tomli_w.dump(cachedict, fp)
|
||||||
|
@ -132,6 +143,24 @@ class SymbologyCache(Struct):
|
||||||
|
|
||||||
return self
|
return self
|
||||||
|
|
||||||
|
def search(
|
||||||
|
self,
|
||||||
|
pattern: str,
|
||||||
|
|
||||||
|
) -> dict[str, Struct]:
|
||||||
|
|
||||||
|
matches = fuzzy.extractBests(
|
||||||
|
pattern,
|
||||||
|
self.mktmaps,
|
||||||
|
score_cutoff=50,
|
||||||
|
)
|
||||||
|
|
||||||
|
# repack in dict[fqme, MktPair] form
|
||||||
|
return {
|
||||||
|
item[0].fqme: item[0]
|
||||||
|
for item in matches
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# actor-process-local in-mem-cache of symcaches (by backend).
|
# actor-process-local in-mem-cache of symcaches (by backend).
|
||||||
_caches: dict[str, SymbologyCache] = {}
|
_caches: dict[str, SymbologyCache] = {}
|
||||||
|
@ -200,18 +229,60 @@ def open_symcache(
|
||||||
f'> {cache.fp}'
|
f'> {cache.fp}'
|
||||||
)
|
)
|
||||||
import time
|
import time
|
||||||
|
from ..accounting import (
|
||||||
|
Asset,
|
||||||
|
MktPair,
|
||||||
|
)
|
||||||
|
|
||||||
now = time.time()
|
now = time.time()
|
||||||
with cachefile.open('rb') as existing_fp:
|
with cachefile.open('rb') as existing_fp:
|
||||||
data: dict[str, dict] = tomllib.load(existing_fp)
|
data: dict[str, dict] = tomllib.load(existing_fp)
|
||||||
log.runtime(f'SYMCACHE TOML LOAD TIME: {time.time() - now}')
|
log.runtime(f'SYMCACHE TOML LOAD TIME: {time.time() - now}')
|
||||||
|
|
||||||
for key, table in data.items():
|
# load `dict` -> `Asset`
|
||||||
attr: dict[str, Any] = getattr(cache, key)
|
assettable = data.pop('assets')
|
||||||
assert not attr
|
for name, asdict in assettable.items():
|
||||||
# if attr != table:
|
cache.assets[name] = Asset.from_msg(asdict)
|
||||||
# log.info(f'OUT-OF-SYNC symbology cache: {key}')
|
|
||||||
|
|
||||||
setattr(cache, key, table)
|
# load `dict` -> `MktPair`
|
||||||
|
dne: list[str] = []
|
||||||
|
mkttable = data.pop('mktmaps')
|
||||||
|
for fqme, mktdict in mkttable.items():
|
||||||
|
|
||||||
|
# pull asset refs from (presumably) now previously
|
||||||
|
# loaded asset set above B)
|
||||||
|
src_k: str = mktdict.pop('src')
|
||||||
|
dst_k: str = mktdict.pop('dst')
|
||||||
|
src: Asset = cache.assets[src_k]
|
||||||
|
|
||||||
|
dst: Asset
|
||||||
|
if not (dst := cache.assets.get(dst_k)):
|
||||||
|
dne.append(dst_k)
|
||||||
|
continue
|
||||||
|
|
||||||
|
mkt = MktPair(
|
||||||
|
src=src,
|
||||||
|
dst=dst,
|
||||||
|
**mktdict,
|
||||||
|
)
|
||||||
|
assert mkt.fqme == fqme
|
||||||
|
cache.mktmaps[fqme] = mkt
|
||||||
|
|
||||||
|
log.warning(
|
||||||
|
f'These `MktPair.dst: Asset`s DNE says `{mod.name}` ?\n'
|
||||||
|
f'{pformat(dne)}'
|
||||||
|
)
|
||||||
|
|
||||||
|
# copy in backend specific pairs table directly without
|
||||||
|
# struct loading for now..
|
||||||
|
pairtable = data.pop('pairs')
|
||||||
|
cache.pairs = pairtable
|
||||||
|
|
||||||
|
# TODO: some kinda way to allow the backend
|
||||||
|
# to provide a struct-loader per entry?
|
||||||
|
# for key, pairtable in pairtable.items():
|
||||||
|
# pair: Struct = cache.mod.load_pair(pairtable)
|
||||||
|
# cache.pairs[key] = pair
|
||||||
|
|
||||||
# TODO: use a real profiling sys..
|
# TODO: use a real profiling sys..
|
||||||
# https://github.com/pikers/piker/issues/337
|
# https://github.com/pikers/piker/issues/337
|
||||||
|
|
Loading…
Reference in New Issue