Cache symbol ids where possible

Cache both in the client and at the function call level inside the
quoter context using a `@afifo_cache`.
kivy_mainline_and_py3.8
Tyler Goodlet 2018-11-25 15:00:08 -05:00
parent af464b45ff
commit 714c203c3e
1 changed files with 31 additions and 24 deletions

View File

@ -14,6 +14,7 @@ from ..calc import humanize, percent_change
from . import config from . import config
from ._util import resproc, BrokerError from ._util import resproc, BrokerError
from ..log import get_logger, colorize_json from ..log import get_logger, colorize_json
from .._async_utils import alifo_cache
# TODO: move to urllib3/requests once supported # TODO: move to urllib3/requests once supported
import asks import asks
@ -112,6 +113,7 @@ class Client:
self.access_data = {} self.access_data = {}
self.user_data = {} self.user_data = {}
self._reload_config(config) self._reload_config(config)
self._symbol_cache = {}
def _reload_config(self, config=None, **kwargs): def _reload_config(self, config=None, **kwargs):
log.warn("Reloading access config data") log.warn("Reloading access config data")
@ -212,12 +214,25 @@ class Client:
async def tickers2ids(self, tickers): async def tickers2ids(self, tickers):
"""Helper routine that take a sequence of ticker symbols and returns """Helper routine that take a sequence of ticker symbols and returns
their corresponding QT symbol ids. their corresponding QT numeric symbol ids.
Cache any symbol to id lookups for later use.
""" """
data = await self.api.symbols(names=','.join(tickers)) cache = self._symbol_cache
symbols2ids = {} symbols2ids = {}
for ticker, symbol in zip(tickers, data['symbols']): for symbol in tickers:
symbols2ids[symbol['symbol']] = str(symbol['symbolId']) id = cache.get(symbol)
if id is not None:
symbols2ids[symbol] = id
# still missing uncached values - hit the server
to_lookup = list(set(tickers) - set(symbols2ids))
if to_lookup:
data = await self.api.symbols(names=','.join(to_lookup))
for ticker, symbol in zip(to_lookup, data['symbols']):
name = symbol['symbol']
assert name == ticker
cache[name] = symbols2ids[name] = str(symbol['symbolId'])
return symbols2ids return symbols2ids
@ -384,30 +399,21 @@ async def quoter(client: Client, tickers: List[str]):
a cache of this map lazily as requests from in for new tickers/symbols. a cache of this map lazily as requests from in for new tickers/symbols.
Most of the closure variables here are to deal with that. Most of the closure variables here are to deal with that.
""" """
t2ids = {}
ids = ''
def filter_symbols(quotes_dict: dict): @alifo_cache(maxsize=128)
nonlocal t2ids async def get_symbol_id_seq(symbols: Tuple[str]):
for symbol, quote in quotes_dict.items(): """For each tuple ``(symbol_1, symbol_2, ... , symbol_n)``
if quote['low52w'] is None: return a symbol id sequence string ``'id_1,id_2, ... , id_n'``.
log.warn( """
f"{symbol} seems to be defunct discarding from tickers") return ','.join(map(str, (await client.tickers2ids(symbols)).values()))
t2ids.pop(symbol)
async def get_quote(tickers): async def get_quote(tickers):
"""Query for quotes using cached symbol ids. """Query for quotes using cached symbol ids.
""" """
if not tickers: if not tickers:
return {} return {}
nonlocal ids, t2ids
new, current = set(tickers), set(t2ids.keys()) ids = await get_symbol_id_seq(tuple(tickers))
if new != current:
# update ticker ids cache
log.debug(f"Tickers set changed {new - current}")
t2ids = await client.tickers2ids(tickers)
# re-save symbol -> ids cache
ids = ','.join(map(str, t2ids.values()))
try: try:
quotes_resp = await client.api.quotes(ids=ids) quotes_resp = await client.api.quotes(ids=ids)
@ -443,9 +449,10 @@ async def quoter(client: Client, tickers: List[str]):
# strip out unknown/invalid symbols # strip out unknown/invalid symbols
first_quotes_dict = await get_quote(tickers) first_quotes_dict = await get_quote(tickers)
filter_symbols(first_quotes_dict) for symbol, quote in first_quotes_dict.items():
# re-save symbol -> ids cache if quote['low52w'] is None:
ids = ','.join(map(str, t2ids.values())) log.warn(
f"{symbol} seems to be defunct")
return get_quote return get_quote