Cache symbol ids where possible
Cache both in the client and at the function call level inside the quoter context using a `@afifo_cache`.kivy_mainline_and_py3.8
parent
af464b45ff
commit
714c203c3e
|
@ -14,6 +14,7 @@ from ..calc import humanize, percent_change
|
||||||
from . import config
|
from . import config
|
||||||
from ._util import resproc, BrokerError
|
from ._util import resproc, BrokerError
|
||||||
from ..log import get_logger, colorize_json
|
from ..log import get_logger, colorize_json
|
||||||
|
from .._async_utils import alifo_cache
|
||||||
|
|
||||||
# TODO: move to urllib3/requests once supported
|
# TODO: move to urllib3/requests once supported
|
||||||
import asks
|
import asks
|
||||||
|
@ -112,6 +113,7 @@ class Client:
|
||||||
self.access_data = {}
|
self.access_data = {}
|
||||||
self.user_data = {}
|
self.user_data = {}
|
||||||
self._reload_config(config)
|
self._reload_config(config)
|
||||||
|
self._symbol_cache = {}
|
||||||
|
|
||||||
def _reload_config(self, config=None, **kwargs):
|
def _reload_config(self, config=None, **kwargs):
|
||||||
log.warn("Reloading access config data")
|
log.warn("Reloading access config data")
|
||||||
|
@ -212,12 +214,25 @@ class Client:
|
||||||
|
|
||||||
async def tickers2ids(self, tickers):
|
async def tickers2ids(self, tickers):
|
||||||
"""Helper routine that take a sequence of ticker symbols and returns
|
"""Helper routine that take a sequence of ticker symbols and returns
|
||||||
their corresponding QT symbol ids.
|
their corresponding QT numeric symbol ids.
|
||||||
|
|
||||||
|
Cache any symbol to id lookups for later use.
|
||||||
"""
|
"""
|
||||||
data = await self.api.symbols(names=','.join(tickers))
|
cache = self._symbol_cache
|
||||||
symbols2ids = {}
|
symbols2ids = {}
|
||||||
for ticker, symbol in zip(tickers, data['symbols']):
|
for symbol in tickers:
|
||||||
symbols2ids[symbol['symbol']] = str(symbol['symbolId'])
|
id = cache.get(symbol)
|
||||||
|
if id is not None:
|
||||||
|
symbols2ids[symbol] = id
|
||||||
|
|
||||||
|
# still missing uncached values - hit the server
|
||||||
|
to_lookup = list(set(tickers) - set(symbols2ids))
|
||||||
|
if to_lookup:
|
||||||
|
data = await self.api.symbols(names=','.join(to_lookup))
|
||||||
|
for ticker, symbol in zip(to_lookup, data['symbols']):
|
||||||
|
name = symbol['symbol']
|
||||||
|
assert name == ticker
|
||||||
|
cache[name] = symbols2ids[name] = str(symbol['symbolId'])
|
||||||
|
|
||||||
return symbols2ids
|
return symbols2ids
|
||||||
|
|
||||||
|
@ -384,30 +399,21 @@ async def quoter(client: Client, tickers: List[str]):
|
||||||
a cache of this map lazily as requests from in for new tickers/symbols.
|
a cache of this map lazily as requests from in for new tickers/symbols.
|
||||||
Most of the closure variables here are to deal with that.
|
Most of the closure variables here are to deal with that.
|
||||||
"""
|
"""
|
||||||
t2ids = {}
|
|
||||||
ids = ''
|
|
||||||
|
|
||||||
def filter_symbols(quotes_dict: dict):
|
@alifo_cache(maxsize=128)
|
||||||
nonlocal t2ids
|
async def get_symbol_id_seq(symbols: Tuple[str]):
|
||||||
for symbol, quote in quotes_dict.items():
|
"""For each tuple ``(symbol_1, symbol_2, ... , symbol_n)``
|
||||||
if quote['low52w'] is None:
|
return a symbol id sequence string ``'id_1,id_2, ... , id_n'``.
|
||||||
log.warn(
|
"""
|
||||||
f"{symbol} seems to be defunct discarding from tickers")
|
return ','.join(map(str, (await client.tickers2ids(symbols)).values()))
|
||||||
t2ids.pop(symbol)
|
|
||||||
|
|
||||||
async def get_quote(tickers):
|
async def get_quote(tickers):
|
||||||
"""Query for quotes using cached symbol ids.
|
"""Query for quotes using cached symbol ids.
|
||||||
"""
|
"""
|
||||||
if not tickers:
|
if not tickers:
|
||||||
return {}
|
return {}
|
||||||
nonlocal ids, t2ids
|
|
||||||
new, current = set(tickers), set(t2ids.keys())
|
ids = await get_symbol_id_seq(tuple(tickers))
|
||||||
if new != current:
|
|
||||||
# update ticker ids cache
|
|
||||||
log.debug(f"Tickers set changed {new - current}")
|
|
||||||
t2ids = await client.tickers2ids(tickers)
|
|
||||||
# re-save symbol -> ids cache
|
|
||||||
ids = ','.join(map(str, t2ids.values()))
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
quotes_resp = await client.api.quotes(ids=ids)
|
quotes_resp = await client.api.quotes(ids=ids)
|
||||||
|
@ -443,9 +449,10 @@ async def quoter(client: Client, tickers: List[str]):
|
||||||
|
|
||||||
# strip out unknown/invalid symbols
|
# strip out unknown/invalid symbols
|
||||||
first_quotes_dict = await get_quote(tickers)
|
first_quotes_dict = await get_quote(tickers)
|
||||||
filter_symbols(first_quotes_dict)
|
for symbol, quote in first_quotes_dict.items():
|
||||||
# re-save symbol -> ids cache
|
if quote['low52w'] is None:
|
||||||
ids = ','.join(map(str, t2ids.values()))
|
log.warn(
|
||||||
|
f"{symbol} seems to be defunct")
|
||||||
|
|
||||||
return get_quote
|
return get_quote
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue