pikers
/
piker
3
0
Fork 0
Code Issues 5 Pull Requests 7 Packages Projects Releases Wiki Activity

Move core tasks to separate module 

Begin abstracting out broker backends by moving core data query tasks
into a module which requires and calls a broker backend API.
kivy_mainline_and_py3.8
Tyler Goodlet 2018-03-20 13:13:07 -04:00
parent 56f38263be
commit bd7eb16ab2
3 changed files with 154 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
piker/brokers/_util.py 100644
View File

@ -0,0 +1,34 @@
"""
Handy utils.
"""
import json
import asks
import logging
from ..log import colorize_json
class BrokerError(Exception):
"Generic broker issue"
def resproc(
resp: asks.response_objects.Response,
log: logging.Logger,
return_json: bool = True
) -> asks.response_objects.Response:
"""Process response and return its json content.
Raise the appropriate error on non-200 OK responses.
"""
if not resp.status_code == 200:
raise BrokerError(resp.body)
try:
data = resp.json()
except json.decoder.JSONDecodeError:
log.exception(f"Failed to process {resp}:\n{resp.text}")
raise BrokerError(resp.text)
else:
log.trace(f"Received json contents:\n{colorize_json(data)}")
return data if return_json else resp

104
piker/brokers/core.py 100644
View File

@ -0,0 +1,104 @@
"""
Core broker-daemon tasks and API.
"""
import time
import inspect
from types import ModuleType
import trio
from .questrade import QuestradeError
from ..log import get_logger
log = get_logger('broker.core')
async def api(brokermod: ModuleType, methname: str, **kwargs) -> dict:
"""Make (proxy through) an api call by name and return its result.
"""
async with brokermod.get_client() as client:
meth = getattr(client.api, methname, None)
if meth is None:
log.error(f"No api method `{methname}` could be found?")
return
elif not kwargs:
# verify kwargs requirements are met
sig = inspect.signature(meth)
if sig.parameters:
log.error(
f"Argument(s) are required by the `{methname}` method: "
f"{tuple(sig.parameters.keys())}")
return
return await meth(**kwargs)
async def quote(brokermod: ModuleType, tickers: [str]) -> dict:
"""Return quotes dict for ``tickers``.
"""
async with brokermod.get_client() as client:
return await client.quote(tickers)
async def poll_tickers(
client: 'Client',
tickers: [str],
q: trio.Queue,
rate: int = 3, # delay between quote requests
diff_cached: bool = True, # only deliver "new" quotes to the queue
) -> None:
"""Stream quotes for a sequence of tickers at the given ``rate``
per second.
"""
t2ids = await client.tickers2ids(tickers)
ids = ','.join(map(str, t2ids.values()))
sleeptime = round(1. / rate, 3)
_cache = {}
while True: # use an event here to trigger exit?
prequote_start = time.time()
try:
quotes_resp = await client.api.quotes(ids=ids)
except QuestradeError as qterr:
if "Access token is invalid" in str(qterr.args[0]):
# out-of-process piker may have renewed already
client._reload_config()
quotes_resp = await client.api.quotes(ids=ids)
else:
raise
postquote_start = time.time()
quotes = quotes_resp['quotes']
payload = []
for quote in quotes:
if quote['delay'] > 0:
log.warning(f"Delayed quote:\n{quote}")
if diff_cached:
# if cache is enabled then only deliver "new" changes
symbol = quote['symbol']
last = _cache.setdefault(symbol, {})
new = set(quote.items()) - set(last.items())
if new:
log.info(
f"New quote {quote['symbol']}:\n{new}")
_cache[symbol] = quote
payload.append(quote)
else:
payload.append(quote)
if payload:
q.put_nowait(payload)
req_time = round(postquote_start - prequote_start, 3)
proc_time = round(time.time() - postquote_start, 3)
tot = req_time + proc_time
log.debug(f"Request + processing took {req_time + proc_time}")
delay = sleeptime - (req_time + proc_time)
if delay <= 0:
log.warn(
f"Took {req_time} (request) + {proc_time} (processing) = {tot}"
f" secs (> {sleeptime}) for processing quotes?")
else:
log.debug(f"Sleeping for {delay}")
await trio.sleep(delay)

158
piker/brokers/questrade.py
View File

@ -1,8 +1,6 @@
"""
Questrade API backend.
"""
import inspect
import json
import time
import datetime
@ -10,6 +8,7 @@ import trio
from async_generator import asynccontextmanager
from . import config
from ._util import resproc, BrokerError
from ..log import get_logger, colorize_json
# TODO: move to urllib3/requests once supported
@ -26,28 +25,6 @@ class QuestradeError(Exception):
"Non-200 OK response code"
def resproc(
resp: asks.response_objects.Response,
return_json: bool = True
) -> asks.response_objects.Response:
"""Process response and return its json content.
Raise the appropriate error on non-200 OK responses.
"""
if not resp.status_code == 200:
raise QuestradeError(resp.body)
try:
data = resp.json()
except json.decoder.JSONDecodeError:
log.exception(f"Failed to process {resp}:\n{resp.text}")
raise QuestradeError(resp.text)
else:
log.trace(f"Received json contents:\n{colorize_json(data)}")
return data if return_json else resp
class Client:
"""API client suitable for use as a long running broker daemon or
single api requests.
@ -80,7 +57,7 @@ class Client:
params={'grant_type': 'refresh_token',
'refresh_token': self.access_data['refresh_token']}
)
data = resproc(resp)
data = resproc(resp, log)
self.access_data.update(data)
return data
@ -121,11 +98,12 @@ class Client:
expires_stamp = datetime.datetime.fromtimestamp(
expires).strftime('%Y-%m-%d %H:%M:%S')
if not access_token or (expires < time.time()) or force_refresh:
log.info(f"Refreshing access token {access_token} which expired at"
f" {expires_stamp}")
log.debug(
f"Refreshing access token {access_token} which expired at"
f" {expires_stamp}")
try:
data = await self._new_auth_token()
except QuestradeError as qterr:
except BrokerError as qterr:
if "We're making some changes" in str(qterr.args[0]):
# API service is down
raise QuestradeError("API is down for maintenance")
@ -135,13 +113,13 @@ class Client:
self._reload_config()
try:
data = await self._new_auth_token()
except QuestradeError as qterr:
except BrokerError as qterr:
if qterr.args[0].decode() == 'Bad Request':
# actually expired; get new from user
self._reload_config(force_from_user=True)
data = await self._new_auth_token()
else:
raise qterr
raise QuestradeError(qterr)
else:
raise qterr
@ -151,8 +129,8 @@ class Client:
# write to config on disk
write_conf(self)
else:
log.info(f"\nCurrent access token {access_token} expires at"
f" {expires_stamp}\n")
log.debug(f"\nCurrent access token {access_token} expires at"
f" {expires_stamp}\n")
self._prep_sess()
return self.access_data
@ -168,12 +146,13 @@ class Client:
return symbols2ids
async def quote(self, tickers):
async def quote(self, tickers: [str]):
"""Return quotes for each ticker in ``tickers``.
"""
t2ids = await self.tickers2ids(tickers)
ids = ','.join(map(str, t2ids.values()))
return (await self.api.quotes(ids=ids))['quotes']
results = (await self.api.quotes(ids=ids))['quotes']
return {sym: quote for sym, quote in zip(tickers, results)}
async def symbols(self, tickers):
"""Return quotes for each ticker in ``tickers``.
@ -196,7 +175,7 @@ class _API:
async def _request(self, path: str, params=None) -> dict:
resp = await self._sess.get(path=f'/{path}', params=params)
return resproc(resp)
return resproc(resp, log)
async def accounts(self) -> dict:
return await self._request('accounts')
@ -268,6 +247,8 @@ def write_conf(client):
@asynccontextmanager
async def get_client() -> Client:
"""Spawn a broker client.
A client must adhere to the method calls in ``piker.broker.core``.
"""
conf = get_config()
log.debug(f"Loaded config:\n{colorize_json(dict(conf['questrade']))}")
@ -290,110 +271,3 @@ async def get_client() -> Client:
yield client
finally:
write_conf(client)
async def serve_forever(tasks) -> None:
"""Start up a client and serve until terminated.
"""
async with get_client() as client:
# pretty sure this doesn't work
# await client._revoke_auth_token()
async with trio.open_nursery() as nursery:
# launch token manager
nursery.start_soon(token_refresher, client)
# launch children
for task in tasks:
nursery.start_soon(task, client)
async def poll_tickers(
client: Client, tickers: [str],
q: trio.Queue,
rate: int = 3, # delay between quote requests
diff_cached: bool = True, # only deliver "new" quotes to the queue
) -> None:
"""Stream quotes for a sequence of tickers at the given ``rate``
per second.
"""
t2ids = await client.tickers2ids(tickers)
ids = ','.join(map(str, t2ids.values()))
sleeptime = round(1. / rate, 3)
_cache = {}
while True: # use an event here to trigger exit?
prequote_start = time.time()
try:
quotes_resp = await client.api.quotes(ids=ids)
except QuestradeError as qterr:
if "Access token is invalid" in str(qterr.args[0]):
# out-of-process piker may have renewed already
client._reload_config()
quotes_resp = await client.api.quotes(ids=ids)
else:
raise
postquote_start = time.time()
quotes = quotes_resp['quotes']
payload = []
for quote in quotes:
if quote['delay'] > 0:
log.warning(f"Delayed quote:\n{quote}")
if diff_cached:
# if cache is enabled then only deliver "new" changes
symbol = quote['symbol']
last = _cache.setdefault(symbol, {})
new = set(quote.items()) - set(last.items())
if new:
log.info(
f"New quote {quote['symbol']}:\n{new}")
_cache[symbol] = quote
payload.append(quote)
else:
payload.append(quote)
if payload:
q.put_nowait(payload)
req_time = round(postquote_start - prequote_start, 3)
proc_time = round(time.time() - postquote_start, 3)
tot = req_time + proc_time
log.debug(f"Request + processing took {req_time + proc_time}")
delay = sleeptime - (req_time + proc_time)
if delay <= 0:
log.warn(
f"Took {req_time} (request) + {proc_time} (processing) = {tot}"
f" secs (> {sleeptime}) for processing quotes?")
else:
log.debug(f"Sleeping for {delay}")
await trio.sleep(delay)
async def api(methname: str, **kwargs) -> dict:
"""Make (proxy through) an api call by name and return its result.
"""
async with get_client() as client:
meth = getattr(client.api, methname, None)
if meth is None:
log.error(f"No api method `{methname}` could be found?")
return
elif not kwargs:
# verify kwargs requirements are met
sig = inspect.signature(meth)
if sig.parameters:
log.error(
f"Argument(s) are required by the `{methname}` method: "
f"{tuple(sig.parameters.keys())}")
return
return await meth(**kwargs)
async def quote(tickers: [str]) -> dict:
"""Return quotes dict for ``tickers``.
"""
async with get_client() as client:
return await client.quote(tickers)