From 0790ae8235d910a7d10c367f6c042ccd15c4ed8c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 8 Apr 2022 11:47:22 -0400 Subject: [PATCH] Limit ohlc queries to 800k datums to avoid `purepc` size error --- piker/data/marketstore.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/piker/data/marketstore.py b/piker/data/marketstore.py index a1a54edb..f82770ab 100644 --- a/piker/data/marketstore.py +++ b/piker/data/marketstore.py @@ -258,16 +258,29 @@ class Storage: if fqsn not in syms: return {} + tfstr = tf_in_1s[1] + + params = Params( + symbols=fqsn, + timeframe=tfstr, + attrgroup='OHLCV', + # limit_from_start=True, + + # TODO: figure the max limit here given the + # ``purepc`` msg size limit of purerpc: 33554432 + limit=int(800e3), + ) + if timeframe is None: log.info(f'starting {fqsn} tsdb granularity scan..') # loop through and try to find highest granularity for tfstr in tf_in_1s.values(): try: log.info(f'querying for {tfstr}@{fqsn}') - result = await client.query( - Params(fqsn, tfstr, 'OHLCV',) - ) + params.set('timeframe', tfstr) + result = await client.query(params) break + except purerpc.grpclib.exceptions.UnknownError: # XXX: this is already logged by the container and # thus shows up through `marketstored` logs relay. @@ -277,8 +290,7 @@ class Storage: return {} else: - tfstr = tf_in_1s[timeframe] - result = await client.query(Params(fqsn, tfstr, 'OHLCV',)) + result = await client.query(params) # TODO: it turns out column access on recarrays is actually slower: # https://jakevdp.github.io/PythonDataScienceHandbook/02.09-structured-data-numpy.html#RecordArrays:-Structured-Arrays-with-a-Twist