From 6a1a62d8c00edf1167c5d40aab5e80ea56e74ede Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 12 Jul 2022 16:51:06 -0400 Subject: [PATCH 01/14] Add (hacky) forex pair support to `Client.find_contract()` --- piker/brokers/ib/api.py | 69 ++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/piker/brokers/ib/api.py b/piker/brokers/ib/api.py index 207f56f7..7e7fbc92 100644 --- a/piker/brokers/ib/api.py +++ b/piker/brokers/ib/api.py @@ -205,6 +205,19 @@ _adhoc_futes_set = { } +# taken from list here: +# https://www.interactivebrokers.com/en/trading/products-spot-currencies.php +_adhoc_fiat_set = set(( + 'USD, AED, AUD, CAD,' + 'CHF, CNH, CZK, DKK,' + 'EUR, GBP, HKD, HUF,' + 'ILS, JPY, MXN, NOK,' + 'NZD, PLN, RUB, SAR,' + 'SEK, SGD, TRY, ZAR' + ).split(' ,') +) + + # map of symbols to contract ids _adhoc_symbol_map = { # https://misc.interactivebrokers.com/cstools/contract_info/v3.10/index.php?action=Conid%20Info&wlId=IB&conid=69067924 @@ -480,6 +493,14 @@ class Client: except RequestError as err: log.warning(err.message) + # forex pairs + elif sectype == 'CASH': + dst, src = tract.localSymbol.split('.') + pair_key = "/".join([dst, src]) + tract.exchange = 'FOREX' + results[f'{pair_key}.forex'] = tract + results.pop(key) + return results async def get_fute( @@ -521,7 +542,7 @@ class Client: async def find_contract( self, pattern: str, - currency: str = 'USD', + currency: str = '', **kwargs, ) -> Contract: @@ -537,26 +558,44 @@ class Client: # XXX UPDATE: we can probably do the tick/trades scraping # inside our eventkit handler instead to bypass this entirely? + # fqsn parsing stage + # ------------------ if '.ib' in pattern: from ..data._source import unpack_fqsn broker, symbol, expiry = unpack_fqsn(pattern) + else: symbol = pattern - # try: - # # give the cache a go - # return self._contracts[symbol] - # except KeyError: - # log.debug(f'Looking up contract for {symbol}') - expiry: str = '' - if symbol.count('.') > 1: - symbol, _, expiry = symbol.rpartition('.') + # another hack for forex pairs lul. + if ( + '.forex' in symbol + # and not '.ib' in pattern + # or '/' in symbol + ): + exch = 'FOREX' + symbol = symbol.removesuffix('.forex') + if '/' in symbol: + symbol, currency = symbol.split('/') - # use heuristics to figure out contract "type" - sym, exch = symbol.upper().rsplit('.', maxsplit=1) + else: + # try: + # # give the cache a go + # return self._contracts[symbol] + # except KeyError: + # log.debug(f'Looking up contract for {symbol}') + expiry: str = '' + if symbol.count('.') > 1: + symbol, _, expiry = symbol.rpartition('.') + + # use heuristics to figure out contract "type" + sym, exch = symbol.upper().rsplit('.', maxsplit=1) qualify: bool = True + # contract searching stage + # ------------------------ + # futes if exch in _futes_venues: if expiry: @@ -578,10 +617,11 @@ class Client: qualify = False elif exch in ('FOREX'): - currency = '' - symbol, currency = sym.split('/') + # if '/' in symbol: + # currency = '' + # symbol, currency = sym.split('/') con = ibis.Forex( - symbol=symbol, + pair=''.join((symbol, currency)), currency=currency, ) con.bars_kwargs = {'whatToShow': 'MIDPOINT'} @@ -649,6 +689,7 @@ class Client: async def get_sym_details( self, symbol: str, + ) -> tuple[Contract, Ticker, ContractDetails]: contract = await self.find_contract(symbol) From 4e5df973a9a9cc0c5eb48ef418e76353b88c4113 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 12 Jul 2022 16:55:15 -0400 Subject: [PATCH 02/14] Support `Forex` tracts in `normalize()` --- piker/brokers/ib/feed.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/piker/brokers/ib/feed.py b/piker/brokers/ib/feed.py index b22ddc1b..92538965 100644 --- a/piker/brokers/ib/feed.py +++ b/piker/brokers/ib/feed.py @@ -573,7 +573,6 @@ def normalize( con = ticker.contract if type(con) in ( ibis.Commodity, - ibis.Forex, ): # commodities and forex don't have an exchange name and # no real volume so we have to calculate the price @@ -581,6 +580,14 @@ def normalize( # no real volume on this tract calc_price = True + elif type(con) in ( + ibis.Forex, + ): + suffix = 'forex' + con.symbol = con.pair() + # no real volume on forex feeds.. + calc_price = True + else: suffix = con.primaryExchange if not suffix: @@ -812,6 +819,9 @@ async def data_reset_hack( successful. - other OS support? - integration with ``ib-gw`` run in docker + Xorg? + - is it possible to offer a local server that can be accessed by + a client? Would be sure be handy for running native java blobs + that need to be wrangle. ''' From cef511092dad521e47cb2b89a59268e7221dff02 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 12 Jul 2022 16:55:58 -0400 Subject: [PATCH 03/14] Support `Forex` in the pp packer --- piker/brokers/ib/broker.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index 4737d376..bc712677 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -36,6 +36,7 @@ import tractor from ib_insync.contract import ( Contract, Option, + Forex, ) from ib_insync.order import ( Trade, @@ -88,20 +89,29 @@ def pack_position( # TODO: lookup fqsn even for derivs. symbol = con.symbol.lower() + # TODO: probably write a mofo exchange mapper routine since ib + # can't get it's shit together like, ever. + # try our best to figure out the exchange / venue exch = (con.primaryExchange or con.exchange).lower() if not exch: - # for wtv cucked reason some futes don't show their - # exchange (like CL.NYMEX) ... - entry = _adhoc_symbol_map.get( - con.symbol or con.localSymbol - ) - if entry: - meta, kwargs = entry - cid = meta.get('conId') - if cid: - assert con.conId == meta['conId'] - exch = meta['exchange'] + + if isinstance(con, Forex): + # bc apparently it's not in the contract obj? + exch = 'idealfx' + + else: + # for wtv cucked reason some futes don't show their + # exchange (like CL.NYMEX) ... + entry = _adhoc_symbol_map.get( + con.symbol or con.localSymbol + ) + if entry: + meta, kwargs = entry + cid = meta.get('conId') + if cid: + assert con.conId == meta['conId'] + exch = meta['exchange'] assert exch, f'No clue:\n {con}' fqsn = '.'.join((symbol, exch)) From a7f0adf1cf2917a360f76fa75bedb9a5c883a270 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 12 Jul 2022 17:21:39 -0400 Subject: [PATCH 04/14] Make forex rt feeds work again --- piker/brokers/ib/feed.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/piker/brokers/ib/feed.py b/piker/brokers/ib/feed.py index 92538965..29b08669 100644 --- a/piker/brokers/ib/feed.py +++ b/piker/brokers/ib/feed.py @@ -41,7 +41,7 @@ from trio_typing import TaskStatus from piker.data._sharedmem import ShmArray from .._util import SymbolNotFound, NoData from .api import ( - _adhoc_futes_set, + # _adhoc_futes_set, log, load_aio_clients, ibis, @@ -239,7 +239,8 @@ async def get_bars( # elif ( # err.code == 162 and - # 'Trading TWS session is connected from a different IP address' in err.message + # 'Trading TWS session is connected from a different IP + # address' in err.message # ): # log.warning("ignoring ip address warning") # continue @@ -571,6 +572,8 @@ def normalize( # check for special contract types con = ticker.contract + symbol = con.symbol + if type(con) in ( ibis.Commodity, ): @@ -584,7 +587,7 @@ def normalize( ibis.Forex, ): suffix = 'forex' - con.symbol = con.pair() + symbol = con.pair() # no real volume on forex feeds.. calc_price = True @@ -631,7 +634,7 @@ def normalize( # generate fqsn with possible specialized suffix # for derivatives, note the lowercase. data['symbol'] = data['fqsn'] = '.'.join( - (con.symbol, suffix) + (symbol, suffix) ).lower() # convert named tuples to dicts for transport @@ -734,7 +737,13 @@ async def stream_quotes( # it might be outside regular trading hours so see if we can at # least grab history. - if isnan(first_ticker.last): + if ( + isnan(first_ticker.last) + and type(first_ticker.contract) not in ( + ibis.Commodity, + ibis.Forex + ) + ): task_status.started((init_msgs, first_quote)) # it's not really live but this will unblock @@ -936,7 +945,8 @@ async def open_symbol_search( # adhoc_match_results = {} # if adhoc_matches: # # TODO: do we need to pull contract details? - # adhoc_match_results = {i[0]: {} for i in adhoc_matches} + # adhoc_match_results = {i[0]: {} for i in + # adhoc_matches} log.debug(f'fuzzy matching stocks {stock_results}') stock_matches = fuzzy.extractBests( From 04004525c19f402e517cbbfc42c4c905e5e55325 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 13 Jul 2022 12:25:04 -0400 Subject: [PATCH 05/14] Specifically denote no-vlm contracts in symbol info --- piker/brokers/ib/feed.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/piker/brokers/ib/feed.py b/piker/brokers/ib/feed.py index 29b08669..c5c39fa0 100644 --- a/piker/brokers/ib/feed.py +++ b/piker/brokers/ib/feed.py @@ -700,6 +700,13 @@ async def stream_quotes( # TODO: more consistent field translation atype = syminfo['asset_type'] = asset_type_map[syminfo['secType']] + if atype in { + 'forex', + 'index', + 'commodity', + }: + syminfo['no_vlm'] = True + # for stocks it seems TWS reports too small a tick size # such that you can't submit orders with that granularity? min_tick = 0.01 if atype == 'stock' else 0 @@ -726,9 +733,9 @@ async def stream_quotes( }, } - return init_msgs + return init_msgs, syminfo - init_msgs = mk_init_msgs() + init_msgs, syminfo = mk_init_msgs() # TODO: we should instead spawn a task that waits on a feed to start # and let it wait indefinitely..instead of this hard coded stuff. @@ -766,10 +773,14 @@ async def stream_quotes( task_status.started((init_msgs, first_quote)) async with aclosing(stream): - if type(first_ticker.contract) not in ( - ibis.Commodity, - ibis.Forex - ): + if syminfo.get('no_vlm', False): + + # generally speaking these feeds don't + # include vlm data. + atype = syminfo['asset_type'] + log.info(f'Non-volume asset {sym}@{atype}, skipping quote poll.') + + else: # wait for real volume on feed (trading might be closed) while True: ticker = await stream.receive() From 3aa72abacffe24c9b00eaf58d81154545948966a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 13 Jul 2022 12:39:23 -0400 Subject: [PATCH 06/14] Primary exchange can never be "smart" --- piker/brokers/ib/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/piker/brokers/ib/api.py b/piker/brokers/ib/api.py index 7e7fbc92..c868f70c 100644 --- a/piker/brokers/ib/api.py +++ b/piker/brokers/ib/api.py @@ -644,8 +644,10 @@ class Client: exch = 'SMART' else: - exch = 'SMART' + # XXX: order is super important here since + # a primary == 'SMART' won't ever work. primaryExchange = exch + exch = 'SMART' con = ibis.Stock( symbol=sym, From 6ce699ae1f0fa9b078f2c84eb1b290600c27f0a7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 13 Jul 2022 12:40:00 -0400 Subject: [PATCH 07/14] Repair display loop to work when no vlm chart is loaded --- piker/ui/_display.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index d63ee17f..ba58ae5a 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -63,7 +63,7 @@ from ..log import get_logger log = get_logger(__name__) # TODO: load this from a config.toml! -_quote_throttle_rate: int = 22 # Hz +_quote_throttle_rate: int = 60 # Hz # a working tick-type-classes template @@ -136,16 +136,16 @@ class DisplayState: # high level chart handles linked: LinkedSplits chart: ChartPlotWidget - vlm_chart: ChartPlotWidget # axis labels l1: L1Labels last_price_sticky: YAxisLabel - vlm_sticky: YAxisLabel # misc state tracking vars: dict[str, Any] + vlm_chart: Optional[ChartPlotWidget] = None + vlm_sticky: Optional[YAxisLabel] = None wap_in_history: bool = False @@ -185,9 +185,6 @@ async def graphics_update_loop( *ohlcv.array[-1][['index', 'close']] ) - if vlm_chart: - vlm_sticky = vlm_chart._ysticks['volume'] - maxmin = partial( chart_maxmin, chart, @@ -236,8 +233,6 @@ async def graphics_update_loop( 'ohlcv': ohlcv, 'chart': chart, 'last_price_sticky': last_price_sticky, - 'vlm_chart': vlm_chart, - 'vlm_sticky': vlm_sticky, 'l1': l1, 'vars': { @@ -250,6 +245,11 @@ async def graphics_update_loop( } }) + if vlm_chart: + vlm_sticky = vlm_chart._ysticks['volume'] + ds.vlm_chart = vlm_chart + ds.vlm_sticky = vlm_sticky + chart.default_view() # main real-time quotes update loop @@ -322,7 +322,7 @@ def graphics_update_cycle( for sym, quote in ds.quotes.items(): # compute the first available graphic's x-units-per-pixel - uppx = vlm_chart.view.x_uppx() + uppx = chart.view.x_uppx() # NOTE: vlm may be written by the ``brokerd`` backend # event though a tick sample is not emitted. @@ -786,7 +786,10 @@ async def display_symbol_data( async with trio.open_nursery() as ln: # if available load volume related built-in display(s) - if has_vlm(ohlcv): + if ( + not symbol.broker_info[provider].get('no_vlm', False) + and has_vlm(ohlcv) + ): vlm_chart = await ln.start( open_vlm_displays, linked, From 0580b204a3a280d72ab3bb66eb88536af3451292 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 13 Jul 2022 13:48:54 -0400 Subject: [PATCH 08/14] A `size` field in ticks is optional --- piker/data/_normalize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/data/_normalize.py b/piker/data/_normalize.py index 677468ad..13708252 100644 --- a/piker/data/_normalize.py +++ b/piker/data/_normalize.py @@ -56,7 +56,7 @@ def iterticks( sig = ( time, tick['price'], - tick['size'] + tick.get('size') ) if ttype == 'dark_trade': From 0ef5da0881ca5a11be21c2e92faa22af251e4b7b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 15 Jul 2022 15:27:51 -0400 Subject: [PATCH 09/14] Unbreak regular searches and stock lookups.. Change `.find_contract()` -> `.find_contracts()` to allow multi-search for so called "ambiguous" contracts (like for `Future`s) such that the method now returns a `list` of tracts and populates the contract cache with all specific tracts retrieved. Let it take in an (unvalidated) contract that will be fqsn-style-tokenized such that it can be called from `.search_symbols()` (though we're not quite yet XD). More stuff, - add `Client.parse_patt2fqsn()` which is an fqsn to token unpacker built from the original logic in the old `.find_contract()`. - handle fiat/forex pairs with the `'CASH'` sectype. - add a flag to allow unqualified contracts to fail with a warning msg. - populate the client's contract cache with all expiries of an ambiguous derivative. - allow `.con_deats()` to warn msg instead of raise on def-not-found. - add commented `assert 0` which was triggering a debugger deadlock in `tractor` which we still haven't been able to create a unit test for. --- piker/brokers/ib/api.py | 164 ++++++++++++++++++++++++++++------------ 1 file changed, 115 insertions(+), 49 deletions(-) diff --git a/piker/brokers/ib/api.py b/piker/brokers/ib/api.py index c868f70c..6d214d22 100644 --- a/piker/brokers/ib/api.py +++ b/piker/brokers/ib/api.py @@ -29,6 +29,7 @@ import itertools from math import isnan from typing import ( Any, + Optional, Union, ) import asyncio @@ -43,8 +44,10 @@ import trio import tractor from tractor import to_asyncio import ib_insync as ibis -from ib_insync.wrapper import RequestError -from ib_insync.contract import Contract, ContractDetails +from ib_insync.contract import ( + Contract, + ContractDetails, +) from ib_insync.order import Order from ib_insync.ticker import Ticker from ib_insync.objects import ( @@ -53,7 +56,10 @@ from ib_insync.objects import ( Execution, CommissionReport, ) -from ib_insync.wrapper import Wrapper +from ib_insync.wrapper import ( + Wrapper, + RequestError, +) from ib_insync.client import Client as ib_Client import numpy as np @@ -184,12 +190,12 @@ _adhoc_futes_set = { 'ethusdrr.cmecrypto', # agriculture - 'he.globex', # lean hogs - 'le.globex', # live cattle (geezers) - 'gf.globex', # feeder cattle (younguns) + 'he.nymex', # lean hogs + 'le.nymex', # live cattle (geezers) + 'gf.nymex', # feeder cattle (younguns) # raw - 'lb.globex', # random len lumber + 'lb.nymex', # random len lumber # metals 'xauusd.cmdty', # gold spot @@ -247,6 +253,7 @@ _exch_skip_list = { 'VALUE', 'FUNDSERV', 'SWB2', + 'PSE', } # https://misc.interactivebrokers.com/cstools/contract_info/v3.10/index.php?action=Conid%20Info&wlId=IB&conid=69067924 @@ -349,7 +356,7 @@ class Client: _enters += 1 - contract = await self.find_contract(fqsn) + contract = (await self.find_contracts(fqsn))[0] bars_kwargs.update(getattr(contract, 'bars_kwargs', {})) # _min = min(2000*100, count) @@ -404,12 +411,19 @@ class Client: futs.append(self.ib.reqContractDetailsAsync(con)) # batch request all details - results = await asyncio.gather(*futs) + try: + results = await asyncio.gather(*futs) + except RequestError as err: + msg = err.message + if ( + 'No security definition' in msg + ): + log.warning(f'{msg}: {contracts}') + return {} # one set per future result details = {} for details_set in results: - # XXX: if there is more then one entry in the details list # then the contract is so called "ambiguous". for d in details_set: @@ -477,21 +491,32 @@ class Client: if sectype == 'IND': results[f'{sym}.IND'] = tract results.pop(key) - exch = tract.exchange + # exch = tract.exchange - if exch in _futes_venues: + # XXX: add back one of these to get the weird deadlock + # on the debugger from root without the latest + # maybe_wait_for_debugger() fix in the `open_context()` + # exit. + # assert 0 + # if con.exchange not in _exch_skip_list: + + exch = tract.exchange + if exch not in _exch_skip_list: # try get all possible contracts for symbol as per, # https://interactivebrokers.github.io/tws-api/basic_contracts.html#fut con = ibis.Future( symbol=sym, exchange=exch, ) - try: - all_deats = await self.con_deats([con]) - results |= all_deats - - except RequestError as err: - log.warning(err.message) + # TODO: make this work, think it's something to do + # with the qualify flag. + # cons = await self.find_contracts( + # contract=con, + # err_on_qualify=False, + # ) + # if cons: + all_deats = await self.con_deats([con]) + results |= all_deats # forex pairs elif sectype == 'CASH': @@ -539,13 +564,11 @@ class Client: ibis.Contract(conId=conid) ) - async def find_contract( + def parse_patt2fqsn( self, pattern: str, - currency: str = '', - **kwargs, - ) -> Contract: + ) -> tuple[str, str, str, str]: # TODO: we can't use this currently because # ``wrapper.starTicker()`` currently cashes ticker instances @@ -558,14 +581,17 @@ class Client: # XXX UPDATE: we can probably do the tick/trades scraping # inside our eventkit handler instead to bypass this entirely? + currency = '' + # fqsn parsing stage # ------------------ if '.ib' in pattern: from ..data._source import unpack_fqsn - broker, symbol, expiry = unpack_fqsn(pattern) + _, symbol, expiry = unpack_fqsn(pattern) else: symbol = pattern + expiry = '' # another hack for forex pairs lul. if ( @@ -579,6 +605,7 @@ class Client: symbol, currency = symbol.split('/') else: + # TODO: yes, a cache.. # try: # # give the cache a go # return self._contracts[symbol] @@ -589,9 +616,32 @@ class Client: symbol, _, expiry = symbol.rpartition('.') # use heuristics to figure out contract "type" - sym, exch = symbol.upper().rsplit('.', maxsplit=1) + symbol, exch = symbol.upper().rsplit('.', maxsplit=1) - qualify: bool = True + return symbol, currency, exch, expiry + + async def find_contracts( + self, + pattern: Optional[str] = None, + contract: Optional[Contract] = None, + qualify: bool = True, + err_on_qualify: bool = True, + + ) -> Contract: + + if pattern is not None: + symbol, currency, exch, expiry = self.parse_patt2fqsn( + pattern, + ) + sectype = '' + + else: + assert contract + symbol = contract.symbol + sectype = contract.secType + exch = contract.exchange or contract.primaryExchange + expiry = contract.lastTradeDateOrContractMonth + currency = contract.currency # contract searching stage # ------------------------ @@ -600,26 +650,27 @@ class Client: if exch in _futes_venues: if expiry: # get the "front" contract - contract = await self.get_fute( - symbol=sym, + con = await self.get_fute( + symbol=symbol, exchange=exch, expiry=expiry, ) else: # get the "front" contract - contract = await self.get_fute( - symbol=sym, + con = await self.get_fute( + symbol=symbol, exchange=exch, front=True, ) - qualify = False - - elif exch in ('FOREX'): + elif ( + exch in ('FOREX') + or sectype == 'CASH' + ): # if '/' in symbol: # currency = '' - # symbol, currency = sym.split('/') + # symbol, currency = symbol.split('/') con = ibis.Forex( pair=''.join((symbol, currency)), currency=currency, @@ -628,7 +679,7 @@ class Client: # commodities elif exch == 'CMDTY': # eg. XAUUSD.CMDTY - con_kwargs, bars_kwargs = _adhoc_symbol_map[sym] + con_kwargs, bars_kwargs = _adhoc_symbol_map[symbol] con = ibis.Commodity(**con_kwargs) con.bars_kwargs = bars_kwargs @@ -650,29 +701,44 @@ class Client: exch = 'SMART' con = ibis.Stock( - symbol=sym, + symbol=symbol, exchange=exch, primaryExchange=primaryExchange, currency=currency, ) - try: exch = 'SMART' if not exch else exch - if qualify: - contract = (await self.ib.qualifyContractsAsync(con))[0] - else: - assert contract - except IndexError: - raise ValueError(f"No contract could be found {con}") + contracts = [con] + if qualify: + try: + contracts = await self.ib.qualifyContractsAsync(con) + except RequestError as err: + msg = err.message + if ( + 'No security definition' in msg + and not err_on_qualify + ): + log.warning( + f'Could not find def for {con}') + return None - self._contracts[pattern] = contract + else: + raise + if not contracts: + raise ValueError(f"No contract could be found {con}") - # add an aditional entry with expiry suffix if available - conexp = contract.lastTradeDateOrContractMonth - if conexp: - self._contracts[pattern + f'.{conexp}'] = contract + # pack all contracts into cache + for tract in contracts: + exch: str = tract.primaryExchange or tract.exchange or exch + pattern = f'{symbol}.{exch}' + expiry = tract.lastTradeDateOrContractMonth + # add an entry with expiry suffix if available + if expiry: + pattern += f'.{expiry}' - return contract + self._contracts[pattern.lower()] = tract + + return contracts async def get_head_time( self, @@ -694,7 +760,7 @@ class Client: ) -> tuple[Contract, Ticker, ContractDetails]: - contract = await self.find_contract(symbol) + contract = (await self.find_contracts(symbol))[0] ticker: Ticker = self.ib.reqMktData( contract, snapshot=True, From 070b9f3dc1fe2967c044b9eac4051017b24d3714 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 15 Jul 2022 16:27:06 -0400 Subject: [PATCH 10/14] Log msg tweak --- piker/brokers/ib/feed.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/piker/brokers/ib/feed.py b/piker/brokers/ib/feed.py index c5c39fa0..0619011a 100644 --- a/piker/brokers/ib/feed.py +++ b/piker/brokers/ib/feed.py @@ -207,8 +207,6 @@ async def get_bars( except RequestError as err: msg = err.message - # why do we always need to rebind this? - # _err = err if 'No market data permissions for' in msg: # TODO: signalling for no permissions searches @@ -778,7 +776,9 @@ async def stream_quotes( # generally speaking these feeds don't # include vlm data. atype = syminfo['asset_type'] - log.info(f'Non-volume asset {sym}@{atype}, skipping quote poll.') + log.info( + f'Non-vlm asset {sym}@{atype}, skipping quote poll...' + ) else: # wait for real volume on feed (trading might be closed) From a27431c34f7307544cc4c28ae6c7369024463db2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 19 Jul 2022 16:54:45 -0400 Subject: [PATCH 11/14] Unify contract->fqsn translation with new cached-helper --- piker/brokers/ib/api.py | 87 +++++++++++++++++++++++++++++++--------- piker/brokers/ib/feed.py | 40 +++--------------- 2 files changed, 72 insertions(+), 55 deletions(-) diff --git a/piker/brokers/ib/api.py b/piker/brokers/ib/api.py index 6d214d22..fa9a7b72 100644 --- a/piker/brokers/ib/api.py +++ b/piker/brokers/ib/api.py @@ -424,23 +424,15 @@ class Client: # one set per future result details = {} for details_set in results: + # XXX: if there is more then one entry in the details list # then the contract is so called "ambiguous". for d in details_set: - con = d.contract - key = '.'.join([ - con.symbol, - con.primaryExchange or con.exchange, - ]) - expiry = con.lastTradeDateOrContractMonth - if expiry: - key += f'.{expiry}' - - # nested dataclass we probably don't need and that - # won't IPC serialize.. + # nested dataclass we probably don't need and that won't + # IPC serialize.. d.secIdList = '' - + key, calc_price = con2fqsn(d.contract) details[key] = d return details @@ -470,7 +462,7 @@ class Client: self, pattern: str, # how many contracts to search "up to" - upto: int = 3, + upto: int = 6, asdicts: bool = True, ) -> dict[str, ContractDetails]: @@ -522,10 +514,14 @@ class Client: elif sectype == 'CASH': dst, src = tract.localSymbol.split('.') pair_key = "/".join([dst, src]) - tract.exchange = 'FOREX' - results[f'{pair_key}.forex'] = tract + exch = tract.exchange.lower() + results[f'{pair_key}.{exch}'] = tract results.pop(key) + # XXX: again seems to trigger the weird tractor + # bug with the debugger.. + # assert 0 + return results async def get_fute( @@ -595,12 +591,11 @@ class Client: # another hack for forex pairs lul. if ( - '.forex' in symbol - # and not '.ib' in pattern + '.idealpro' in symbol # or '/' in symbol ): - exch = 'FOREX' - symbol = symbol.removesuffix('.forex') + exch = 'IDEALPRO' + symbol = symbol.removesuffix('.idealpro') if '/' in symbol: symbol, currency = symbol.split('/') @@ -665,7 +660,7 @@ class Client: ) elif ( - exch in ('FOREX') + exch in ('IDEALPRO') or sectype == 'CASH' ): # if '/' in symbol: @@ -948,6 +943,58 @@ class Client: return self.ib.positions(account=account) +def con2fqsn( + con: Contract, + _cache: dict[int, (str, bool)] = {} + +) -> tuple[str, bool]: + ''' + Convert contracts to fqsn-style strings to be used both in symbol-search + matching and as feed tokens passed to the front end data deed layer. + + Previously seen contracts are cached by id. + + ''' + # should be real volume for this contract by default + calc_price = False + if con.conId: + try: + return _cache[con.conId] + except KeyError: + pass + + suffix = con.primaryExchange or con.exchange + symbol = con.symbol + expiry = con.lastTradeDateOrContractMonth or '' + + match con: + case ibis.Commodity(): + # commodities and forex don't have an exchange name and + # no real volume so we have to calculate the price + suffix = con.secType + + # no real volume on this tract + calc_price = True + + case ibis.Forex() | ibis.Contract(secType='CASH'): + dst, src = con.localSymbol.split('.') + symbol = ''.join([dst, src]) + suffix = con.exchange + + # no real volume on forex feeds.. + calc_price = True + + # append a `.` to the returned symbol + # key for derivatives that normally is the expiry + # date key. + if expiry: + suffix += f'.{expiry}' + + fqsn_key = '.'.join((symbol, suffix)).lower() + _cache[con.conId] = fqsn_key, calc_price + return fqsn_key, calc_price + + # per-actor API ep caching _client_cache: dict[tuple[str, int], Client] = {} _scan_ignore: set[tuple[str, int]] = set() diff --git a/piker/brokers/ib/feed.py b/piker/brokers/ib/feed.py index 0619011a..c3ac985f 100644 --- a/piker/brokers/ib/feed.py +++ b/piker/brokers/ib/feed.py @@ -42,6 +42,7 @@ from piker.data._sharedmem import ShmArray from .._util import SymbolNotFound, NoData from .api import ( # _adhoc_futes_set, + con2fqsn, log, load_aio_clients, ibis, @@ -559,47 +560,18 @@ async def open_aio_quote_stream( # TODO: cython/mypyc/numba this! +# or we can at least cache a majority of the values +# except for the ones we expect to change?.. def normalize( ticker: Ticker, calc_price: bool = False ) -> dict: - # should be real volume for this contract by default - calc_price = False - # check for special contract types con = ticker.contract - symbol = con.symbol - if type(con) in ( - ibis.Commodity, - ): - # commodities and forex don't have an exchange name and - # no real volume so we have to calculate the price - suffix = con.secType - # no real volume on this tract - calc_price = True - - elif type(con) in ( - ibis.Forex, - ): - suffix = 'forex' - symbol = con.pair() - # no real volume on forex feeds.. - calc_price = True - - else: - suffix = con.primaryExchange - if not suffix: - suffix = con.exchange - - # append a `.` to the returned symbol - # key for derivatives that normally is the expiry - # date key. - expiry = con.lastTradeDateOrContractMonth - if expiry: - suffix += f'.{expiry}' + fqsn, calc_price = con2fqsn(con) # convert named tuples to dicts so we send usable keys new_ticks = [] @@ -631,9 +603,7 @@ def normalize( # generate fqsn with possible specialized suffix # for derivatives, note the lowercase. - data['symbol'] = data['fqsn'] = '.'.join( - (symbol, suffix) - ).lower() + data['symbol'] = data['fqsn'] = fqsn # convert named tuples to dicts for transport tbts = data.get('tickByTicks') From fb39da19f4fec6383b99c471dc0cfcd66dae7d04 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 21 Jul 2022 10:14:29 -0400 Subject: [PATCH 12/14] Add option and adhoc meta-info support to `con2fqsn()` --- piker/brokers/ib/api.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/piker/brokers/ib/api.py b/piker/brokers/ib/api.py index fa9a7b72..af99e9ad 100644 --- a/piker/brokers/ib/api.py +++ b/piker/brokers/ib/api.py @@ -47,6 +47,7 @@ import ib_insync as ibis from ib_insync.contract import ( Contract, ContractDetails, + Option, ) from ib_insync.order import Order from ib_insync.ticker import Ticker @@ -473,7 +474,6 @@ class Client: pattern, upto=upto, ) - for key, deats in results.copy().items(): tract = deats.contract @@ -968,6 +968,10 @@ def con2fqsn( expiry = con.lastTradeDateOrContractMonth or '' match con: + case Option(): + # TODO: option symbol parsing and sane display: + symbol = con.localSymbol.replace(' ', '') + case ibis.Commodity(): # commodities and forex don't have an exchange name and # no real volume so we have to calculate the price @@ -984,6 +988,17 @@ def con2fqsn( # no real volume on forex feeds.. calc_price = True + if not suffix: + entry = _adhoc_symbol_map.get( + con.symbol or con.localSymbol + ) + if entry: + meta, kwargs = entry + cid = meta.get('conId') + if cid: + assert con.conId == meta['conId'] + suffix = meta['exchange'] + # append a `.` to the returned symbol # key for derivatives that normally is the expiry # date key. From 2393965e833886cb4ea5f125db593ef775bb6b50 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 21 Jul 2022 11:37:18 -0400 Subject: [PATCH 13/14] Fix bottom axis when no fsps/subplots --- piker/ui/_chart.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 688b97eb..3231698b 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -453,13 +453,6 @@ class LinkedSplits(QWidget): # add crosshair graphic self.chart.addItem(self.cursor) - # axis placement - if ( - _xaxis_at == 'bottom' and - 'bottom' in self.chart.plotItem.axes - ): - self.chart.hideAxis('bottom') - # style? self.chart.setFrameStyle( QFrame.StyledPanel | @@ -524,21 +517,26 @@ class LinkedSplits(QWidget): cpw.hideAxis('left') cpw.hideAxis('bottom') - if self.xaxis_chart: - self.xaxis_chart.hideAxis('bottom') + if ( + _xaxis_at == 'bottom' and ( + self.xaxis_chart + or ( + not self.subplots + and self.xaxis_chart is None + ) + ) + ): + if self.xaxis_chart: + self.xaxis_chart.hideAxis('bottom') # presuming we only want it at the true bottom of all charts. # XXX: uses new api from our ``pyqtgraph`` fork. # https://github.com/pikers/pyqtgraph/tree/plotitemoverlay_onto_pg_master # _ = self.xaxis_chart.removeAxis('bottom', unlink=False) # assert 'bottom' not in self.xaxis_chart.plotItem.axes - self.xaxis_chart = cpw cpw.showAxis('bottom') - if self.xaxis_chart is None: - self.xaxis_chart = cpw - qframe.chart = cpw qframe.hbox.addWidget(cpw) From e30a3c5b546b39d411fd42eedbcfcb06dd06c957 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 21 Jul 2022 11:38:07 -0400 Subject: [PATCH 14/14] Single chart requires view reset to size to data on startup --- piker/ui/_display.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index ba58ae5a..f79c56ae 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -824,6 +824,9 @@ async def display_symbol_data( order_mode_started ) ): + if not vlm_chart: + chart.default_view() + # let Qt run to render all widgets and make sure the # sidepanes line up vertically. await trio.sleep(0)