From 3a4794e9d1c9215a03fc7ce3a1e5bc0571730a07 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Mar 2023 19:03:40 -0500 Subject: [PATCH 1/3] Backward-compat: don't require `'lot_tick_size'` In order to support existing `pps.toml` files in the wild which don't have the `asset_type, price_tick_size, lot_tick_size` fields, we need to only optionally read them and instead expect that backends will write the fields going forward (coming in follow patches). Further this makes some small asset-size (vlm accounting) quantization related adjustments: - rename `Symbol.decimal_quant()` -> `.quantize_size()` since that is explicitly what this method is doing. - and expect an input `size: float` which we cast to decimal instead of doing it inside the `.calc_size()` caller code. - drop `Symbol.iterfqsns()` which wasn't being used anywhere at all.. Additionally, this drafts out a new replacement market-trading-pair data type to eventually replace `.data._source.Symbol` -> `MktPair` which we aren't using yet, but serves as the documentation-driven motivator ;) and, it relates to https://github.com/pikers/piker/issues/467. --- piker/data/_source.py | 83 +++++++++++++++++++++++++++++++++++-------- piker/pp.py | 30 +++++++++++----- 2 files changed, 89 insertions(+), 24 deletions(-) diff --git a/piker/data/_source.py b/piker/data/_source.py index e8f09484..d358cd96 100644 --- a/piker/data/_source.py +++ b/piker/data/_source.py @@ -18,7 +18,10 @@ numpy data source coversion helpers. """ from __future__ import annotations -from decimal import Decimal, ROUND_HALF_EVEN +from decimal import ( + Decimal, + ROUND_HALF_EVEN, +) from typing import Any from bidict import bidict @@ -77,6 +80,10 @@ def mk_fqsn( def float_digits( value: float, ) -> int: + ''' + Return the number of precision digits read from a float value. + + ''' if value == 0: return 0 @@ -127,6 +134,56 @@ def unpack_fqsn(fqsn: str) -> tuple[str, str, str]: ) +class MktPair(Struct, frozen=True): + + src: str # source asset name being used to buy + src_type: str # source asset's financial type/classification name + # ^ specifies a "class" of financial instrument + # egs. stock, futer, option, bond etc. + + dst: str # destination asset name being bought + dst_type: str # destination asset's financial type/classification name + + price_tick: float # minimum price increment value increment + price_tick_digits: int # required decimal digits for above + + size_tick: float # minimum size (aka vlm) increment value increment + size_tick_digits: int # required decimal digits for above + + venue: str | None = None # market venue provider name + expiry: str | None = None # for derivs, expiry datetime parseable str + + # for derivs, info describing contract, egs. + # strike price, call or put, swap type, exercise model, etc. + contract_info: str | None = None + + @classmethod + def from_msg( + self, + msg: dict[str, Any], + + ) -> MktPair: + ''' + Constructor for a received msg-dict normally received over IPC. + + ''' + ... + + # fqa, fqma, .. etc. see issue: + # https://github.com/pikers/piker/issues/467 + @property + def fqsn(self) -> str: + ''' + Return the fully qualified market (endpoint) name for the + pair of transacting assets. + + ''' + ... + + +# TODO: rework the below `Symbol` (which was originally inspired and +# derived from stuff in quantdom) into a simpler, ipc msg ready, market +# endpoint meta-data container type as per the drafted interace above. class Symbol(Struct): ''' I guess this is some kinda container thing for dealing with @@ -141,10 +198,6 @@ class Symbol(Struct): suffix: str = '' broker_info: dict[str, dict[str, Any]] = {} - # specifies a "class" of financial instrument - # ex. stock, futer, option, bond etc. - - # @validate_arguments @classmethod def from_broker_info( cls, @@ -244,23 +297,23 @@ class Symbol(Struct): fqsn = '.'.join(map(str.lower, tokens)) return fqsn - def iterfqsns(self) -> list[str]: - keys = [] - for broker in self.broker_info.keys(): - fqsn = mk_fqsn(self.key, broker) - if self.suffix: - fqsn += f'.{self.suffix}' - keys.append(fqsn) + def quantize_size( + self, + size: float, - return keys + ) -> Decimal: + ''' + Truncate input ``size: float`` using ``Decimal`` + and ``.lot_size_digits``. - def decimal_quant(self, d: Decimal): + ''' digits = self.lot_size_digits - return d.quantize( + return Decimal(size).quantize( Decimal(f'1.{"0".ljust(digits, "0")}'), rounding=ROUND_HALF_EVEN ) + def _nan_to_closest_num(array: np.ndarray): """Return interpolated values instead of NaN. diff --git a/piker/pp.py b/piker/pp.py index 1a2f5e6b..38ff1566 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -22,8 +22,6 @@ that doesn't try to cuk most humans who prefer to not lose their moneys.. ''' from __future__ import annotations from contextlib import contextmanager as cm -from pathlib import Path -from decimal import Decimal, ROUND_HALF_EVEN from pprint import pformat import os from os import path @@ -91,10 +89,12 @@ def open_trade_ledger( yield cpy finally: if cpy != ledger: + # TODO: show diff output? # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries log.info(f'Updating ledger for {tradesfile}:\n') - ledger.update(cpy) + ledger.update(cpy) + # we write on close the mutated ledger data with open(tradesfile, 'w') as cf: toml.dump(ledger, cf) @@ -476,7 +476,7 @@ class Position(Struct): if self.split_ratio is not None: size = round(size * self.split_ratio) - return float(self.symbol.decimal_quant(Decimal(size))) + return float(self.symbol.quantize_size(size)) def minimize_clears( self, @@ -934,10 +934,18 @@ def open_pps( for fqsn, entry in pps.items(): bsuid = entry['bsuid'] symbol = Symbol.from_fqsn( - fqsn, info={ - 'asset_type': entry['asset_type'], - 'price_tick_size': entry['price_tick_size'], - 'lot_tick_size': entry['lot_tick_size'] + fqsn, + + # NOTE & TODO: right now we fill in the defaults from + # `.data._source.Symbol` but eventually these should always + # either be already written to the pos table or provided at + # write time to ensure always having these values somewhere + # and thus allowing us to get our pos sizing precision + # correct! + info={ + 'asset_type': entry.get('asset_type', ''), + 'price_tick_size': entry.get('price_tick_size', 0.01), + 'lot_tick_size': entry.get('lot_tick_size', 0.0), } ) @@ -977,7 +985,11 @@ def open_pps( size = entry['size'] # TODO: remove but, handle old field name for now - ppu = entry.get('ppu', entry.get('be_price', 0)) + ppu = entry.get( + 'ppu', + entry.get('be_price', 0), + ) + split_ratio = entry.get('split_ratio') expiry = entry.get('expiry') From 69b85aa7e588452158bdd823254c311353e8d014 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Mar 2023 19:23:47 -0500 Subject: [PATCH 2/3] `ib`: parse and load info for new `Transaction.sym: Symbol` field --- piker/brokers/ib/broker.py | 63 ++++++++++++++++++++++++++++++++------ piker/brokers/ib/feed.py | 2 +- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index 4537e7f0..56756a76 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -70,7 +70,10 @@ from piker.clearing._messages import ( BrokerdFill, BrokerdError, ) -from piker.data._source import Symbol +from piker.data._source import ( + Symbol, + float_digits, +) from .api import ( _accounts2clients, con2fqsn, @@ -304,6 +307,9 @@ async def update_ledger_from_api_trades( entry['listingExchange'] = pexch + # pack in the ``Contract.secType`` + entry['asset_type'] = condict['secType'] + conf = get_config() entries = api_trades_to_ledger_entries( conf['accounts'].inverse, @@ -616,9 +622,10 @@ async def trades_dialogue( # from the api trades it seems we get a key # error from ``update[bsuid]`` ? pp = table.pps[bsuid] + pairinfo = pp.symbol if msg.size != pp.size: log.error( - f'Position mismatch {pp.symbol.front_fqsn()}:\n' + f'Pos size mismatch {pairinfo.front_fqsn()}:\n' f'ib: {msg.size}\n' f'piker: {pp.size}\n' ) @@ -1095,13 +1102,15 @@ def norm_trade_records( ''' records: list[Transaction] = [] - for tid, record in ledger.items(): + for tid, record in ledger.items(): conid = record.get('conId') or record['conid'] comms = record.get('commission') if comms is None: comms = -1*record['ibCommission'] + price = record.get('price') or record['tradePrice'] + price_tick_digits = float_digits(price) # the api doesn't do the -/+ on the quantity for you but flex # records do.. are you fucking serious ib...!? @@ -1144,9 +1153,14 @@ def norm_trade_records( # special handling of symbol extraction from # flex records using some ad-hoc schema parsing. - instr = record.get('assetCategory') - if instr == 'FUT': - symbol = record['description'][:3] + asset_type: str = record.get('assetCategory') or record['secType'] + + # TODO: XXX: WOA this is kinda hacky.. probably + # should figure out the correct future pair key more + # explicitly and consistently? + if asset_type == 'FUT': + # (flex) ledger entries don't have any simple 3-char key? + symbol = record['symbol'][:3] # try to build out piker fqsn from record. expiry = record.get( @@ -1156,10 +1170,34 @@ def norm_trade_records( suffix = f'{exch}.{expiry}' expiry = pendulum.parse(expiry) - fqsn = Symbol.from_fqsn( + src: str = record['currency'] + + pair = Symbol.from_fqsn( fqsn=f'{symbol}.{suffix}.ib', - info={}, - ).front_fqsn().rstrip('.ib') + info={ + 'tick_size_digits': price_tick_digits, + + # NOTE: for "legacy" assets, volume is normally discreet, not + # a float, but we keep a digit in case the suitz decide + # to get crazy and change it; we'll be kinda ready + # schema-wise.. + 'lot_size_digits': 1, + + # TODO: remove when we switching from + # ``Symbol`` -> ``MktPair`` + 'asset_type': asset_type, + + # TODO: figure out a target fin-type name + # set and normalize to that here! + 'dst_type': asset_type.lower(), + + # starting to use new key naming as in ``MktPair`` + # type have drafted... + 'src': src, + 'src_type': 'fiat', + }, + ) + fqsn = pair.front_fqsn().rstrip('.ib') # NOTE: for flex records the normal fields for defining an fqsn # sometimes won't be available so we rely on two approaches for @@ -1175,6 +1213,7 @@ def norm_trade_records( records, Transaction( fqsn=fqsn, + sym=pair, tid=tid, size=size, price=price, @@ -1201,7 +1240,11 @@ def parse_flex_dt( def api_trades_to_ledger_entries( accounts: bidict, - trade_entries: list[object], + + # TODO: maybe we should just be passing through the + # ``ib_insync.order.Trade`` instance directly here + # instead of pre-casting to dicts? + trade_entries: list[dict], ) -> dict: ''' diff --git a/piker/brokers/ib/feed.py b/piker/brokers/ib/feed.py index bee69ae6..345e4ade 100644 --- a/piker/brokers/ib/feed.py +++ b/piker/brokers/ib/feed.py @@ -770,7 +770,7 @@ async def stream_quotes( syminfo['price_tick_size'] = max(syminfo['minTick'], min_tick) - # for "traditional" assets, volume is normally discreet, not + # for "legacy" assets, volume is normally discreet, not # a float syminfo['lot_tick_size'] = 0.0 From b4a1cc8f22f2079ab6b0087aee8b5410c8100c03 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Mar 2023 19:25:43 -0500 Subject: [PATCH 3/3] `kraken`: parse and load info `Transaction.sym: Symbol` Also includes a retyping of `Client._pair: dict[str, Pair]` to look up pair structs and map all alt-key-name-sets to each for easy precision info lookup to set the `.sym` field for each transaction including for on-chain transfers which kraken provides as an "asset decimals" field, presumably pulled from the particular block-token's limitation info. --- piker/brokers/kraken/api.py | 131 ++++++++++++++++++++++++++------- piker/brokers/kraken/broker.py | 16 +++- piker/brokers/kraken/feed.py | 59 +++------------ 3 files changed, 129 insertions(+), 77 deletions(-) diff --git a/piker/brokers/kraken/api.py b/piker/brokers/kraken/api.py index d590148f..94d6dc41 100644 --- a/piker/brokers/kraken/api.py +++ b/piker/brokers/kraken/api.py @@ -40,6 +40,8 @@ import base64 import trio from piker import config +from piker.data.types import Struct +from piker.data._source import Symbol from piker.brokers._util import ( resproc, SymbolNotFound, @@ -113,11 +115,53 @@ class InvalidKey(ValueError): ''' +# https://www.kraken.com/features/api#get-tradable-pairs +class Pair(Struct): + altname: str # alternate pair name + wsname: str # WebSocket pair name (if available) + aclass_base: str # asset class of base component + base: str # asset id of base component + aclass_quote: str # asset class of quote component + quote: str # asset id of quote component + lot: str # volume lot size + + cost_decimals: int + costmin: float + pair_decimals: int # scaling decimal places for pair + lot_decimals: int # scaling decimal places for volume + + # amount to multiply lot volume by to get currency volume + lot_multiplier: float + + # array of leverage amounts available when buying + leverage_buy: list[int] + # array of leverage amounts available when selling + leverage_sell: list[int] + + # fee schedule array in [volume, percent fee] tuples + fees: list[tuple[int, float]] + + # maker fee schedule array in [volume, percent fee] tuples (if on + # maker/taker) + fees_maker: list[tuple[int, float]] + + fee_volume_currency: str # volume discount currency + margin_call: str # margin call level + margin_stop: str # stop-out/liquidation margin level + ordermin: float # minimum order volume for pair + tick_size: float # min price step size + status: str + + short_position_limit: float = 0 + long_position_limit: float = float('inf') + + class Client: # global symbol normalization table _ntable: dict[str, str] = {} _atable: bidict[str, str] = bidict() + _pairs: dict[str, Pair] = {} def __init__( self, @@ -133,13 +177,12 @@ class Client: 'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)' }) self.conf: dict[str, str] = config - self._pairs: list[str] = [] self._name = name self._api_key = api_key self._secret = secret @property - def pairs(self) -> dict[str, Any]: + def pairs(self) -> dict[str, Pair]: if self._pairs is None: raise RuntimeError( "Make sure to run `cache_symbols()` on startup!" @@ -295,15 +338,28 @@ class Client: trans: dict[str, Transaction] = {} for entry in xfers: - # look up the normalized name - asset = self._atable[entry['asset']].lower() + + # look up the normalized name and asset info + asset_key = entry['asset'] + asset_info = self.assets[asset_key] + asset = self._atable[asset_key].lower() # XXX: this is in the asset units (likely) so it isn't # quite the same as a commisions cost necessarily..) cost = float(entry['fee']) + fqsn = asset + '.kraken' + pairinfo = Symbol.from_fqsn( + fqsn, + info={ + 'asset_type': 'crypto', + 'lot_tick_size': asset_info['decimals'], + }, + ) + tran = Transaction( - fqsn=asset + '.kraken', + fqsn=fqsn, + sym=pairinfo, tid=entry['txid'], dt=pendulum.from_timestamp(entry['time']), bsuid=f'{asset}{src_asset}', @@ -317,7 +373,7 @@ class Client: price='NaN', # XXX: see note above - cost=0, + cost=cost, ) trans[tran.tid] = tran @@ -372,7 +428,7 @@ class Client: self, pair: Optional[str] = None, - ) -> dict[str, dict[str, str]]: + ) -> dict[str, Pair] | Pair: if pair is not None: pairs = {'pair': pair} @@ -389,19 +445,36 @@ class Client: if pair is not None: _, data = next(iter(pairs.items())) - return data + return Pair(**data) else: - return pairs + return {key: Pair(**data) for key, data in pairs.items()} - async def cache_symbols( - self, - ) -> dict: + async def cache_symbols(self) -> dict: + ''' + Load all market pair info build and cache it for downstream use. + + A ``._ntable: dict[str, str]`` is available for mapping the + websocket pair name-keys and their http endpoint API (smh) + equivalents to the "alternative name" which is generally the one + we actually want to use XD + + ''' if not self._pairs: - self._pairs = await self.symbol_info() + self._pairs.update(await self.symbol_info()) - ntable = {} - for restapikey, info in self._pairs.items(): - ntable[restapikey] = ntable[info['wsname']] = info['altname'] + # table of all ws and rest keys to their alt-name values. + ntable: dict[str, str] = {} + + for rest_key in list(self._pairs.keys()): + + pair: Pair = self._pairs[rest_key] + altname = pair.altname + wsname = pair.wsname + ntable[rest_key] = ntable[wsname] = altname + + # register the pair under all monikers, a giant flat + # surjection of all possible names to each info obj. + self._pairs[altname] = self._pairs[wsname] = pair self._ntable.update(ntable) @@ -411,26 +484,34 @@ class Client: self, pattern: str, limit: int = None, + ) -> dict[str, Any]: - if self._pairs is not None: - data = self._pairs - else: - data = await self.symbol_info() + ''' + Search for a symbol by "alt name".. + + It is expected that the ``Client._pairs`` table + gets populated before conducting the underlying fuzzy-search + over the pair-key set. + + ''' + if not len(self._pairs): + await self.cache_symbols() + assert self._pairs, '`Client.cache_symbols()` was never called!?' matches = fuzzy.extractBests( pattern, - data, + self._pairs, score_cutoff=50, ) # repack in dict form - return {item[0]['altname']: item[0] for item in matches} + return {item[0].altname: item[0] for item in matches} async def bars( self, symbol: str = 'XBTUSD', # UTC 2017-07-02 12:53:20 - since: Optional[Union[int, datetime]] = None, + since: Union[int, datetime] | None = None, count: int = 720, # <- max allowed per query as_np: bool = True, @@ -506,7 +587,7 @@ class Client: def normalize_symbol( cls, ticker: str - ) -> str: + ) -> tuple[str, Pair]: ''' Normalize symbol names to to a 3x3 pair from the global definition map which we build out from the data retreived from @@ -514,7 +595,7 @@ class Client: ''' ticker = cls._ntable[ticker] - return ticker.lower() + return ticker.lower(), cls._pairs[ticker] @acm diff --git a/piker/brokers/kraken/broker.py b/piker/brokers/kraken/broker.py index d3419800..e09dd35a 100644 --- a/piker/brokers/kraken/broker.py +++ b/piker/brokers/kraken/broker.py @@ -48,6 +48,7 @@ from piker.pp import ( open_trade_ledger, open_pps, ) +from piker.data._source import Symbol from piker.clearing._messages import ( Order, Status, @@ -1196,10 +1197,21 @@ def norm_trade_records( }[record['type']] # we normalize to kraken's `altname` always.. - bsuid = norm_sym = Client.normalize_symbol(record['pair']) + bsuid, pair_info = Client.normalize_symbol(record['pair']) + fqsn = f'{bsuid}.kraken' + + mktpair = Symbol.from_fqsn( + fqsn, + info={ + 'lot_size_digits': pair_info.lot_decimals, + 'tick_size_digits': pair_info.pair_decimals, + 'asset_type': 'crypto', + }, + ) records[tid] = Transaction( - fqsn=f'{norm_sym}.kraken', + fqsn=fqsn, + sym=mktpair, tid=tid, size=size, price=float(record['price']), diff --git a/piker/brokers/kraken/feed.py b/piker/brokers/kraken/feed.py index 3d795098..b4a2e666 100644 --- a/piker/brokers/kraken/feed.py +++ b/piker/brokers/kraken/feed.py @@ -42,56 +42,15 @@ from piker.brokers._util import ( DataUnavailable, ) from piker.log import get_console_log -from piker.data import ShmArray from piker.data.types import Struct from piker.data._web_bs import open_autorecon_ws, NoBsWs from . import log from .api import ( Client, + Pair, ) -# https://www.kraken.com/features/api#get-tradable-pairs -class Pair(Struct): - altname: str # alternate pair name - wsname: str # WebSocket pair name (if available) - aclass_base: str # asset class of base component - base: str # asset id of base component - aclass_quote: str # asset class of quote component - quote: str # asset id of quote component - lot: str # volume lot size - - cost_decimals: int - costmin: float - pair_decimals: int # scaling decimal places for pair - lot_decimals: int # scaling decimal places for volume - - # amount to multiply lot volume by to get currency volume - lot_multiplier: float - - # array of leverage amounts available when buying - leverage_buy: list[int] - # array of leverage amounts available when selling - leverage_sell: list[int] - - # fee schedule array in [volume, percent fee] tuples - fees: list[tuple[int, float]] - - # maker fee schedule array in [volume, percent fee] tuples (if on - # maker/taker) - fees_maker: list[tuple[int, float]] - - fee_volume_currency: str # volume discount currency - margin_call: str # margin call level - margin_stop: str # stop-out/liquidation margin level - ordermin: float # minimum order volume for pair - tick_size: float # min price step size - status: str - - short_position_limit: float - long_position_limit: float - - class OHLC(Struct): ''' Description of the flattened OHLC quote format. @@ -336,14 +295,14 @@ async def stream_quotes( # transform to upper since piker style is always lower sym = sym.upper() - sym_info = await client.symbol_info(sym) - try: - si = Pair(**sym_info) # validation - except TypeError: - fields_diff = set(sym_info) - set(Pair.__struct_fields__) - raise TypeError( - f'Missing msg fields {fields_diff}' - ) + si: Pair = await client.symbol_info(sym) + # try: + # si = Pair(**sym_info) # validation + # except TypeError: + # fields_diff = set(sym_info) - set(Pair.__struct_fields__) + # raise TypeError( + # f'Missing msg fields {fields_diff}' + # ) syminfo = si.to_dict() syminfo['price_tick_size'] = 1. / 10**si.pair_decimals syminfo['lot_tick_size'] = 1. / 10**si.lot_decimals