From dc78994dcf2d9165dd66ebd3c772ac4b92b2895b Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Mon, 27 Feb 2023 16:47:02 -0300 Subject: [PATCH 1/9] Fixed float dust bug on zero position case --- piker/pp.py | 4 +++- tests/test_paper.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/piker/pp.py b/piker/pp.py index b4ab2d0c..1e4cd014 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -22,6 +22,7 @@ that doesn't try to cuk most humans who prefer to not lose their moneys.. ''' from __future__ import annotations from contextlib import contextmanager as cm +from decimal import Decimal, ROUND_HALF_EVEN from pprint import pformat import os from os import path @@ -466,7 +467,8 @@ class Position(Struct): if self.split_ratio is not None: size = round(size * self.split_ratio) - return size + return float(Decimal(size).quantize( + Decimal('1.0000'), rounding=ROUND_HALF_EVEN)) def minimize_clears( self, diff --git a/tests/test_paper.py b/tests/test_paper.py index 3339db6c..f8a7fd24 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -206,8 +206,6 @@ def test_sell( ), ) - -@pytest.mark.xfail(reason='Due to precision issues, this test will currently fail') def test_multi_sell( open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir From f5b8b9a14fbb2cec35eff4813b8e7e5301c8b3ff Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Tue, 28 Feb 2023 01:51:03 -0300 Subject: [PATCH 2/9] Add sym registry to PaperBoi as well as a sym ref on Transaction Add decimal quantize API to Symbol to simplify by-broker truncation Add symbol info to `pps.toml` Move _assert call to outside the _async_main context manager Minor indentation and styling changes, also convert a few prints to log calls Fix multi write / race condition on open_pps call Switch open_pps to not write by default Fix integer math kraken syminfo _tick_size initialization --- piker/brokers/kraken/broker.py | 3 +-- piker/brokers/kraken/feed.py | 4 +-- piker/clearing/_paper_engine.py | 47 ++++++++++++++++++++------------- piker/data/_source.py | 16 +++++++---- piker/pp.py | 41 ++++++++++++++++++---------- tests/test_paper.py | 27 ++++++++----------- 6 files changed, 80 insertions(+), 58 deletions(-) diff --git a/piker/brokers/kraken/broker.py b/piker/brokers/kraken/broker.py index 9378db17..7700d86b 100644 --- a/piker/brokers/kraken/broker.py +++ b/piker/brokers/kraken/broker.py @@ -469,8 +469,7 @@ async def trades_dialogue( with ( open_pps( 'kraken', - acctid, - write_on_exit=True, + acctid ) as table, open_trade_ledger( diff --git a/piker/brokers/kraken/feed.py b/piker/brokers/kraken/feed.py index 57fc0126..3d795098 100644 --- a/piker/brokers/kraken/feed.py +++ b/piker/brokers/kraken/feed.py @@ -345,8 +345,8 @@ async def stream_quotes( f'Missing msg fields {fields_diff}' ) syminfo = si.to_dict() - syminfo['price_tick_size'] = 1 / 10**si.pair_decimals - syminfo['lot_tick_size'] = 1 / 10**si.lot_decimals + syminfo['price_tick_size'] = 1. / 10**si.pair_decimals + syminfo['lot_tick_size'] = 1. / 10**si.lot_decimals syminfo['asset_type'] = 'crypto' sym_infos[sym] = syminfo ws_pairs[sym] = si.wsname diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index fc37f1e4..7a093ad4 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -38,6 +38,7 @@ import tractor from .. import data from ..data.types import Struct +from ..data._source import Symbol from ..pp import ( Position, Transaction, @@ -81,6 +82,7 @@ class PaperBoi(Struct): _reqids: bidict _positions: dict[str, Position] _trade_ledger: dict[str, Any] + _syms: dict[str, Symbol] = {} # init edge case L1 spread last_ask: tuple[float, float] = (float('inf'), 0) # price, size @@ -252,6 +254,7 @@ class PaperBoi(Struct): key = fqsn.rstrip(f'.{self.broker}') t = Transaction( fqsn=fqsn, + sym=self._syms[fqsn], tid=oid, size=size, price=price, @@ -261,27 +264,29 @@ class PaperBoi(Struct): ) with ( - open_trade_ledger(self.broker, 'paper') as ledger, - open_pps(self.broker, 'paper', True) as table - ): - ledger.update({oid: t.to_dict()}) - # Write to pps toml right now - table.update_from_trans({oid: t}) + open_trade_ledger(self.broker, 'paper') as ledger, + open_pps(self.broker, 'paper', write_on_exit=True) as table + ): + tx = t.to_dict() + tx.pop('sym') + ledger.update({oid: tx}) + # Write to pps toml right now + table.update_from_trans({oid: t}) - pp = table.pps[key] - pp_msg = BrokerdPosition( - broker=self.broker, - account='paper', - symbol=fqsn, - # TODO: we need to look up the asset currency from - # broker info. i guess for crypto this can be - # inferred from the pair? - currency=key, - size=pp.size, - avg_price=pp.ppu, - ) + pp = table.pps[key] + pp_msg = BrokerdPosition( + broker=self.broker, + account='paper', + symbol=fqsn, + # TODO: we need to look up the asset currency from + # broker info. i guess for crypto this can be + # inferred from the pair? + currency=key, + size=pp.size, + avg_price=pp.ppu, + ) - await self.ems_trades_stream.send(pp_msg) + await self.ems_trades_stream.send(pp_msg) async def simulate_fills( @@ -567,6 +572,10 @@ async def trades_dialogue( # TODO: load postions from ledger file _trade_ledger={}, + _syms={ + fqsn: flume.symbol + for fqsn, flume in feed.flumes.items() + } ) n.start_soon( diff --git a/piker/data/_source.py b/piker/data/_source.py index 87ba74a3..e8f09484 100644 --- a/piker/data/_source.py +++ b/piker/data/_source.py @@ -18,8 +18,8 @@ numpy data source coversion helpers. """ from __future__ import annotations +from decimal import Decimal, ROUND_HALF_EVEN from typing import Any -import decimal from bidict import bidict import numpy as np @@ -80,7 +80,7 @@ def float_digits( if value == 0: return 0 - return int(-decimal.Decimal(str(value)).as_tuple().exponent) + return int(-Decimal(str(value)).as_tuple().exponent) def ohlc_zeros(length: int) -> np.ndarray: @@ -156,14 +156,14 @@ class Symbol(Struct): ) -> Symbol: tick_size = info.get('price_tick_size', 0.01) - lot_tick_size = info.get('lot_tick_size', 0.0) + lot_size = info.get('lot_tick_size', 0.0) return Symbol( key=symbol, tick_size=tick_size, - lot_tick_size=lot_tick_size, + lot_tick_size=lot_size, tick_size_digits=float_digits(tick_size), - lot_size_digits=float_digits(lot_tick_size), + lot_size_digits=float_digits(lot_size), suffix=suffix, broker_info={broker: info}, ) @@ -254,6 +254,12 @@ class Symbol(Struct): return keys + def decimal_quant(self, d: Decimal): + digits = self.lot_size_digits + return d.quantize( + Decimal(f'1.{"0".ljust(digits, "0")}'), + rounding=ROUND_HALF_EVEN + ) def _nan_to_closest_num(array: np.ndarray): """Return interpolated values instead of NaN. diff --git a/piker/pp.py b/piker/pp.py index 1e4cd014..83e43731 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -45,7 +45,7 @@ import toml from . import config from .brokers import get_brokermod from .clearing._messages import BrokerdPosition, Status -from .data._source import Symbol +from .data._source import Symbol, unpack_fqsn from .log import get_logger from .data.types import Struct @@ -83,7 +83,7 @@ def open_trade_ledger( with open(tradesfile, 'rb') as cf: start = time.time() ledger = tomli.load(cf) - print(f'Ledger load took {time.time() - start}s') + log.info(f'Ledger load took {time.time() - start}s') cpy = ledger.copy() try: @@ -92,7 +92,7 @@ def open_trade_ledger( if cpy != ledger: # TODO: show diff output? # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries - print(f'Updating ledger for {tradesfile}:\n') + log.info(f'Updating ledger for {tradesfile}:\n') ledger.update(cpy) # we write on close the mutated ledger data with open(tradesfile, 'w') as cf: @@ -103,17 +103,18 @@ class Transaction(Struct, frozen=True): # TODO: should this be ``.to`` (see below)? fqsn: str + sym: Symbol tid: Union[str, int] # unique transaction id size: float price: float cost: float # commisions or other additional costs dt: datetime - expiry: Optional[datetime] = None + expiry: datetime | None = None # optional key normally derived from the broker # backend which ensures the instrument-symbol this record # is for is truly unique. - bsuid: Optional[Union[str, int]] = None + bsuid: Union[str, int] | None = None # optional fqsn for the source "asset"/money symbol? # from: Optional[str] = None @@ -190,9 +191,20 @@ class Position(Struct): # listing venue here even when the backend isn't providing # it via the trades ledger.. # drop symbol obj in serialized form - s = d.pop('symbol') + s = d.get('symbol') fqsn = s.front_fqsn() + broker, key, suffix = unpack_fqsn(fqsn) + sym_info = s.broker_info[broker] + + d['symbol'] = { + 'info': { + 'asset_type': sym_info['asset_type'], + 'price_tick_size': sym_info['price_tick_size'], + 'lot_tick_size': sym_info['lot_tick_size'] + } + } + if self.expiry is None: d.pop('expiry', None) elif expiry: @@ -467,8 +479,7 @@ class Position(Struct): if self.split_ratio is not None: size = round(size * self.split_ratio) - return float(Decimal(size).quantize( - Decimal('1.0000'), rounding=ROUND_HALF_EVEN)) + return float(self.symbol.decimal_quant(Decimal(size))) def minimize_clears( self, @@ -512,7 +523,7 @@ class Position(Struct): 'cost': t.cost, 'price': t.price, 'size': t.size, - 'dt': t.dt, + 'dt': t.dt } # TODO: compute these incrementally instead @@ -559,7 +570,7 @@ class PpTable(Struct): Symbol.from_fqsn( t.fqsn, info={}, - ), + ) if not t.sym else t.sym, size=0.0, ppu=0.0, bsuid=t.bsuid, @@ -620,7 +631,7 @@ class PpTable(Struct): # XXX: debug hook for size mismatches # qqqbsuid = 320227571 # if bsuid == qqqbsuid: - # breakpoint() + # xbreakpoint() pp.ensure_state() @@ -682,10 +693,10 @@ class PpTable(Struct): ''' # TODO: show diff output? # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries - print(f'Updating ``pps.toml`` for {path}:\n') - + log.info(f'Updating ``pps.toml`` for {path}:\n') # active, closed_pp_objs = table.dump_active() pp_entries = self.to_toml() + log.info(pp_entries) self.conf[self.brokername][self.acctid] = pp_entries # TODO: why tf haven't they already done this for inline @@ -883,7 +894,6 @@ def open_pps( brokername: str, acctid: str, write_on_exit: bool = False, - ) -> Generator[PpTable, None, None]: ''' Read out broker-specific position entries from @@ -937,8 +947,11 @@ def open_pps( dtstr = clears_table['dt'] dt = pendulum.parse(dtstr) clears_table['dt'] = dt + trans.append(Transaction( fqsn=bsuid, + sym=Symbol.from_fqsn( + fqsn, entry['symbol']), bsuid=bsuid, tid=tid, size=clears_table['size'], diff --git a/tests/test_paper.py b/tests/test_paper.py index f8a7fd24..8da1cf12 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -84,25 +84,20 @@ async def _async_main( case {'name': 'position'}: break - if ( - assert_entries - or assert_pps - or assert_zeroed_pps - or assert_msg - ): - _assert( - assert_entries, - assert_pps, - assert_zeroed_pps, - pps, - last_msg, - size, - executions, - ) - # Teardown piker like a user would raise KeyboardInterrupt + if assert_entries or assert_pps or assert_zeroed_pps or assert_msg: + _assert( + assert_entries, + assert_pps, + assert_zeroed_pps, + pps, + last_msg, + size, + executions, + ) + def _assert( assert_entries, From 6c23c79f2ac0095146f6ec12af5b207af3bc7ae6 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 1 Mar 2023 15:21:56 -0300 Subject: [PATCH 3/9] Minor fixes after fomo's review --- piker/pp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/pp.py b/piker/pp.py index 83e43731..33040654 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -631,7 +631,7 @@ class PpTable(Struct): # XXX: debug hook for size mismatches # qqqbsuid = 320227571 # if bsuid == qqqbsuid: - # xbreakpoint() + # breakpoint() pp.ensure_state() @@ -696,7 +696,7 @@ class PpTable(Struct): log.info(f'Updating ``pps.toml`` for {path}:\n') # active, closed_pp_objs = table.dump_active() pp_entries = self.to_toml() - log.info(pp_entries) + log.info(f'Current positions:\n{pp_entries}') self.conf[self.brokername][self.acctid] = pp_entries # TODO: why tf haven't they already done this for inline From 20d91f5e0690e7229613d390409304b464f9c9fa Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 1 Mar 2023 15:39:30 -0300 Subject: [PATCH 4/9] Good catch by j, unnecesary kwarg on open_pps --- piker/brokers/kraken/broker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/kraken/broker.py b/piker/brokers/kraken/broker.py index 7700d86b..d3419800 100644 --- a/piker/brokers/kraken/broker.py +++ b/piker/brokers/kraken/broker.py @@ -474,7 +474,7 @@ async def trades_dialogue( open_trade_ledger( 'kraken', - acctid, + acctid ) as ledger_dict, ): # transaction-ify the ledger entries From d704b153caeb2abd1c35a92833ec5012b28a6f80 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 1 Mar 2023 16:44:46 -0300 Subject: [PATCH 5/9] Fix mayor bug found by fomo, sym info getting stored incorrectly on pps.toml causing it to load pp wrong on second open, also fix header leak bug --- piker/config.py | 3 ++- piker/pp.py | 28 +++++++++++++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/piker/config.py b/piker/config.py index 8bf14759..3ae6a665 100644 --- a/piker/config.py +++ b/piker/config.py @@ -237,6 +237,7 @@ def write( config: dict, # toml config as dict name: str = 'brokers', path: str = None, + fail_empty: bool = True, **toml_kwargs, ) -> None: @@ -252,7 +253,7 @@ def write( log.debug(f"Creating config dir {_config_dir}") os.makedirs(dirname) - if not config: + if not config and fail_empty: raise ValueError( "Watch out you're trying to write a blank config!") diff --git a/piker/pp.py b/piker/pp.py index 33040654..5d792aec 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -22,6 +22,7 @@ that doesn't try to cuk most humans who prefer to not lose their moneys.. ''' from __future__ import annotations from contextlib import contextmanager as cm +from pathlib import Path from decimal import Decimal, ROUND_HALF_EVEN from pprint import pformat import os @@ -198,11 +199,9 @@ class Position(Struct): sym_info = s.broker_info[broker] d['symbol'] = { - 'info': { - 'asset_type': sym_info['asset_type'], - 'price_tick_size': sym_info['price_tick_size'], - 'lot_tick_size': sym_info['lot_tick_size'] - } + 'asset_type': sym_info['asset_type'], + 'price_tick_size': sym_info['price_tick_size'], + 'lot_tick_size': sym_info['lot_tick_size'] } if self.expiry is None: @@ -693,11 +692,20 @@ class PpTable(Struct): ''' # TODO: show diff output? # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries - log.info(f'Updating ``pps.toml`` for {path}:\n') # active, closed_pp_objs = table.dump_active() pp_entries = self.to_toml() - log.info(f'Current positions:\n{pp_entries}') - self.conf[self.brokername][self.acctid] = pp_entries + if pp_entries: + log.info(f'Updating ``pps.toml`` for {path}:\n') + log.info(f'Current positions:\n{pp_entries}') + self.conf[self.brokername][self.acctid] = pp_entries + + elif ( + self.brokername in self.conf and + self.acctid in self.conf[self.brokername] + ): + del self.conf[self.brokername][self.acctid] + if len(self.conf[self.brokername]) == 0: + del self.conf[self.brokername] # TODO: why tf haven't they already done this for inline # tables smh.. @@ -711,6 +719,7 @@ class PpTable(Struct): self.conf, 'pps', encoder=enc, + fail_empty=False ) @@ -972,7 +981,8 @@ def open_pps( expiry = pendulum.parse(expiry) pp = pp_objs[bsuid] = Position( - Symbol.from_fqsn(fqsn, info={}), + Symbol.from_fqsn( + fqsn, entry['symbol']), size=size, ppu=ppu, split_ratio=split_ratio, From 6be96a96aaf37d736b99535803a7383124c1c1a8 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 1 Mar 2023 21:04:36 -0300 Subject: [PATCH 6/9] Drop symbol section on Position serialization --- piker/pp.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/piker/pp.py b/piker/pp.py index 5d792aec..1a2f5e6b 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -192,17 +192,15 @@ class Position(Struct): # listing venue here even when the backend isn't providing # it via the trades ledger.. # drop symbol obj in serialized form - s = d.get('symbol') + s = d.pop('symbol') fqsn = s.front_fqsn() broker, key, suffix = unpack_fqsn(fqsn) sym_info = s.broker_info[broker] - d['symbol'] = { - 'asset_type': sym_info['asset_type'], - 'price_tick_size': sym_info['price_tick_size'], - 'lot_tick_size': sym_info['lot_tick_size'] - } + d['asset_type'] = sym_info['asset_type'] + d['price_tick_size'] = sym_info['price_tick_size'] + d['lot_tick_size'] = sym_info['lot_tick_size'] if self.expiry is None: d.pop('expiry', None) @@ -935,6 +933,13 @@ def open_pps( # and update `PpTable` obj entries. for fqsn, entry in pps.items(): bsuid = entry['bsuid'] + symbol = Symbol.from_fqsn( + fqsn, info={ + 'asset_type': entry['asset_type'], + 'price_tick_size': entry['price_tick_size'], + 'lot_tick_size': entry['lot_tick_size'] + } + ) # convert clears sub-tables (only in this form # for toml re-presentation) back into a master table. @@ -959,8 +964,7 @@ def open_pps( trans.append(Transaction( fqsn=bsuid, - sym=Symbol.from_fqsn( - fqsn, entry['symbol']), + sym=symbol, bsuid=bsuid, tid=tid, size=clears_table['size'], @@ -981,8 +985,7 @@ def open_pps( expiry = pendulum.parse(expiry) pp = pp_objs[bsuid] = Position( - Symbol.from_fqsn( - fqsn, entry['symbol']), + symbol, size=size, ppu=ppu, split_ratio=split_ratio, From 3a4794e9d1c9215a03fc7ce3a1e5bc0571730a07 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Mar 2023 19:03:40 -0500 Subject: [PATCH 7/9] Backward-compat: don't require `'lot_tick_size'` In order to support existing `pps.toml` files in the wild which don't have the `asset_type, price_tick_size, lot_tick_size` fields, we need to only optionally read them and instead expect that backends will write the fields going forward (coming in follow patches). Further this makes some small asset-size (vlm accounting) quantization related adjustments: - rename `Symbol.decimal_quant()` -> `.quantize_size()` since that is explicitly what this method is doing. - and expect an input `size: float` which we cast to decimal instead of doing it inside the `.calc_size()` caller code. - drop `Symbol.iterfqsns()` which wasn't being used anywhere at all.. Additionally, this drafts out a new replacement market-trading-pair data type to eventually replace `.data._source.Symbol` -> `MktPair` which we aren't using yet, but serves as the documentation-driven motivator ;) and, it relates to https://github.com/pikers/piker/issues/467. --- piker/data/_source.py | 83 +++++++++++++++++++++++++++++++++++-------- piker/pp.py | 30 +++++++++++----- 2 files changed, 89 insertions(+), 24 deletions(-) diff --git a/piker/data/_source.py b/piker/data/_source.py index e8f09484..d358cd96 100644 --- a/piker/data/_source.py +++ b/piker/data/_source.py @@ -18,7 +18,10 @@ numpy data source coversion helpers. """ from __future__ import annotations -from decimal import Decimal, ROUND_HALF_EVEN +from decimal import ( + Decimal, + ROUND_HALF_EVEN, +) from typing import Any from bidict import bidict @@ -77,6 +80,10 @@ def mk_fqsn( def float_digits( value: float, ) -> int: + ''' + Return the number of precision digits read from a float value. + + ''' if value == 0: return 0 @@ -127,6 +134,56 @@ def unpack_fqsn(fqsn: str) -> tuple[str, str, str]: ) +class MktPair(Struct, frozen=True): + + src: str # source asset name being used to buy + src_type: str # source asset's financial type/classification name + # ^ specifies a "class" of financial instrument + # egs. stock, futer, option, bond etc. + + dst: str # destination asset name being bought + dst_type: str # destination asset's financial type/classification name + + price_tick: float # minimum price increment value increment + price_tick_digits: int # required decimal digits for above + + size_tick: float # minimum size (aka vlm) increment value increment + size_tick_digits: int # required decimal digits for above + + venue: str | None = None # market venue provider name + expiry: str | None = None # for derivs, expiry datetime parseable str + + # for derivs, info describing contract, egs. + # strike price, call or put, swap type, exercise model, etc. + contract_info: str | None = None + + @classmethod + def from_msg( + self, + msg: dict[str, Any], + + ) -> MktPair: + ''' + Constructor for a received msg-dict normally received over IPC. + + ''' + ... + + # fqa, fqma, .. etc. see issue: + # https://github.com/pikers/piker/issues/467 + @property + def fqsn(self) -> str: + ''' + Return the fully qualified market (endpoint) name for the + pair of transacting assets. + + ''' + ... + + +# TODO: rework the below `Symbol` (which was originally inspired and +# derived from stuff in quantdom) into a simpler, ipc msg ready, market +# endpoint meta-data container type as per the drafted interace above. class Symbol(Struct): ''' I guess this is some kinda container thing for dealing with @@ -141,10 +198,6 @@ class Symbol(Struct): suffix: str = '' broker_info: dict[str, dict[str, Any]] = {} - # specifies a "class" of financial instrument - # ex. stock, futer, option, bond etc. - - # @validate_arguments @classmethod def from_broker_info( cls, @@ -244,23 +297,23 @@ class Symbol(Struct): fqsn = '.'.join(map(str.lower, tokens)) return fqsn - def iterfqsns(self) -> list[str]: - keys = [] - for broker in self.broker_info.keys(): - fqsn = mk_fqsn(self.key, broker) - if self.suffix: - fqsn += f'.{self.suffix}' - keys.append(fqsn) + def quantize_size( + self, + size: float, - return keys + ) -> Decimal: + ''' + Truncate input ``size: float`` using ``Decimal`` + and ``.lot_size_digits``. - def decimal_quant(self, d: Decimal): + ''' digits = self.lot_size_digits - return d.quantize( + return Decimal(size).quantize( Decimal(f'1.{"0".ljust(digits, "0")}'), rounding=ROUND_HALF_EVEN ) + def _nan_to_closest_num(array: np.ndarray): """Return interpolated values instead of NaN. diff --git a/piker/pp.py b/piker/pp.py index 1a2f5e6b..38ff1566 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -22,8 +22,6 @@ that doesn't try to cuk most humans who prefer to not lose their moneys.. ''' from __future__ import annotations from contextlib import contextmanager as cm -from pathlib import Path -from decimal import Decimal, ROUND_HALF_EVEN from pprint import pformat import os from os import path @@ -91,10 +89,12 @@ def open_trade_ledger( yield cpy finally: if cpy != ledger: + # TODO: show diff output? # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries log.info(f'Updating ledger for {tradesfile}:\n') - ledger.update(cpy) + ledger.update(cpy) + # we write on close the mutated ledger data with open(tradesfile, 'w') as cf: toml.dump(ledger, cf) @@ -476,7 +476,7 @@ class Position(Struct): if self.split_ratio is not None: size = round(size * self.split_ratio) - return float(self.symbol.decimal_quant(Decimal(size))) + return float(self.symbol.quantize_size(size)) def minimize_clears( self, @@ -934,10 +934,18 @@ def open_pps( for fqsn, entry in pps.items(): bsuid = entry['bsuid'] symbol = Symbol.from_fqsn( - fqsn, info={ - 'asset_type': entry['asset_type'], - 'price_tick_size': entry['price_tick_size'], - 'lot_tick_size': entry['lot_tick_size'] + fqsn, + + # NOTE & TODO: right now we fill in the defaults from + # `.data._source.Symbol` but eventually these should always + # either be already written to the pos table or provided at + # write time to ensure always having these values somewhere + # and thus allowing us to get our pos sizing precision + # correct! + info={ + 'asset_type': entry.get('asset_type', ''), + 'price_tick_size': entry.get('price_tick_size', 0.01), + 'lot_tick_size': entry.get('lot_tick_size', 0.0), } ) @@ -977,7 +985,11 @@ def open_pps( size = entry['size'] # TODO: remove but, handle old field name for now - ppu = entry.get('ppu', entry.get('be_price', 0)) + ppu = entry.get( + 'ppu', + entry.get('be_price', 0), + ) + split_ratio = entry.get('split_ratio') expiry = entry.get('expiry') From 69b85aa7e588452158bdd823254c311353e8d014 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Mar 2023 19:23:47 -0500 Subject: [PATCH 8/9] `ib`: parse and load info for new `Transaction.sym: Symbol` field --- piker/brokers/ib/broker.py | 63 ++++++++++++++++++++++++++++++++------ piker/brokers/ib/feed.py | 2 +- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index 4537e7f0..56756a76 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -70,7 +70,10 @@ from piker.clearing._messages import ( BrokerdFill, BrokerdError, ) -from piker.data._source import Symbol +from piker.data._source import ( + Symbol, + float_digits, +) from .api import ( _accounts2clients, con2fqsn, @@ -304,6 +307,9 @@ async def update_ledger_from_api_trades( entry['listingExchange'] = pexch + # pack in the ``Contract.secType`` + entry['asset_type'] = condict['secType'] + conf = get_config() entries = api_trades_to_ledger_entries( conf['accounts'].inverse, @@ -616,9 +622,10 @@ async def trades_dialogue( # from the api trades it seems we get a key # error from ``update[bsuid]`` ? pp = table.pps[bsuid] + pairinfo = pp.symbol if msg.size != pp.size: log.error( - f'Position mismatch {pp.symbol.front_fqsn()}:\n' + f'Pos size mismatch {pairinfo.front_fqsn()}:\n' f'ib: {msg.size}\n' f'piker: {pp.size}\n' ) @@ -1095,13 +1102,15 @@ def norm_trade_records( ''' records: list[Transaction] = [] - for tid, record in ledger.items(): + for tid, record in ledger.items(): conid = record.get('conId') or record['conid'] comms = record.get('commission') if comms is None: comms = -1*record['ibCommission'] + price = record.get('price') or record['tradePrice'] + price_tick_digits = float_digits(price) # the api doesn't do the -/+ on the quantity for you but flex # records do.. are you fucking serious ib...!? @@ -1144,9 +1153,14 @@ def norm_trade_records( # special handling of symbol extraction from # flex records using some ad-hoc schema parsing. - instr = record.get('assetCategory') - if instr == 'FUT': - symbol = record['description'][:3] + asset_type: str = record.get('assetCategory') or record['secType'] + + # TODO: XXX: WOA this is kinda hacky.. probably + # should figure out the correct future pair key more + # explicitly and consistently? + if asset_type == 'FUT': + # (flex) ledger entries don't have any simple 3-char key? + symbol = record['symbol'][:3] # try to build out piker fqsn from record. expiry = record.get( @@ -1156,10 +1170,34 @@ def norm_trade_records( suffix = f'{exch}.{expiry}' expiry = pendulum.parse(expiry) - fqsn = Symbol.from_fqsn( + src: str = record['currency'] + + pair = Symbol.from_fqsn( fqsn=f'{symbol}.{suffix}.ib', - info={}, - ).front_fqsn().rstrip('.ib') + info={ + 'tick_size_digits': price_tick_digits, + + # NOTE: for "legacy" assets, volume is normally discreet, not + # a float, but we keep a digit in case the suitz decide + # to get crazy and change it; we'll be kinda ready + # schema-wise.. + 'lot_size_digits': 1, + + # TODO: remove when we switching from + # ``Symbol`` -> ``MktPair`` + 'asset_type': asset_type, + + # TODO: figure out a target fin-type name + # set and normalize to that here! + 'dst_type': asset_type.lower(), + + # starting to use new key naming as in ``MktPair`` + # type have drafted... + 'src': src, + 'src_type': 'fiat', + }, + ) + fqsn = pair.front_fqsn().rstrip('.ib') # NOTE: for flex records the normal fields for defining an fqsn # sometimes won't be available so we rely on two approaches for @@ -1175,6 +1213,7 @@ def norm_trade_records( records, Transaction( fqsn=fqsn, + sym=pair, tid=tid, size=size, price=price, @@ -1201,7 +1240,11 @@ def parse_flex_dt( def api_trades_to_ledger_entries( accounts: bidict, - trade_entries: list[object], + + # TODO: maybe we should just be passing through the + # ``ib_insync.order.Trade`` instance directly here + # instead of pre-casting to dicts? + trade_entries: list[dict], ) -> dict: ''' diff --git a/piker/brokers/ib/feed.py b/piker/brokers/ib/feed.py index bee69ae6..345e4ade 100644 --- a/piker/brokers/ib/feed.py +++ b/piker/brokers/ib/feed.py @@ -770,7 +770,7 @@ async def stream_quotes( syminfo['price_tick_size'] = max(syminfo['minTick'], min_tick) - # for "traditional" assets, volume is normally discreet, not + # for "legacy" assets, volume is normally discreet, not # a float syminfo['lot_tick_size'] = 0.0 From b4a1cc8f22f2079ab6b0087aee8b5410c8100c03 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Mar 2023 19:25:43 -0500 Subject: [PATCH 9/9] `kraken`: parse and load info `Transaction.sym: Symbol` Also includes a retyping of `Client._pair: dict[str, Pair]` to look up pair structs and map all alt-key-name-sets to each for easy precision info lookup to set the `.sym` field for each transaction including for on-chain transfers which kraken provides as an "asset decimals" field, presumably pulled from the particular block-token's limitation info. --- piker/brokers/kraken/api.py | 131 ++++++++++++++++++++++++++------- piker/brokers/kraken/broker.py | 16 +++- piker/brokers/kraken/feed.py | 59 +++------------ 3 files changed, 129 insertions(+), 77 deletions(-) diff --git a/piker/brokers/kraken/api.py b/piker/brokers/kraken/api.py index d590148f..94d6dc41 100644 --- a/piker/brokers/kraken/api.py +++ b/piker/brokers/kraken/api.py @@ -40,6 +40,8 @@ import base64 import trio from piker import config +from piker.data.types import Struct +from piker.data._source import Symbol from piker.brokers._util import ( resproc, SymbolNotFound, @@ -113,11 +115,53 @@ class InvalidKey(ValueError): ''' +# https://www.kraken.com/features/api#get-tradable-pairs +class Pair(Struct): + altname: str # alternate pair name + wsname: str # WebSocket pair name (if available) + aclass_base: str # asset class of base component + base: str # asset id of base component + aclass_quote: str # asset class of quote component + quote: str # asset id of quote component + lot: str # volume lot size + + cost_decimals: int + costmin: float + pair_decimals: int # scaling decimal places for pair + lot_decimals: int # scaling decimal places for volume + + # amount to multiply lot volume by to get currency volume + lot_multiplier: float + + # array of leverage amounts available when buying + leverage_buy: list[int] + # array of leverage amounts available when selling + leverage_sell: list[int] + + # fee schedule array in [volume, percent fee] tuples + fees: list[tuple[int, float]] + + # maker fee schedule array in [volume, percent fee] tuples (if on + # maker/taker) + fees_maker: list[tuple[int, float]] + + fee_volume_currency: str # volume discount currency + margin_call: str # margin call level + margin_stop: str # stop-out/liquidation margin level + ordermin: float # minimum order volume for pair + tick_size: float # min price step size + status: str + + short_position_limit: float = 0 + long_position_limit: float = float('inf') + + class Client: # global symbol normalization table _ntable: dict[str, str] = {} _atable: bidict[str, str] = bidict() + _pairs: dict[str, Pair] = {} def __init__( self, @@ -133,13 +177,12 @@ class Client: 'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)' }) self.conf: dict[str, str] = config - self._pairs: list[str] = [] self._name = name self._api_key = api_key self._secret = secret @property - def pairs(self) -> dict[str, Any]: + def pairs(self) -> dict[str, Pair]: if self._pairs is None: raise RuntimeError( "Make sure to run `cache_symbols()` on startup!" @@ -295,15 +338,28 @@ class Client: trans: dict[str, Transaction] = {} for entry in xfers: - # look up the normalized name - asset = self._atable[entry['asset']].lower() + + # look up the normalized name and asset info + asset_key = entry['asset'] + asset_info = self.assets[asset_key] + asset = self._atable[asset_key].lower() # XXX: this is in the asset units (likely) so it isn't # quite the same as a commisions cost necessarily..) cost = float(entry['fee']) + fqsn = asset + '.kraken' + pairinfo = Symbol.from_fqsn( + fqsn, + info={ + 'asset_type': 'crypto', + 'lot_tick_size': asset_info['decimals'], + }, + ) + tran = Transaction( - fqsn=asset + '.kraken', + fqsn=fqsn, + sym=pairinfo, tid=entry['txid'], dt=pendulum.from_timestamp(entry['time']), bsuid=f'{asset}{src_asset}', @@ -317,7 +373,7 @@ class Client: price='NaN', # XXX: see note above - cost=0, + cost=cost, ) trans[tran.tid] = tran @@ -372,7 +428,7 @@ class Client: self, pair: Optional[str] = None, - ) -> dict[str, dict[str, str]]: + ) -> dict[str, Pair] | Pair: if pair is not None: pairs = {'pair': pair} @@ -389,19 +445,36 @@ class Client: if pair is not None: _, data = next(iter(pairs.items())) - return data + return Pair(**data) else: - return pairs + return {key: Pair(**data) for key, data in pairs.items()} - async def cache_symbols( - self, - ) -> dict: + async def cache_symbols(self) -> dict: + ''' + Load all market pair info build and cache it for downstream use. + + A ``._ntable: dict[str, str]`` is available for mapping the + websocket pair name-keys and their http endpoint API (smh) + equivalents to the "alternative name" which is generally the one + we actually want to use XD + + ''' if not self._pairs: - self._pairs = await self.symbol_info() + self._pairs.update(await self.symbol_info()) - ntable = {} - for restapikey, info in self._pairs.items(): - ntable[restapikey] = ntable[info['wsname']] = info['altname'] + # table of all ws and rest keys to their alt-name values. + ntable: dict[str, str] = {} + + for rest_key in list(self._pairs.keys()): + + pair: Pair = self._pairs[rest_key] + altname = pair.altname + wsname = pair.wsname + ntable[rest_key] = ntable[wsname] = altname + + # register the pair under all monikers, a giant flat + # surjection of all possible names to each info obj. + self._pairs[altname] = self._pairs[wsname] = pair self._ntable.update(ntable) @@ -411,26 +484,34 @@ class Client: self, pattern: str, limit: int = None, + ) -> dict[str, Any]: - if self._pairs is not None: - data = self._pairs - else: - data = await self.symbol_info() + ''' + Search for a symbol by "alt name".. + + It is expected that the ``Client._pairs`` table + gets populated before conducting the underlying fuzzy-search + over the pair-key set. + + ''' + if not len(self._pairs): + await self.cache_symbols() + assert self._pairs, '`Client.cache_symbols()` was never called!?' matches = fuzzy.extractBests( pattern, - data, + self._pairs, score_cutoff=50, ) # repack in dict form - return {item[0]['altname']: item[0] for item in matches} + return {item[0].altname: item[0] for item in matches} async def bars( self, symbol: str = 'XBTUSD', # UTC 2017-07-02 12:53:20 - since: Optional[Union[int, datetime]] = None, + since: Union[int, datetime] | None = None, count: int = 720, # <- max allowed per query as_np: bool = True, @@ -506,7 +587,7 @@ class Client: def normalize_symbol( cls, ticker: str - ) -> str: + ) -> tuple[str, Pair]: ''' Normalize symbol names to to a 3x3 pair from the global definition map which we build out from the data retreived from @@ -514,7 +595,7 @@ class Client: ''' ticker = cls._ntable[ticker] - return ticker.lower() + return ticker.lower(), cls._pairs[ticker] @acm diff --git a/piker/brokers/kraken/broker.py b/piker/brokers/kraken/broker.py index d3419800..e09dd35a 100644 --- a/piker/brokers/kraken/broker.py +++ b/piker/brokers/kraken/broker.py @@ -48,6 +48,7 @@ from piker.pp import ( open_trade_ledger, open_pps, ) +from piker.data._source import Symbol from piker.clearing._messages import ( Order, Status, @@ -1196,10 +1197,21 @@ def norm_trade_records( }[record['type']] # we normalize to kraken's `altname` always.. - bsuid = norm_sym = Client.normalize_symbol(record['pair']) + bsuid, pair_info = Client.normalize_symbol(record['pair']) + fqsn = f'{bsuid}.kraken' + + mktpair = Symbol.from_fqsn( + fqsn, + info={ + 'lot_size_digits': pair_info.lot_decimals, + 'tick_size_digits': pair_info.pair_decimals, + 'asset_type': 'crypto', + }, + ) records[tid] = Transaction( - fqsn=f'{norm_sym}.kraken', + fqsn=fqsn, + sym=mktpair, tid=tid, size=size, price=float(record['price']), diff --git a/piker/brokers/kraken/feed.py b/piker/brokers/kraken/feed.py index 3d795098..b4a2e666 100644 --- a/piker/brokers/kraken/feed.py +++ b/piker/brokers/kraken/feed.py @@ -42,56 +42,15 @@ from piker.brokers._util import ( DataUnavailable, ) from piker.log import get_console_log -from piker.data import ShmArray from piker.data.types import Struct from piker.data._web_bs import open_autorecon_ws, NoBsWs from . import log from .api import ( Client, + Pair, ) -# https://www.kraken.com/features/api#get-tradable-pairs -class Pair(Struct): - altname: str # alternate pair name - wsname: str # WebSocket pair name (if available) - aclass_base: str # asset class of base component - base: str # asset id of base component - aclass_quote: str # asset class of quote component - quote: str # asset id of quote component - lot: str # volume lot size - - cost_decimals: int - costmin: float - pair_decimals: int # scaling decimal places for pair - lot_decimals: int # scaling decimal places for volume - - # amount to multiply lot volume by to get currency volume - lot_multiplier: float - - # array of leverage amounts available when buying - leverage_buy: list[int] - # array of leverage amounts available when selling - leverage_sell: list[int] - - # fee schedule array in [volume, percent fee] tuples - fees: list[tuple[int, float]] - - # maker fee schedule array in [volume, percent fee] tuples (if on - # maker/taker) - fees_maker: list[tuple[int, float]] - - fee_volume_currency: str # volume discount currency - margin_call: str # margin call level - margin_stop: str # stop-out/liquidation margin level - ordermin: float # minimum order volume for pair - tick_size: float # min price step size - status: str - - short_position_limit: float - long_position_limit: float - - class OHLC(Struct): ''' Description of the flattened OHLC quote format. @@ -336,14 +295,14 @@ async def stream_quotes( # transform to upper since piker style is always lower sym = sym.upper() - sym_info = await client.symbol_info(sym) - try: - si = Pair(**sym_info) # validation - except TypeError: - fields_diff = set(sym_info) - set(Pair.__struct_fields__) - raise TypeError( - f'Missing msg fields {fields_diff}' - ) + si: Pair = await client.symbol_info(sym) + # try: + # si = Pair(**sym_info) # validation + # except TypeError: + # fields_diff = set(sym_info) - set(Pair.__struct_fields__) + # raise TypeError( + # f'Missing msg fields {fields_diff}' + # ) syminfo = si.to_dict() syminfo['price_tick_size'] = 1. / 10**si.pair_decimals syminfo['lot_tick_size'] = 1. / 10**si.lot_decimals