pikers
/
piker
3
0
Fork 0
Code Issues 5 Pull Requests 7 Packages Projects Releases Wiki Activity

Merge pull request #477 from pikers/backward_compat_trans_with_symbolinfo 

Backward compat support for `Transaction.sym: Symbol`
decimalization_take_2
Guillermo Rodriguez 2023-03-02 23:19:55 -03:00 committed by GitHub
parent 6be96a96aa b4a1cc8f22
commit d4ac8972ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 272 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
piker/brokers/ib/broker.py
View File

@ -70,7 +70,10 @@ from piker.clearing._messages import (
BrokerdFill,
BrokerdError,
)
from piker.data._source import Symbol
from piker.data._source import (
Symbol,
float_digits,
)
from .api import (
_accounts2clients,
con2fqsn,
@ -304,6 +307,9 @@ async def update_ledger_from_api_trades(
entry['listingExchange'] = pexch
# pack in the ``Contract.secType``
entry['asset_type'] = condict['secType']
conf = get_config()
entries = api_trades_to_ledger_entries(
conf['accounts'].inverse,
@ -616,9 +622,10 @@ async def trades_dialogue(
# from the api trades it seems we get a key
# error from ``update[bsuid]`` ?
pp = table.pps[bsuid]
pairinfo = pp.symbol
if msg.size != pp.size:
log.error(
f'Position mismatch {pp.symbol.front_fqsn()}:\n'
f'Pos size mismatch {pairinfo.front_fqsn()}:\n'
f'ib: {msg.size}\n'
f'piker: {pp.size}\n'
)
@ -1095,13 +1102,15 @@ def norm_trade_records(
'''
records: list[Transaction] = []
for tid, record in ledger.items():
for tid, record in ledger.items():
conid = record.get('conId') or record['conid']
comms = record.get('commission')
if comms is None:
comms = -1*record['ibCommission']
price = record.get('price') or record['tradePrice']
price_tick_digits = float_digits(price)
# the api doesn't do the -/+ on the quantity for you but flex
# records do.. are you fucking serious ib...!?
@ -1144,9 +1153,14 @@ def norm_trade_records(
# special handling of symbol extraction from
# flex records using some ad-hoc schema parsing.
instr = record.get('assetCategory')
if instr == 'FUT':
symbol = record['description'][:3]
asset_type: str = record.get('assetCategory') or record['secType']
# TODO: XXX: WOA this is kinda hacky.. probably
# should figure out the correct future pair key more
# explicitly and consistently?
if asset_type == 'FUT':
# (flex) ledger entries don't have any simple 3-char key?
symbol = record['symbol'][:3]
# try to build out piker fqsn from record.
expiry = record.get(
@ -1156,10 +1170,34 @@ def norm_trade_records(
suffix = f'{exch}.{expiry}'
expiry = pendulum.parse(expiry)
fqsn = Symbol.from_fqsn(
src: str = record['currency']
pair = Symbol.from_fqsn(
fqsn=f'{symbol}.{suffix}.ib',
info={},
).front_fqsn().rstrip('.ib')
info={
'tick_size_digits': price_tick_digits,
# NOTE: for "legacy" assets, volume is normally discreet, not
# a float, but we keep a digit in case the suitz decide
# to get crazy and change it; we'll be kinda ready
# schema-wise..
'lot_size_digits': 1,
# TODO: remove when we switching from
# ``Symbol`` -> ``MktPair``
'asset_type': asset_type,
# TODO: figure out a target fin-type name
# set and normalize to that here!
'dst_type': asset_type.lower(),
# starting to use new key naming as in ``MktPair``
# type have drafted...
'src': src,
'src_type': 'fiat',
},
)
fqsn = pair.front_fqsn().rstrip('.ib')
# NOTE: for flex records the normal fields for defining an fqsn
# sometimes won't be available so we rely on two approaches for
@ -1175,6 +1213,7 @@ def norm_trade_records(
records,
Transaction(
fqsn=fqsn,
sym=pair,
tid=tid,
size=size,
price=price,
@ -1201,7 +1240,11 @@ def parse_flex_dt(
def api_trades_to_ledger_entries(
accounts: bidict,
trade_entries: list[object],
# TODO: maybe we should just be passing through the
# ``ib_insync.order.Trade`` instance directly here
# instead of pre-casting to dicts?
trade_entries: list[dict],
) -> dict:
'''

2
piker/brokers/ib/feed.py
View File

@ -770,7 +770,7 @@ async def stream_quotes(
syminfo['price_tick_size'] = max(syminfo['minTick'], min_tick)
# for "traditional" assets, volume is normally discreet, not
# for "legacy" assets, volume is normally discreet, not
# a float
syminfo['lot_tick_size'] = 0.0

131
piker/brokers/kraken/api.py
View File

@ -40,6 +40,8 @@ import base64
import trio
from piker import config
from piker.data.types import Struct
from piker.data._source import Symbol
from piker.brokers._util import (
resproc,
SymbolNotFound,
@ -113,11 +115,53 @@ class InvalidKey(ValueError):
'''
# https://www.kraken.com/features/api#get-tradable-pairs
class Pair(Struct):
altname: str # alternate pair name
wsname: str # WebSocket pair name (if available)
aclass_base: str # asset class of base component
base: str # asset id of base component
aclass_quote: str # asset class of quote component
quote: str # asset id of quote component
lot: str # volume lot size
cost_decimals: int
costmin: float
pair_decimals: int # scaling decimal places for pair
lot_decimals: int # scaling decimal places for volume
# amount to multiply lot volume by to get currency volume
lot_multiplier: float
# array of leverage amounts available when buying
leverage_buy: list[int]
# array of leverage amounts available when selling
leverage_sell: list[int]
# fee schedule array in [volume, percent fee] tuples
fees: list[tuple[int, float]]
# maker fee schedule array in [volume, percent fee] tuples (if on
# maker/taker)
fees_maker: list[tuple[int, float]]
fee_volume_currency: str # volume discount currency
margin_call: str # margin call level
margin_stop: str # stop-out/liquidation margin level
ordermin: float # minimum order volume for pair
tick_size: float # min price step size
status: str
short_position_limit: float = 0
long_position_limit: float = float('inf')
class Client:
# global symbol normalization table
_ntable: dict[str, str] = {}
_atable: bidict[str, str] = bidict()
_pairs: dict[str, Pair] = {}
def __init__(
self,
@ -133,13 +177,12 @@ class Client:
'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)'
})
self.conf: dict[str, str] = config
self._pairs: list[str] = []
self._name = name
self._api_key = api_key
self._secret = secret
@property
def pairs(self) -> dict[str, Any]:
def pairs(self) -> dict[str, Pair]:
if self._pairs is None:
raise RuntimeError(
"Make sure to run `cache_symbols()` on startup!"
@ -295,15 +338,28 @@ class Client:
trans: dict[str, Transaction] = {}
for entry in xfers:
# look up the normalized name
asset = self._atable[entry['asset']].lower()
# look up the normalized name and asset info
asset_key = entry['asset']
asset_info = self.assets[asset_key]
asset = self._atable[asset_key].lower()
# XXX: this is in the asset units (likely) so it isn't
# quite the same as a commisions cost necessarily..)
cost = float(entry['fee'])
fqsn = asset + '.kraken'
pairinfo = Symbol.from_fqsn(
fqsn,
info={
'asset_type': 'crypto',
'lot_tick_size': asset_info['decimals'],
},
)
tran = Transaction(
fqsn=asset + '.kraken',
fqsn=fqsn,
sym=pairinfo,
tid=entry['txid'],
dt=pendulum.from_timestamp(entry['time']),
bsuid=f'{asset}{src_asset}',
@ -317,7 +373,7 @@ class Client:
price='NaN',
# XXX: see note above
cost=0,
cost=cost,
)
trans[tran.tid] = tran
@ -372,7 +428,7 @@ class Client:
self,
pair: Optional[str] = None,
) -> dict[str, dict[str, str]]:
) -> dict[str, Pair] | Pair:
if pair is not None:
pairs = {'pair': pair}
@ -389,19 +445,36 @@ class Client:
if pair is not None:
_, data = next(iter(pairs.items()))
return data
return Pair(**data)
else:
return pairs
return {key: Pair(**data) for key, data in pairs.items()}
async def cache_symbols(
self,
) -> dict:
async def cache_symbols(self) -> dict:
'''
Load all market pair info build and cache it for downstream use.
A ``._ntable: dict[str, str]`` is available for mapping the
websocket pair name-keys and their http endpoint API (smh)
equivalents to the "alternative name" which is generally the one
we actually want to use XD
'''
if not self._pairs:
self._pairs = await self.symbol_info()
self._pairs.update(await self.symbol_info())
ntable = {}
for restapikey, info in self._pairs.items():
ntable[restapikey] = ntable[info['wsname']] = info['altname']
# table of all ws and rest keys to their alt-name values.
ntable: dict[str, str] = {}
for rest_key in list(self._pairs.keys()):
pair: Pair = self._pairs[rest_key]
altname = pair.altname
wsname = pair.wsname
ntable[rest_key] = ntable[wsname] = altname
# register the pair under all monikers, a giant flat
# surjection of all possible names to each info obj.
self._pairs[altname] = self._pairs[wsname] = pair
self._ntable.update(ntable)
@ -411,26 +484,34 @@ class Client:
self,
pattern: str,
limit: int = None,
) -> dict[str, Any]:
if self._pairs is not None:
data = self._pairs
else:
data = await self.symbol_info()
'''
Search for a symbol by "alt name"..
It is expected that the ``Client._pairs`` table
gets populated before conducting the underlying fuzzy-search
over the pair-key set.
'''
if not len(self._pairs):
await self.cache_symbols()
assert self._pairs, '`Client.cache_symbols()` was never called!?'
matches = fuzzy.extractBests(
pattern,
data,
self._pairs,
score_cutoff=50,
)
# repack in dict form
return {item[0]['altname']: item[0] for item in matches}
return {item[0].altname: item[0] for item in matches}
async def bars(
self,
symbol: str = 'XBTUSD',
# UTC 2017-07-02 12:53:20
since: Optional[Union[int, datetime]] = None,
since: Union[int, datetime] | None = None,
count: int = 720, # <- max allowed per query
as_np: bool = True,
@ -506,7 +587,7 @@ class Client:
def normalize_symbol(
cls,
ticker: str
) -> str:
) -> tuple[str, Pair]:
'''
Normalize symbol names to to a 3x3 pair from the global
definition map which we build out from the data retreived from
@ -514,7 +595,7 @@ class Client:
'''
ticker = cls._ntable[ticker]
return ticker.lower()
return ticker.lower(), cls._pairs[ticker]
@acm

16
piker/brokers/kraken/broker.py
View File

@ -48,6 +48,7 @@ from piker.pp import (
open_trade_ledger,
open_pps,
)
from piker.data._source import Symbol
from piker.clearing._messages import (
Order,
Status,
@ -1196,10 +1197,21 @@ def norm_trade_records(
}[record['type']]
# we normalize to kraken's `altname` always..
bsuid = norm_sym = Client.normalize_symbol(record['pair'])
bsuid, pair_info = Client.normalize_symbol(record['pair'])
fqsn = f'{bsuid}.kraken'
mktpair = Symbol.from_fqsn(
fqsn,
info={
'lot_size_digits': pair_info.lot_decimals,
'tick_size_digits': pair_info.pair_decimals,
'asset_type': 'crypto',
},
)
records[tid] = Transaction(
fqsn=f'{norm_sym}.kraken',
fqsn=fqsn,
sym=mktpair,
tid=tid,
size=size,
price=float(record['price']),

59
piker/brokers/kraken/feed.py
View File

@ -42,56 +42,15 @@ from piker.brokers._util import (
DataUnavailable,
)
from piker.log import get_console_log
from piker.data import ShmArray
from piker.data.types import Struct
from piker.data._web_bs import open_autorecon_ws, NoBsWs
from . import log
from .api import (
Client,
Pair,
)
# https://www.kraken.com/features/api#get-tradable-pairs
class Pair(Struct):
altname: str # alternate pair name
wsname: str # WebSocket pair name (if available)
aclass_base: str # asset class of base component
base: str # asset id of base component
aclass_quote: str # asset class of quote component
quote: str # asset id of quote component
lot: str # volume lot size
cost_decimals: int
costmin: float
pair_decimals: int # scaling decimal places for pair
lot_decimals: int # scaling decimal places for volume
# amount to multiply lot volume by to get currency volume
lot_multiplier: float
# array of leverage amounts available when buying
leverage_buy: list[int]
# array of leverage amounts available when selling
leverage_sell: list[int]
# fee schedule array in [volume, percent fee] tuples
fees: list[tuple[int, float]]
# maker fee schedule array in [volume, percent fee] tuples (if on
# maker/taker)
fees_maker: list[tuple[int, float]]
fee_volume_currency: str # volume discount currency
margin_call: str # margin call level
margin_stop: str # stop-out/liquidation margin level
ordermin: float # minimum order volume for pair
tick_size: float # min price step size
status: str
short_position_limit: float
long_position_limit: float
class OHLC(Struct):
'''
Description of the flattened OHLC quote format.
@ -336,14 +295,14 @@ async def stream_quotes(
# transform to upper since piker style is always lower
sym = sym.upper()
sym_info = await client.symbol_info(sym)
try:
si = Pair(**sym_info) # validation
except TypeError:
fields_diff = set(sym_info) - set(Pair.__struct_fields__)
raise TypeError(
f'Missing msg fields {fields_diff}'
)
si: Pair = await client.symbol_info(sym)
# try:
# si = Pair(**sym_info) # validation
# except TypeError:
# fields_diff = set(sym_info) - set(Pair.__struct_fields__)
# raise TypeError(
# f'Missing msg fields {fields_diff}'
# )
syminfo = si.to_dict()
syminfo['price_tick_size'] = 1. / 10**si.pair_decimals
syminfo['lot_tick_size'] = 1. / 10**si.lot_decimals

83
piker/data/_source.py
View File

@ -18,7 +18,10 @@
numpy data source coversion helpers.
"""
from __future__ import annotations
from decimal import Decimal, ROUND_HALF_EVEN
from decimal import (
Decimal,
ROUND_HALF_EVEN,
)
from typing import Any
from bidict import bidict
@ -77,6 +80,10 @@ def mk_fqsn(
def float_digits(
value: float,
) -> int:
'''
Return the number of precision digits read from a float value.
'''
if value == 0:
return 0
@ -127,6 +134,56 @@ def unpack_fqsn(fqsn: str) -> tuple[str, str, str]:
)
class MktPair(Struct, frozen=True):
src: str # source asset name being used to buy
src_type: str # source asset's financial type/classification name
# ^ specifies a "class" of financial instrument
# egs. stock, futer, option, bond etc.
dst: str # destination asset name being bought
dst_type: str # destination asset's financial type/classification name
price_tick: float # minimum price increment value increment
price_tick_digits: int # required decimal digits for above
size_tick: float # minimum size (aka vlm) increment value increment
size_tick_digits: int # required decimal digits for above
venue: str | None = None # market venue provider name
expiry: str | None = None # for derivs, expiry datetime parseable str
# for derivs, info describing contract, egs.
# strike price, call or put, swap type, exercise model, etc.
contract_info: str | None = None
@classmethod
def from_msg(
self,
msg: dict[str, Any],
) -> MktPair:
'''
Constructor for a received msg-dict normally received over IPC.
'''
...
# fqa, fqma, .. etc. see issue:
# https://github.com/pikers/piker/issues/467
@property
def fqsn(self) -> str:
'''
Return the fully qualified market (endpoint) name for the
pair of transacting assets.
'''
...
# TODO: rework the below `Symbol` (which was originally inspired and
# derived from stuff in quantdom) into a simpler, ipc msg ready, market
# endpoint meta-data container type as per the drafted interace above.
class Symbol(Struct):
'''
I guess this is some kinda container thing for dealing with
@ -141,10 +198,6 @@ class Symbol(Struct):
suffix: str = ''
broker_info: dict[str, dict[str, Any]] = {}
# specifies a "class" of financial instrument
# ex. stock, futer, option, bond etc.
# @validate_arguments
@classmethod
def from_broker_info(
cls,
@ -244,23 +297,23 @@ class Symbol(Struct):
fqsn = '.'.join(map(str.lower, tokens))
return fqsn
def iterfqsns(self) -> list[str]:
keys = []
for broker in self.broker_info.keys():
fqsn = mk_fqsn(self.key, broker)
if self.suffix:
fqsn += f'.{self.suffix}'
keys.append(fqsn)
def quantize_size(
self,
size: float,
return keys
) -> Decimal:
'''
Truncate input ``size: float`` using ``Decimal``
and ``.lot_size_digits``.
def decimal_quant(self, d: Decimal):
'''
digits = self.lot_size_digits
return d.quantize(
return Decimal(size).quantize(
Decimal(f'1.{"0".ljust(digits, "0")}'),
rounding=ROUND_HALF_EVEN
)
def _nan_to_closest_num(array: np.ndarray):
"""Return interpolated values instead of NaN.

30
piker/pp.py
View File

@ -22,8 +22,6 @@ that doesn't try to cuk most humans who prefer to not lose their moneys..
'''
from __future__ import annotations
from contextlib import contextmanager as cm
from pathlib import Path
from decimal import Decimal, ROUND_HALF_EVEN
from pprint import pformat
import os
from os import path
@ -91,10 +89,12 @@ def open_trade_ledger(
yield cpy
finally:
if cpy != ledger:
# TODO: show diff output?
# https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries
log.info(f'Updating ledger for {tradesfile}:\n')
ledger.update(cpy)
ledger.update(cpy)
# we write on close the mutated ledger data
with open(tradesfile, 'w') as cf:
toml.dump(ledger, cf)
@ -476,7 +476,7 @@ class Position(Struct):
if self.split_ratio is not None:
size = round(size * self.split_ratio)
return float(self.symbol.decimal_quant(Decimal(size)))
return float(self.symbol.quantize_size(size))
def minimize_clears(
self,
@ -934,10 +934,18 @@ def open_pps(
for fqsn, entry in pps.items():
bsuid = entry['bsuid']
symbol = Symbol.from_fqsn(
fqsn, info={
'asset_type': entry['asset_type'],
'price_tick_size': entry['price_tick_size'],
'lot_tick_size': entry['lot_tick_size']
fqsn,
# NOTE & TODO: right now we fill in the defaults from
# `.data._source.Symbol` but eventually these should always
# either be already written to the pos table or provided at
# write time to ensure always having these values somewhere
# and thus allowing us to get our pos sizing precision
# correct!
info={
'asset_type': entry.get('asset_type', '<unknown>'),
'price_tick_size': entry.get('price_tick_size', 0.01),
'lot_tick_size': entry.get('lot_tick_size', 0.0),
}
)
@ -977,7 +985,11 @@ def open_pps(
size = entry['size']
# TODO: remove but, handle old field name for now
ppu = entry.get('ppu', entry.get('be_price', 0))
ppu = entry.get(
'ppu',
entry.get('be_price', 0),
)
split_ratio = entry.get('split_ratio')
expiry = entry.get('expiry')