Merge pull request #477 from pikers/backward_compat_trans_with_symbolinfo

Backward compat support for `Transaction.sym: Symbol`

piker/brokers/ib/broker.py

@@ -70,7 +70,10 @@ from piker.clearing._messages import (
     BrokerdFill,
     BrokerdError,
 )
-from piker.data._source import Symbol
+from piker.data._source import (
+    Symbol,
+    float_digits,
+)
 from .api import (
     _accounts2clients,
     con2fqsn,
@@ -304,6 +307,9 @@ async def update_ledger_from_api_trades(
 
         entry['listingExchange'] = pexch
 
+        # pack in the ``Contract.secType``
+        entry['asset_type'] = condict['secType']
+
     conf = get_config()
     entries = api_trades_to_ledger_entries(
         conf['accounts'].inverse,
@@ -616,9 +622,10 @@ async def trades_dialogue(
                         # from the api trades it seems we get a key
                         # error from ``update[bsuid]`` ?
                         pp = table.pps[bsuid]
+                        pairinfo = pp.symbol
                         if msg.size != pp.size:
                             log.error(
-                                f'Position mismatch {pp.symbol.front_fqsn()}:\n'
+                                f'Pos size mismatch {pairinfo.front_fqsn()}:\n'
                                 f'ib: {msg.size}\n'
                                 f'piker: {pp.size}\n'
                             )
@@ -1095,13 +1102,15 @@ def norm_trade_records(
 
     '''
     records: list[Transaction] = []
-    for tid, record in ledger.items():
 
+    for tid, record in ledger.items():
         conid = record.get('conId') or record['conid']
         comms = record.get('commission')
         if comms is None:
             comms = -1*record['ibCommission']
+
         price = record.get('price') or record['tradePrice']
+        price_tick_digits = float_digits(price)
 
         # the api doesn't do the -/+ on the quantity for you but flex
         # records do.. are you fucking serious ib...!?
@@ -1144,9 +1153,14 @@ def norm_trade_records(
 
         # special handling of symbol extraction from
         # flex records using some ad-hoc schema parsing.
-        instr = record.get('assetCategory')
+        asset_type: str = record.get('assetCategory') or record['secType']
-        if instr == 'FUT':
+
-            symbol = record['description'][:3]
+        # TODO: XXX: WOA this is kinda hacky.. probably
+        # should figure out the correct future pair key more
+        # explicitly and consistently?
+        if asset_type == 'FUT':
+            # (flex) ledger entries don't have any simple 3-char key?
+            symbol = record['symbol'][:3]
 
         # try to build out piker fqsn from record.
         expiry = record.get(
@@ -1156,10 +1170,34 @@ def norm_trade_records(
             suffix = f'{exch}.{expiry}'
             expiry = pendulum.parse(expiry)
 
-        fqsn = Symbol.from_fqsn(
+        src: str = record['currency']
+
+        pair = Symbol.from_fqsn(
             fqsn=f'{symbol}.{suffix}.ib',
-            info={},
+            info={
-        ).front_fqsn().rstrip('.ib')
+                'tick_size_digits': price_tick_digits,
+
+                # NOTE: for "legacy" assets, volume is normally discreet, not
+                # a float, but we keep a digit in case the suitz decide
+                # to get crazy and change it; we'll be kinda ready
+                # schema-wise..
+                'lot_size_digits': 1,
+
+                # TODO: remove when we switching from
+                # ``Symbol`` -> ``MktPair``
+                'asset_type': asset_type,
+
+                # TODO: figure out a target fin-type name
+                # set and normalize to that here!
+                'dst_type': asset_type.lower(),
+
+                # starting to use new key naming as in ``MktPair``
+                # type have drafted...
+                'src': src,
+                'src_type': 'fiat',
+            },
+        )
+        fqsn = pair.front_fqsn().rstrip('.ib')
 
         # NOTE: for flex records the normal fields for defining an fqsn
         # sometimes won't be available so we rely on two approaches for
@@ -1175,6 +1213,7 @@ def norm_trade_records(
             records,
             Transaction(
                 fqsn=fqsn,
+                sym=pair,
                 tid=tid,
                 size=size,
                 price=price,
@@ -1201,7 +1240,11 @@ def parse_flex_dt(
 
 def api_trades_to_ledger_entries(
     accounts: bidict,
-    trade_entries: list[object],
+
+    # TODO: maybe we should just be passing through the
+    # ``ib_insync.order.Trade`` instance directly here
+    # instead of pre-casting to dicts?
+    trade_entries: list[dict],
 
 ) -> dict:
     '''

piker/brokers/ib/feed.py

@@ -770,7 +770,7 @@ async def stream_quotes(
 
             syminfo['price_tick_size'] = max(syminfo['minTick'], min_tick)
 
-            # for "traditional" assets, volume is normally discreet, not
+            # for "legacy" assets, volume is normally discreet, not
             # a float
             syminfo['lot_tick_size'] = 0.0
 

piker/brokers/kraken/api.py

@@ -40,6 +40,8 @@ import base64
 import trio
 
 from piker import config
+from piker.data.types import Struct
+from piker.data._source import Symbol
 from piker.brokers._util import (
     resproc,
     SymbolNotFound,
@@ -113,11 +115,53 @@ class InvalidKey(ValueError):
     '''
 
 
+# https://www.kraken.com/features/api#get-tradable-pairs
+class Pair(Struct):
+    altname: str  # alternate pair name
+    wsname: str  # WebSocket pair name (if available)
+    aclass_base: str  # asset class of base component
+    base: str  # asset id of base component
+    aclass_quote: str  # asset class of quote component
+    quote: str  # asset id of quote component
+    lot: str  # volume lot size
+
+    cost_decimals: int
+    costmin: float
+    pair_decimals: int  # scaling decimal places for pair
+    lot_decimals: int  # scaling decimal places for volume
+
+    # amount to multiply lot volume by to get currency volume
+    lot_multiplier: float
+
+    # array of leverage amounts available when buying
+    leverage_buy: list[int]
+    # array of leverage amounts available when selling
+    leverage_sell: list[int]
+
+    # fee schedule array in [volume, percent fee] tuples
+    fees: list[tuple[int, float]]
+
+    # maker fee schedule array in [volume, percent fee] tuples (if on
+    # maker/taker)
+    fees_maker: list[tuple[int, float]]
+
+    fee_volume_currency: str  # volume discount currency
+    margin_call: str  # margin call level
+    margin_stop: str  # stop-out/liquidation margin level
+    ordermin: float  # minimum order volume for pair
+    tick_size: float  # min price step size
+    status: str
+
+    short_position_limit: float = 0
+    long_position_limit: float = float('inf')
+
+
 class Client:
 
     # global symbol normalization table
     _ntable: dict[str, str] = {}
     _atable: bidict[str, str] = bidict()
+    _pairs: dict[str, Pair] = {}
 
     def __init__(
         self,
@@ -133,13 +177,12 @@ class Client:
                 'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)'
         })
         self.conf: dict[str, str] = config
-        self._pairs: list[str] = []
         self._name = name
         self._api_key = api_key
         self._secret = secret
 
     @property
-    def pairs(self) -> dict[str, Any]:
+    def pairs(self) -> dict[str, Pair]:
         if self._pairs is None:
             raise RuntimeError(
                 "Make sure to run `cache_symbols()` on startup!"
@@ -295,15 +338,28 @@ class Client:
 
         trans: dict[str, Transaction] = {}
         for entry in xfers:
-            # look up the normalized name
+
-            asset = self._atable[entry['asset']].lower()
+            # look up the normalized name and asset info
+            asset_key = entry['asset']
+            asset_info = self.assets[asset_key]
+            asset = self._atable[asset_key].lower()
 
             # XXX: this is in the asset units (likely) so it isn't
             # quite the same as a commisions cost necessarily..)
             cost = float(entry['fee'])
 
+            fqsn = asset + '.kraken'
+            pairinfo = Symbol.from_fqsn(
+                fqsn,
+                info={
+                    'asset_type': 'crypto',
+                    'lot_tick_size': asset_info['decimals'],
+                },
+            )
+
             tran = Transaction(
-                fqsn=asset + '.kraken',
+                fqsn=fqsn,
+                sym=pairinfo,
                 tid=entry['txid'],
                 dt=pendulum.from_timestamp(entry['time']),
                 bsuid=f'{asset}{src_asset}',
@@ -317,7 +373,7 @@ class Client:
                 price='NaN',
 
                 # XXX: see note above
-                cost=0,
+                cost=cost,
             )
             trans[tran.tid] = tran
 
@@ -372,7 +428,7 @@ class Client:
         self,
         pair: Optional[str] = None,
 
-    ) -> dict[str, dict[str, str]]:
+    ) -> dict[str, Pair] | Pair:
 
         if pair is not None:
             pairs = {'pair': pair}
@@ -389,19 +445,36 @@ class Client:
 
         if pair is not None:
             _, data = next(iter(pairs.items()))
-            return data
+            return Pair(**data)
         else:
-            return pairs
+            return {key: Pair(**data) for key, data in pairs.items()}
 
-    async def cache_symbols(
+    async def cache_symbols(self) -> dict:
-        self,
+        '''
-    ) -> dict:
+        Load all market pair info build and cache it for downstream use.
+
+        A ``._ntable: dict[str, str]`` is available for mapping the
+        websocket pair name-keys and their http endpoint API (smh)
+        equivalents to the "alternative name" which is generally the one
+        we actually want to use XD
+
+        '''
         if not self._pairs:
-            self._pairs = await self.symbol_info()
+            self._pairs.update(await self.symbol_info())
 
-            ntable = {}
+            # table of all ws and rest keys to their alt-name values.
-            for restapikey, info in self._pairs.items():
+            ntable: dict[str, str] = {}
-                ntable[restapikey] = ntable[info['wsname']] = info['altname']
+
+            for rest_key in list(self._pairs.keys()):
+
+                pair: Pair = self._pairs[rest_key]
+                altname = pair.altname
+                wsname = pair.wsname
+                ntable[rest_key] = ntable[wsname] = altname
+
+                # register the pair under all monikers, a giant flat
+                # surjection of all possible names to each info obj.
+                self._pairs[altname] = self._pairs[wsname] = pair
 
             self._ntable.update(ntable)
 
@@ -411,26 +484,34 @@ class Client:
         self,
         pattern: str,
         limit: int = None,
+
     ) -> dict[str, Any]:
-        if self._pairs is not None:
+        '''
-            data = self._pairs
+        Search for a symbol by "alt name"..
-        else:
+
-            data = await self.symbol_info()
+        It is expected that the ``Client._pairs`` table
+        gets populated before conducting the underlying fuzzy-search
+        over the pair-key set.
+
+        '''
+        if not len(self._pairs):
+            await self.cache_symbols()
+            assert self._pairs, '`Client.cache_symbols()` was never called!?'
 
         matches = fuzzy.extractBests(
             pattern,
-            data,
+            self._pairs,
             score_cutoff=50,
         )
         # repack in dict form
-        return {item[0]['altname']: item[0] for item in matches}
+        return {item[0].altname: item[0] for item in matches}
 
     async def bars(
         self,
         symbol: str = 'XBTUSD',
 
         # UTC 2017-07-02 12:53:20
-        since: Optional[Union[int, datetime]] = None,
+        since: Union[int, datetime] | None = None,
         count: int = 720,  # <- max allowed per query
         as_np: bool = True,
 
@@ -506,7 +587,7 @@ class Client:
     def normalize_symbol(
         cls,
         ticker: str
-    ) -> str:
+    ) -> tuple[str, Pair]:
         '''
         Normalize symbol names to to a 3x3 pair from the global
         definition map which we build out from the data retreived from
@@ -514,7 +595,7 @@ class Client:
 
         '''
         ticker = cls._ntable[ticker]
-        return ticker.lower()
+        return ticker.lower(), cls._pairs[ticker]
 
 
 @acm

piker/brokers/kraken/broker.py

@@ -48,6 +48,7 @@ from piker.pp import (
     open_trade_ledger,
     open_pps,
 )
+from piker.data._source import Symbol
 from piker.clearing._messages import (
     Order,
     Status,
@@ -1196,10 +1197,21 @@ def norm_trade_records(
         }[record['type']]
 
         # we normalize to kraken's `altname` always..
-        bsuid = norm_sym = Client.normalize_symbol(record['pair'])
+        bsuid, pair_info = Client.normalize_symbol(record['pair'])
+        fqsn = f'{bsuid}.kraken'
+
+        mktpair = Symbol.from_fqsn(
+            fqsn,
+            info={
+                'lot_size_digits': pair_info.lot_decimals,
+                'tick_size_digits': pair_info.pair_decimals,
+                'asset_type': 'crypto',
+            },
+        )
 
         records[tid] = Transaction(
-            fqsn=f'{norm_sym}.kraken',
+            fqsn=fqsn,
+            sym=mktpair,
             tid=tid,
             size=size,
             price=float(record['price']),

piker/brokers/kraken/feed.py

@@ -42,56 +42,15 @@ from piker.brokers._util import (
     DataUnavailable,
 )
 from piker.log import get_console_log
-from piker.data import ShmArray
 from piker.data.types import Struct
 from piker.data._web_bs import open_autorecon_ws, NoBsWs
 from . import log
 from .api import (
     Client,
+    Pair,
 )
 
 
-# https://www.kraken.com/features/api#get-tradable-pairs
-class Pair(Struct):
-    altname: str  # alternate pair name
-    wsname: str  # WebSocket pair name (if available)
-    aclass_base: str  # asset class of base component
-    base: str  # asset id of base component
-    aclass_quote: str  # asset class of quote component
-    quote: str  # asset id of quote component
-    lot: str  # volume lot size
-
-    cost_decimals: int
-    costmin: float
-    pair_decimals: int  # scaling decimal places for pair
-    lot_decimals: int  # scaling decimal places for volume
-
-    # amount to multiply lot volume by to get currency volume
-    lot_multiplier: float
-
-    # array of leverage amounts available when buying
-    leverage_buy: list[int]
-    # array of leverage amounts available when selling
-    leverage_sell: list[int]
-
-    # fee schedule array in [volume, percent fee] tuples
-    fees: list[tuple[int, float]]
-
-    # maker fee schedule array in [volume, percent fee] tuples (if on
-    # maker/taker)
-    fees_maker: list[tuple[int, float]]
-
-    fee_volume_currency: str  # volume discount currency
-    margin_call: str  # margin call level
-    margin_stop: str  # stop-out/liquidation margin level
-    ordermin: float  # minimum order volume for pair
-    tick_size: float # min price step size
-    status: str
-
-    short_position_limit: float
-    long_position_limit: float
-
-
 class OHLC(Struct):
     '''
     Description of the flattened OHLC quote format.
@@ -336,14 +295,14 @@ async def stream_quotes(
 
             # transform to upper since piker style is always lower
             sym = sym.upper()
-            sym_info = await client.symbol_info(sym)
+            si: Pair = await client.symbol_info(sym)
-            try:
+            # try:
-                si = Pair(**sym_info)  # validation
+            #     si = Pair(**sym_info)  # validation
-            except TypeError:
+            # except TypeError:
-                fields_diff = set(sym_info) - set(Pair.__struct_fields__)
+            #     fields_diff = set(sym_info) - set(Pair.__struct_fields__)
-                raise TypeError(
+            #     raise TypeError(
-                    f'Missing msg fields {fields_diff}'
+            #         f'Missing msg fields {fields_diff}'
-                )
+            #     )
             syminfo = si.to_dict()
             syminfo['price_tick_size'] = 1. / 10**si.pair_decimals
             syminfo['lot_tick_size'] = 1. / 10**si.lot_decimals

piker/data/_source.py

@@ -18,7 +18,10 @@
 numpy data source coversion helpers.
 """
 from __future__ import annotations
-from decimal import Decimal, ROUND_HALF_EVEN
+from decimal import (
+    Decimal,
+    ROUND_HALF_EVEN,
+)
 from typing import Any
 
 from bidict import bidict
@@ -77,6 +80,10 @@ def mk_fqsn(
 def float_digits(
     value: float,
 ) -> int:
+    '''
+    Return the number of precision digits read from a float value.
+
+    '''
     if value == 0:
         return 0
 
@@ -127,6 +134,56 @@ def unpack_fqsn(fqsn: str) -> tuple[str, str, str]:
     )
 
 
+class MktPair(Struct, frozen=True):
+
+    src: str  # source asset name being used to buy
+    src_type: str  # source asset's financial type/classification name
+    # ^ specifies a "class" of financial instrument
+    # egs. stock, futer, option, bond etc.
+
+    dst: str  # destination asset name being bought
+    dst_type: str  # destination asset's financial type/classification name
+
+    price_tick: float  # minimum price increment value increment
+    price_tick_digits: int  # required decimal digits for above
+
+    size_tick: float  # minimum size (aka vlm) increment value increment
+    size_tick_digits: int  # required decimal digits for above
+
+    venue: str | None = None  # market venue provider name
+    expiry: str | None = None  # for derivs, expiry datetime parseable str
+
+    # for derivs, info describing contract, egs.
+    # strike price, call or put, swap type, exercise model, etc.
+    contract_info: str | None = None
+
+    @classmethod
+    def from_msg(
+        self,
+        msg: dict[str, Any],
+
+    ) -> MktPair:
+        '''
+        Constructor for a received msg-dict normally received over IPC.
+
+        '''
+        ...
+
+    # fqa, fqma, .. etc. see issue:
+    # https://github.com/pikers/piker/issues/467
+    @property
+    def fqsn(self) -> str:
+        '''
+        Return the fully qualified market (endpoint) name for the
+        pair of transacting assets.
+
+        '''
+        ...
+
+
+# TODO: rework the below `Symbol` (which was originally inspired and
+# derived from stuff in quantdom) into a simpler, ipc msg ready, market
+# endpoint meta-data container type as per the drafted interace above.
 class Symbol(Struct):
     '''
     I guess this is some kinda container thing for dealing with
@@ -141,10 +198,6 @@ class Symbol(Struct):
     suffix: str = ''
     broker_info: dict[str, dict[str, Any]] = {}
 
-    # specifies a "class" of financial instrument
-    # ex. stock, futer, option, bond etc.
-
-    # @validate_arguments
     @classmethod
     def from_broker_info(
         cls,
@@ -244,23 +297,23 @@ class Symbol(Struct):
         fqsn = '.'.join(map(str.lower, tokens))
         return fqsn
 
-    def iterfqsns(self) -> list[str]:
+    def quantize_size(
-        keys = []
+        self,
-        for broker in self.broker_info.keys():
+        size: float,
-            fqsn = mk_fqsn(self.key, broker)
-            if self.suffix:
-                fqsn += f'.{self.suffix}'
-            keys.append(fqsn)
 
-        return keys
+    ) -> Decimal:
+        '''
+        Truncate input ``size: float`` using ``Decimal``
+        and ``.lot_size_digits``.
 
-    def decimal_quant(self, d: Decimal):
+        '''
         digits = self.lot_size_digits
-        return d.quantize(
+        return Decimal(size).quantize(
             Decimal(f'1.{"0".ljust(digits, "0")}'),
             rounding=ROUND_HALF_EVEN
         )
 
+
 def _nan_to_closest_num(array: np.ndarray):
     """Return interpolated values instead of NaN.
 

piker/pp.py

@@ -22,8 +22,6 @@ that doesn't try to cuk most humans who prefer to not lose their moneys..
 '''
 from __future__ import annotations
 from contextlib import contextmanager as cm
-from pathlib import Path
-from decimal import Decimal, ROUND_HALF_EVEN
 from pprint import pformat
 import os
 from os import path
@@ -91,10 +89,12 @@ def open_trade_ledger(
         yield cpy
     finally:
         if cpy != ledger:
+
             # TODO: show diff output?
             # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries
             log.info(f'Updating ledger for {tradesfile}:\n')
             ledger.update(cpy)
+
             # we write on close the mutated ledger data
             with open(tradesfile, 'w') as cf:
                 toml.dump(ledger, cf)
@@ -476,7 +476,7 @@ class Position(Struct):
         if self.split_ratio is not None:
             size = round(size * self.split_ratio)
 
-        return float(self.symbol.decimal_quant(Decimal(size)))
+        return float(self.symbol.quantize_size(size))
 
     def minimize_clears(
         self,
@@ -934,10 +934,18 @@ def open_pps(
     for fqsn, entry in pps.items():
         bsuid = entry['bsuid']
         symbol = Symbol.from_fqsn(
-            fqsn, info={
+            fqsn,
-                'asset_type': entry['asset_type'],
+
-                'price_tick_size': entry['price_tick_size'],
+            # NOTE & TODO: right now we fill in the defaults from
-                'lot_tick_size': entry['lot_tick_size']
+            # `.data._source.Symbol` but eventually these should always
+            # either be already written to the pos table or provided at
+            # write time to ensure always having these values somewhere
+            # and thus allowing us to get our pos sizing precision
+            # correct!
+            info={
+                'asset_type': entry.get('asset_type', '<unknown>'),
+                'price_tick_size': entry.get('price_tick_size', 0.01),
+                'lot_tick_size': entry.get('lot_tick_size', 0.0),
             }
         )
 
@@ -977,7 +985,11 @@ def open_pps(
         size = entry['size']
 
         # TODO: remove but, handle old field name for now
-        ppu = entry.get('ppu', entry.get('be_price', 0))
+        ppu = entry.get(
+            'ppu',
+            entry.get('be_price', 0),
+        )
+
         split_ratio = entry.get('split_ratio')
 
         expiry = entry.get('expiry')