From 9f03484c4d809d1870627f290081ec99fa62103b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 13 Mar 2023 17:42:20 -0400 Subject: [PATCH] Move all fqsn parsing and `Symbol` to new `accounting._mktinfo --- piker/accounting/_ledger.py | 2 +- piker/accounting/_mktinfo.py | 302 ++++++++++++++++++++++++++++++++ piker/accounting/_pos.py | 6 +- piker/brokers/ib/api.py | 2 +- piker/brokers/ib/broker.py | 2 +- piker/brokers/kraken/api.py | 2 +- piker/brokers/kraken/broker.py | 2 +- piker/clearing/_allocate.py | 2 +- piker/clearing/_client.py | 2 +- piker/clearing/_ems.py | 3 +- piker/clearing/_messages.py | 2 +- piker/clearing/_paper_engine.py | 2 +- piker/data/_source.py | 267 +--------------------------- piker/data/feed.py | 4 +- piker/data/flows.py | 4 +- piker/fsp/_engine.py | 2 +- piker/ui/_app.py | 2 +- piker/ui/_axes.py | 2 +- piker/ui/_chart.py | 2 +- piker/ui/_fsp.py | 2 +- piker/ui/order_mode.py | 2 +- tests/test_feeds.py | 2 +- 22 files changed, 335 insertions(+), 283 deletions(-) create mode 100644 piker/accounting/_mktinfo.py diff --git a/piker/accounting/_ledger.py b/piker/accounting/_ledger.py index 74bee9ad..df7bb4aa 100644 --- a/piker/accounting/_ledger.py +++ b/piker/accounting/_ledger.py @@ -33,9 +33,9 @@ import tomli import toml from .. import config -from ..data._source import Symbol from ..data.types import Struct from ..log import get_logger +from ._mktinfo import Symbol log = get_logger(__name__) diff --git a/piker/accounting/_mktinfo.py b/piker/accounting/_mktinfo.py new file mode 100644 index 00000000..a9036170 --- /dev/null +++ b/piker/accounting/_mktinfo.py @@ -0,0 +1,302 @@ +# piker: trading gear for hackers +# Copyright (C) Tyler Goodlet (in stewardship for pikers) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License + +# along with this program. If not, see . + +''' +Market (pair) meta-info layer: sane addressing semantics and meta-data +for cross-provider marketplaces. + +We intoduce the concept of, + +- a FQMA: fully qualified market address, +- a sane schema for FQMAs including derivatives, +- a msg-serializeable description of markets for + easy sharing with other pikers B) + +''' +from __future__ import annotations +from decimal import ( + Decimal, + ROUND_HALF_EVEN, +) +from typing import ( + Any, +) + +from ..data.types import Struct + + +class MktPair(Struct, frozen=True): + + src: str # source asset name being used to buy + src_type: str # source asset's financial type/classification name + # ^ specifies a "class" of financial instrument + # egs. stock, futer, option, bond etc. + + dst: str # destination asset name being bought + dst_type: str # destination asset's financial type/classification name + + price_tick: float # minimum price increment value increment + price_tick_digits: int # required decimal digits for above + + size_tick: float # minimum size (aka vlm) increment value increment + + # size_tick_digits: int # required decimal digits for above + @property + def size_tick_digits(self) -> int: + return self.size_tick + + venue: str | None = None # market venue provider name + expiry: str | None = None # for derivs, expiry datetime parseable str + + # for derivs, info describing contract, egs. + # strike price, call or put, swap type, exercise model, etc. + contract_info: str | None = None + + @classmethod + def from_msg( + self, + msg: dict[str, Any], + + ) -> MktPair: + ''' + Constructor for a received msg-dict normally received over IPC. + + ''' + ... + + # fqa, fqma, .. etc. see issue: + # https://github.com/pikers/piker/issues/467 + @property + def fqsn(self) -> str: + ''' + Return the fully qualified market (endpoint) name for the + pair of transacting assets. + + ''' + ... + + +def mk_fqsn( + provider: str, + symbol: str, + +) -> str: + ''' + Generate a "fully qualified symbol name" which is + a reverse-hierarchical cross broker/provider symbol + + ''' + return '.'.join([symbol, provider]).lower() + + +def float_digits( + value: float, +) -> int: + ''' + Return the number of precision digits read from a float value. + + ''' + if value == 0: + return 0 + + return int(-Decimal(str(value)).as_tuple().exponent) + + +def digits_to_dec( + ndigits: int, +) -> Decimal: + ''' + Return the minimum float value for an input integer value. + + eg. 3 -> 0.001 + + ''' + if ndigits == 0: + return Decimal('0') + + return Decimal('0.' + '0'*(ndigits-1) + '1') + + +def unpack_fqsn(fqsn: str) -> tuple[str, str, str]: + ''' + Unpack a fully-qualified-symbol-name to ``tuple``. + + ''' + venue = '' + suffix = '' + + # TODO: probably reverse the order of all this XD + tokens = fqsn.split('.') + if len(tokens) < 3: + # probably crypto + symbol, broker = tokens + return ( + broker, + symbol, + '', + ) + + elif len(tokens) > 3: + symbol, venue, suffix, broker = tokens + else: + symbol, venue, broker = tokens + suffix = '' + + # head, _, broker = fqsn.rpartition('.') + # symbol, _, suffix = head.rpartition('.') + return ( + broker, + '.'.join([symbol, venue]), + suffix, + ) + +# TODO: rework the below `Symbol` (which was originally inspired and +# derived from stuff in quantdom) into a simpler, ipc msg ready, market +# endpoint meta-data container type as per the drafted interace above. +class Symbol(Struct): + ''' + I guess this is some kinda container thing for dealing with + all the different meta-data formats from brokers? + + ''' + key: str + tick_size: float = 0.01 + lot_tick_size: float = 0.0 # "volume" precision as min step value + tick_size_digits: int = 2 + lot_size_digits: int = 0 + suffix: str = '' + broker_info: dict[str, dict[str, Any]] = {} + + @classmethod + def from_broker_info( + cls, + broker: str, + symbol: str, + info: dict[str, Any], + suffix: str = '', + + ) -> Symbol: + + tick_size = info.get('price_tick_size', 0.01) + lot_size = info.get('lot_tick_size', 0.0) + + return Symbol( + key=symbol, + + tick_size=tick_size, + lot_tick_size=lot_size, + + tick_size_digits=float_digits(tick_size), + lot_size_digits=float_digits(lot_size), + + suffix=suffix, + broker_info={broker: info}, + ) + + @classmethod + def from_fqsn( + cls, + fqsn: str, + info: dict[str, Any], + + ) -> Symbol: + broker, key, suffix = unpack_fqsn(fqsn) + return cls.from_broker_info( + broker, + key, + info=info, + suffix=suffix, + ) + + @property + def type_key(self) -> str: + return list(self.broker_info.values())[0]['asset_type'] + + @property + def brokers(self) -> list[str]: + return list(self.broker_info.keys()) + + def nearest_tick(self, value: float) -> float: + ''' + Return the nearest tick value based on mininum increment. + + ''' + mult = 1 / self.tick_size + return round(value * mult) / mult + + def front_feed(self) -> tuple[str, str]: + ''' + Return the "current" feed key for this symbol. + + (i.e. the broker + symbol key in a tuple). + + ''' + return ( + list(self.broker_info.keys())[0], + self.key, + ) + + def tokens(self) -> tuple[str]: + broker, key = self.front_feed() + if self.suffix: + return (key, self.suffix, broker) + else: + return (key, broker) + + @property + def fqsn(self) -> str: + return '.'.join(self.tokens()).lower() + + def front_fqsn(self) -> str: + ''' + fqsn = "fully qualified symbol name" + + Basically the idea here is for all client-ish code (aka programs/actors + that ask the provider agnostic layers in the stack for data) should be + able to tell which backend / venue / derivative each data feed/flow is + from by an explicit string key of the current form: + + ... + + TODO: I have thoughts that we should actually change this to be + more like an "attr lookup" (like how the web should have done + urls, but marketting peeps ruined it etc. etc.): + + ... + + ''' + tokens = self.tokens() + fqsn = '.'.join(map(str.lower, tokens)) + return fqsn + + def quantize_size( + self, + size: float, + + ) -> Decimal: + ''' + Truncate input ``size: float`` using ``Decimal`` + and ``.lot_size_digits``. + + ''' + digits = self.lot_size_digits + return Decimal(size).quantize( + Decimal(f'1.{"0".ljust(digits, "0")}'), + rounding=ROUND_HALF_EVEN + ) + + diff --git a/piker/accounting/_pos.py b/piker/accounting/_pos.py index 2a9ca0d8..204e7a8e 100644 --- a/piker/accounting/_pos.py +++ b/piker/accounting/_pos.py @@ -43,10 +43,13 @@ from ._ledger import ( iter_by_dt, open_trade_ledger, ) +from ._mktinfo import ( + Symbol, + unpack_fqsn, +) from .. import config from ..brokers import get_brokermod from ..clearing._messages import BrokerdPosition, Status -from ..data._source import Symbol, unpack_fqsn from ..data.types import Struct from ..log import get_logger @@ -154,6 +157,7 @@ class Position(Struct): inline_table['tid'] = tid toml_clears_list.append(inline_table) + d['clears'] = toml_clears_list return fqsn, d diff --git a/piker/brokers/ib/api.py b/piker/brokers/ib/api.py index bfa66a9d..c6513204 100644 --- a/piker/brokers/ib/api.py +++ b/piker/brokers/ib/api.py @@ -644,7 +644,7 @@ class Client: # fqsn parsing stage # ------------------ if '.ib' in pattern: - from ..data._source import unpack_fqsn + from ..accounting._mktinfo import unpack_fqsn _, symbol, expiry = unpack_fqsn(pattern) else: diff --git a/piker/brokers/ib/broker.py b/piker/brokers/ib/broker.py index bc65d6a2..77f0bb53 100644 --- a/piker/brokers/ib/broker.py +++ b/piker/brokers/ib/broker.py @@ -70,7 +70,7 @@ from piker.clearing._messages import ( BrokerdFill, BrokerdError, ) -from piker.data._source import ( +from piker.accounting._mktinfo import ( Symbol, float_digits, ) diff --git a/piker/brokers/kraken/api.py b/piker/brokers/kraken/api.py index 74ad734b..82479329 100644 --- a/piker/brokers/kraken/api.py +++ b/piker/brokers/kraken/api.py @@ -42,7 +42,7 @@ import trio from piker import config from piker.data.types import Struct -from piker.data._source import Symbol +from piker.accounting._mktinfo import Symbol from piker.brokers._util import ( resproc, SymbolNotFound, diff --git a/piker/brokers/kraken/broker.py b/piker/brokers/kraken/broker.py index 5d1bbb01..72d6f0fe 100644 --- a/piker/brokers/kraken/broker.py +++ b/piker/brokers/kraken/broker.py @@ -48,7 +48,7 @@ from piker.accounting import ( open_pps, get_likely_pair, ) -from piker.data._source import ( +from piker.accounting._mktinfo import ( Symbol, digits_to_dec, ) diff --git a/piker/clearing/_allocate.py b/piker/clearing/_allocate.py index c457de05..023d1e92 100644 --- a/piker/clearing/_allocate.py +++ b/piker/clearing/_allocate.py @@ -23,7 +23,7 @@ from typing import Optional from bidict import bidict -from ..data._source import Symbol +from ..accounting._mktinfo import Symbol from ..data.types import Struct from ..accounting import Position diff --git a/piker/clearing/_client.py b/piker/clearing/_client.py index 7d03406a..ee176f87 100644 --- a/piker/clearing/_client.py +++ b/piker/clearing/_client.py @@ -27,6 +27,7 @@ import trio import tractor from tractor.trionics import broadcast_receiver +from ..accounting._mktinfo import unpack_fqsn from ..log import get_logger from ..data.types import Struct from ..service import maybe_open_emsd @@ -228,7 +229,6 @@ async def open_ems( # ready for order commands book = get_orders() - from ..data._source import unpack_fqsn broker, symbol, suffix = unpack_fqsn(fqsn) async with maybe_open_emsd(broker) as portal: diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 477da310..b2c4c614 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -43,7 +43,7 @@ import tractor from ..log import get_logger from ..data._normalize import iterticks -from ..data._source import ( +from ..accounting._mktinfo import ( unpack_fqsn, mk_fqsn, float_digits, @@ -521,7 +521,6 @@ class Router(Struct): none already exists. ''' - from ..data._source import unpack_fqsn broker, symbol, suffix = unpack_fqsn(fqsn) async with ( diff --git a/piker/clearing/_messages.py b/piker/clearing/_messages.py index c7693b9f..f084af05 100644 --- a/piker/clearing/_messages.py +++ b/piker/clearing/_messages.py @@ -29,7 +29,7 @@ from typing import ( from msgspec import field -from ..data._source import Symbol +from ..accounting._mktinfo import Symbol from ..data.types import Struct diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 39d5a474..00611e6d 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -38,7 +38,7 @@ import tractor from .. import data from ..data.types import Struct -from ..data._source import Symbol +from ..accounting._mktinfo import Symbol from ..accounting import ( Position, Transaction, diff --git a/piker/data/_source.py b/piker/data/_source.py index e503105e..61c2e52f 100644 --- a/piker/data/_source.py +++ b/piker/data/_source.py @@ -28,8 +28,12 @@ from bidict import bidict import numpy as np from .types import Struct -# from numba import from_dtype - +from ..accounting._mktinfo import ( + # mkfqsn, + unpack_fqsn, + # digits_to_dec, + float_digits, +) ohlc_fields = [ ('time', float), @@ -50,6 +54,7 @@ base_ohlc_dtype = np.dtype(ohlc_fields) # TODO: for now need to construct this manually for readonly arrays, see # https://github.com/numba/numba/issues/4511 +# from numba import from_dtype # numba_ohlc_dtype = from_dtype(base_ohlc_dtype) # map time frame "keys" to seconds values @@ -64,47 +69,6 @@ tf_in_1s = bidict({ }) -def mk_fqsn( - provider: str, - symbol: str, - -) -> str: - ''' - Generate a "fully qualified symbol name" which is - a reverse-hierarchical cross broker/provider symbol - - ''' - return '.'.join([symbol, provider]).lower() - - -def float_digits( - value: float, -) -> int: - ''' - Return the number of precision digits read from a float value. - - ''' - if value == 0: - return 0 - - return int(-Decimal(str(value)).as_tuple().exponent) - - -def digits_to_dec( - ndigits: int, -) -> Decimal: - ''' - Return the minimum float value for an input integer value. - - eg. 3 -> 0.001 - - ''' - if ndigits == 0: - return Decimal('0') - - return Decimal('0.' + '0'*(ndigits-1) + '1') - - def ohlc_zeros(length: int) -> np.ndarray: """Construct an OHLC field formatted structarray. @@ -115,223 +79,6 @@ def ohlc_zeros(length: int) -> np.ndarray: return np.zeros(length, dtype=base_ohlc_dtype) -def unpack_fqsn(fqsn: str) -> tuple[str, str, str]: - ''' - Unpack a fully-qualified-symbol-name to ``tuple``. - - ''' - venue = '' - suffix = '' - - # TODO: probably reverse the order of all this XD - tokens = fqsn.split('.') - if len(tokens) < 3: - # probably crypto - symbol, broker = tokens - return ( - broker, - symbol, - '', - ) - - elif len(tokens) > 3: - symbol, venue, suffix, broker = tokens - else: - symbol, venue, broker = tokens - suffix = '' - - # head, _, broker = fqsn.rpartition('.') - # symbol, _, suffix = head.rpartition('.') - return ( - broker, - '.'.join([symbol, venue]), - suffix, - ) - - -class MktPair(Struct, frozen=True): - - src: str # source asset name being used to buy - src_type: str # source asset's financial type/classification name - # ^ specifies a "class" of financial instrument - # egs. stock, futer, option, bond etc. - - dst: str # destination asset name being bought - dst_type: str # destination asset's financial type/classification name - - price_tick: float # minimum price increment value increment - price_tick_digits: int # required decimal digits for above - - size_tick: float # minimum size (aka vlm) increment value increment - size_tick_digits: int # required decimal digits for above - - venue: str | None = None # market venue provider name - expiry: str | None = None # for derivs, expiry datetime parseable str - - # for derivs, info describing contract, egs. - # strike price, call or put, swap type, exercise model, etc. - contract_info: str | None = None - - @classmethod - def from_msg( - self, - msg: dict[str, Any], - - ) -> MktPair: - ''' - Constructor for a received msg-dict normally received over IPC. - - ''' - ... - - # fqa, fqma, .. etc. see issue: - # https://github.com/pikers/piker/issues/467 - @property - def fqsn(self) -> str: - ''' - Return the fully qualified market (endpoint) name for the - pair of transacting assets. - - ''' - ... - - -# TODO: rework the below `Symbol` (which was originally inspired and -# derived from stuff in quantdom) into a simpler, ipc msg ready, market -# endpoint meta-data container type as per the drafted interace above. -class Symbol(Struct): - ''' - I guess this is some kinda container thing for dealing with - all the different meta-data formats from brokers? - - ''' - key: str - tick_size: float = 0.01 - lot_tick_size: float = 0.0 # "volume" precision as min step value - tick_size_digits: int = 2 - lot_size_digits: int = 0 - suffix: str = '' - broker_info: dict[str, dict[str, Any]] = {} - - @classmethod - def from_broker_info( - cls, - broker: str, - symbol: str, - info: dict[str, Any], - suffix: str = '', - - ) -> Symbol: - - tick_size = info.get('price_tick_size', 0.01) - lot_size = info.get('lot_tick_size', 0.0) - - return Symbol( - key=symbol, - - tick_size=tick_size, - lot_tick_size=lot_size, - - tick_size_digits=float_digits(tick_size), - lot_size_digits=float_digits(lot_size), - - suffix=suffix, - broker_info={broker: info}, - ) - - @classmethod - def from_fqsn( - cls, - fqsn: str, - info: dict[str, Any], - - ) -> Symbol: - broker, key, suffix = unpack_fqsn(fqsn) - return cls.from_broker_info( - broker, - key, - info=info, - suffix=suffix, - ) - - @property - def type_key(self) -> str: - return list(self.broker_info.values())[0]['asset_type'] - - @property - def brokers(self) -> list[str]: - return list(self.broker_info.keys()) - - def nearest_tick(self, value: float) -> float: - ''' - Return the nearest tick value based on mininum increment. - - ''' - mult = 1 / self.tick_size - return round(value * mult) / mult - - def front_feed(self) -> tuple[str, str]: - ''' - Return the "current" feed key for this symbol. - - (i.e. the broker + symbol key in a tuple). - - ''' - return ( - list(self.broker_info.keys())[0], - self.key, - ) - - def tokens(self) -> tuple[str]: - broker, key = self.front_feed() - if self.suffix: - return (key, self.suffix, broker) - else: - return (key, broker) - - @property - def fqsn(self) -> str: - return '.'.join(self.tokens()).lower() - - def front_fqsn(self) -> str: - ''' - fqsn = "fully qualified symbol name" - - Basically the idea here is for all client-ish code (aka programs/actors - that ask the provider agnostic layers in the stack for data) should be - able to tell which backend / venue / derivative each data feed/flow is - from by an explicit string key of the current form: - - ... - - TODO: I have thoughts that we should actually change this to be - more like an "attr lookup" (like how the web should have done - urls, but marketting peeps ruined it etc. etc.): - - ... - - ''' - tokens = self.tokens() - fqsn = '.'.join(map(str.lower, tokens)) - return fqsn - - def quantize_size( - self, - size: float, - - ) -> Decimal: - ''' - Truncate input ``size: float`` using ``Decimal`` - and ``.lot_size_digits``. - - ''' - digits = self.lot_size_digits - return Decimal(size).quantize( - Decimal(f'1.{"0".ljust(digits, "0")}'), - rounding=ROUND_HALF_EVEN - ) - - def _nan_to_closest_num(array: np.ndarray): """Return interpolated values instead of NaN. diff --git a/piker/data/feed.py b/piker/data/feed.py index 7efd5eb3..5e1a1aec 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -70,11 +70,11 @@ from ._sharedmem import ( ) from .ingest import get_ingestormod from .types import Struct -from ._source import ( - base_iohlc_dtype, +from ..accounting._mktinfo import ( Symbol, unpack_fqsn, ) +from ._source import base_iohlc_dtype from ..ui import _search from ._sampling import ( open_sample_stream, diff --git a/piker/data/flows.py b/piker/data/flows.py index 9d8b3103..19615f61 100644 --- a/piker/data/flows.py +++ b/piker/data/flows.py @@ -30,10 +30,10 @@ import tractor import pendulum import numpy as np -from .types import Struct -from ._source import ( +from ..accounting._mktinfo import ( Symbol, ) +from .types import Struct from ._sharedmem import ( attach_shm_array, ShmArray, diff --git a/piker/fsp/_engine.py b/piker/fsp/_engine.py index 37852cfc..a77e662f 100644 --- a/piker/fsp/_engine.py +++ b/piker/fsp/_engine.py @@ -45,7 +45,7 @@ from ..data._sampling import ( _default_delay_s, open_sample_stream, ) -from ..data._source import Symbol +from ..accounting._mktinfo import Symbol from ._api import ( Fsp, _load_builtins, diff --git a/piker/ui/_app.py b/piker/ui/_app.py index 9978dbe3..0e7dad47 100644 --- a/piker/ui/_app.py +++ b/piker/ui/_app.py @@ -28,7 +28,7 @@ from ..service import maybe_spawn_brokerd from . import _event from ._exec import run_qtractor from ..data.feed import install_brokerd_search -from ..data._source import unpack_fqsn +from ..accounting._mktinfo import unpack_fqsn from . import _search from ._chart import GodWidget from ..log import get_logger diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 62214f60..040d0552 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -29,7 +29,7 @@ from PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtCore import QPointF from . import _pg_overrides as pgo -from ..data._source import float_digits +from ..accounting._mktinfo import float_digits from ._label import Label from ._style import DpiAwareFont, hcolor, _font from ._interaction import ChartView diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 7811278b..b05d6fcf 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -68,7 +68,7 @@ from ..data.feed import ( Feed, Flume, ) -from ..data._source import Symbol +from ..accounting._mktinfo import Symbol from ..log import get_logger from ._interaction import ChartView from ._forms import FieldsForm diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 6e600743..960b287a 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -46,7 +46,7 @@ from ..data._sharedmem import ( try_read, ) from ..data.feed import Flume -from ..data._source import Symbol +from ..accounting._mktinfo import Symbol from ._chart import ( ChartPlotWidget, LinkedSplits, diff --git a/piker/ui/order_mode.py b/piker/ui/order_mode.py index bf60c0e6..6ac0f1f4 100644 --- a/piker/ui/order_mode.py +++ b/piker/ui/order_mode.py @@ -42,7 +42,7 @@ from ..clearing._allocate import ( mk_allocator, ) from ._style import _font -from ..data._source import Symbol +from ..accounting._mktinfo import Symbol from ..data.feed import ( Feed, Flume, diff --git a/tests/test_feeds.py b/tests/test_feeds.py index a79ca861..0435ed61 100644 --- a/tests/test_feeds.py +++ b/tests/test_feeds.py @@ -13,7 +13,7 @@ from piker.data import ( ShmArray, open_feed, ) -from piker.data._source import ( +from piker.accounting._mktinfo import ( unpack_fqsn, )