Move all fqsn parsing and `Symbol` to new `accounting._mktinfo

pre_overruns_ctxcancelled
Tyler Goodlet 2023-03-13 17:42:20 -04:00
parent 33ee647224
commit c0a3c6dff7
22 changed files with 335 additions and 283 deletions

View File

@ -33,9 +33,9 @@ import tomli
import toml import toml
from .. import config from .. import config
from ..data._source import Symbol
from ..data.types import Struct from ..data.types import Struct
from ..log import get_logger from ..log import get_logger
from ._mktinfo import Symbol
log = get_logger(__name__) log = get_logger(__name__)

View File

@ -0,0 +1,302 @@
# piker: trading gear for hackers
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
Market (pair) meta-info layer: sane addressing semantics and meta-data
for cross-provider marketplaces.
We intoduce the concept of,
- a FQMA: fully qualified market address,
- a sane schema for FQMAs including derivatives,
- a msg-serializeable description of markets for
easy sharing with other pikers B)
'''
from __future__ import annotations
from decimal import (
Decimal,
ROUND_HALF_EVEN,
)
from typing import (
Any,
)
from ..data.types import Struct
class MktPair(Struct, frozen=True):
src: str # source asset name being used to buy
src_type: str # source asset's financial type/classification name
# ^ specifies a "class" of financial instrument
# egs. stock, futer, option, bond etc.
dst: str # destination asset name being bought
dst_type: str # destination asset's financial type/classification name
price_tick: float # minimum price increment value increment
price_tick_digits: int # required decimal digits for above
size_tick: float # minimum size (aka vlm) increment value increment
# size_tick_digits: int # required decimal digits for above
@property
def size_tick_digits(self) -> int:
return self.size_tick
venue: str | None = None # market venue provider name
expiry: str | None = None # for derivs, expiry datetime parseable str
# for derivs, info describing contract, egs.
# strike price, call or put, swap type, exercise model, etc.
contract_info: str | None = None
@classmethod
def from_msg(
self,
msg: dict[str, Any],
) -> MktPair:
'''
Constructor for a received msg-dict normally received over IPC.
'''
...
# fqa, fqma, .. etc. see issue:
# https://github.com/pikers/piker/issues/467
@property
def fqsn(self) -> str:
'''
Return the fully qualified market (endpoint) name for the
pair of transacting assets.
'''
...
def mk_fqsn(
provider: str,
symbol: str,
) -> str:
'''
Generate a "fully qualified symbol name" which is
a reverse-hierarchical cross broker/provider symbol
'''
return '.'.join([symbol, provider]).lower()
def float_digits(
value: float,
) -> int:
'''
Return the number of precision digits read from a float value.
'''
if value == 0:
return 0
return int(-Decimal(str(value)).as_tuple().exponent)
def digits_to_dec(
ndigits: int,
) -> Decimal:
'''
Return the minimum float value for an input integer value.
eg. 3 -> 0.001
'''
if ndigits == 0:
return Decimal('0')
return Decimal('0.' + '0'*(ndigits-1) + '1')
def unpack_fqsn(fqsn: str) -> tuple[str, str, str]:
'''
Unpack a fully-qualified-symbol-name to ``tuple``.
'''
venue = ''
suffix = ''
# TODO: probably reverse the order of all this XD
tokens = fqsn.split('.')
if len(tokens) < 3:
# probably crypto
symbol, broker = tokens
return (
broker,
symbol,
'',
)
elif len(tokens) > 3:
symbol, venue, suffix, broker = tokens
else:
symbol, venue, broker = tokens
suffix = ''
# head, _, broker = fqsn.rpartition('.')
# symbol, _, suffix = head.rpartition('.')
return (
broker,
'.'.join([symbol, venue]),
suffix,
)
# TODO: rework the below `Symbol` (which was originally inspired and
# derived from stuff in quantdom) into a simpler, ipc msg ready, market
# endpoint meta-data container type as per the drafted interace above.
class Symbol(Struct):
'''
I guess this is some kinda container thing for dealing with
all the different meta-data formats from brokers?
'''
key: str
tick_size: float = 0.01
lot_tick_size: float = 0.0 # "volume" precision as min step value
tick_size_digits: int = 2
lot_size_digits: int = 0
suffix: str = ''
broker_info: dict[str, dict[str, Any]] = {}
@classmethod
def from_broker_info(
cls,
broker: str,
symbol: str,
info: dict[str, Any],
suffix: str = '',
) -> Symbol:
tick_size = info.get('price_tick_size', 0.01)
lot_size = info.get('lot_tick_size', 0.0)
return Symbol(
key=symbol,
tick_size=tick_size,
lot_tick_size=lot_size,
tick_size_digits=float_digits(tick_size),
lot_size_digits=float_digits(lot_size),
suffix=suffix,
broker_info={broker: info},
)
@classmethod
def from_fqsn(
cls,
fqsn: str,
info: dict[str, Any],
) -> Symbol:
broker, key, suffix = unpack_fqsn(fqsn)
return cls.from_broker_info(
broker,
key,
info=info,
suffix=suffix,
)
@property
def type_key(self) -> str:
return list(self.broker_info.values())[0]['asset_type']
@property
def brokers(self) -> list[str]:
return list(self.broker_info.keys())
def nearest_tick(self, value: float) -> float:
'''
Return the nearest tick value based on mininum increment.
'''
mult = 1 / self.tick_size
return round(value * mult) / mult
def front_feed(self) -> tuple[str, str]:
'''
Return the "current" feed key for this symbol.
(i.e. the broker + symbol key in a tuple).
'''
return (
list(self.broker_info.keys())[0],
self.key,
)
def tokens(self) -> tuple[str]:
broker, key = self.front_feed()
if self.suffix:
return (key, self.suffix, broker)
else:
return (key, broker)
@property
def fqsn(self) -> str:
return '.'.join(self.tokens()).lower()
def front_fqsn(self) -> str:
'''
fqsn = "fully qualified symbol name"
Basically the idea here is for all client-ish code (aka programs/actors
that ask the provider agnostic layers in the stack for data) should be
able to tell which backend / venue / derivative each data feed/flow is
from by an explicit string key of the current form:
<instrumentname>.<venue>.<suffixwithmetadata>.<brokerbackendname>
TODO: I have thoughts that we should actually change this to be
more like an "attr lookup" (like how the web should have done
urls, but marketting peeps ruined it etc. etc.):
<broker>.<venue>.<instrumentname>.<suffixwithmetadata>
'''
tokens = self.tokens()
fqsn = '.'.join(map(str.lower, tokens))
return fqsn
def quantize_size(
self,
size: float,
) -> Decimal:
'''
Truncate input ``size: float`` using ``Decimal``
and ``.lot_size_digits``.
'''
digits = self.lot_size_digits
return Decimal(size).quantize(
Decimal(f'1.{"0".ljust(digits, "0")}'),
rounding=ROUND_HALF_EVEN
)

View File

@ -43,10 +43,13 @@ from ._ledger import (
iter_by_dt, iter_by_dt,
open_trade_ledger, open_trade_ledger,
) )
from ._mktinfo import (
Symbol,
unpack_fqsn,
)
from .. import config from .. import config
from ..brokers import get_brokermod from ..brokers import get_brokermod
from ..clearing._messages import BrokerdPosition, Status from ..clearing._messages import BrokerdPosition, Status
from ..data._source import Symbol, unpack_fqsn
from ..data.types import Struct from ..data.types import Struct
from ..log import get_logger from ..log import get_logger
@ -154,6 +157,7 @@ class Position(Struct):
inline_table['tid'] = tid inline_table['tid'] = tid
toml_clears_list.append(inline_table) toml_clears_list.append(inline_table)
d['clears'] = toml_clears_list d['clears'] = toml_clears_list
return fqsn, d return fqsn, d

View File

@ -644,7 +644,7 @@ class Client:
# fqsn parsing stage # fqsn parsing stage
# ------------------ # ------------------
if '.ib' in pattern: if '.ib' in pattern:
from ..data._source import unpack_fqsn from ..accounting._mktinfo import unpack_fqsn
_, symbol, expiry = unpack_fqsn(pattern) _, symbol, expiry = unpack_fqsn(pattern)
else: else:

View File

@ -70,7 +70,7 @@ from piker.clearing._messages import (
BrokerdFill, BrokerdFill,
BrokerdError, BrokerdError,
) )
from piker.data._source import ( from piker.accounting._mktinfo import (
Symbol, Symbol,
float_digits, float_digits,
) )

View File

@ -42,7 +42,7 @@ import trio
from piker import config from piker import config
from piker.data.types import Struct from piker.data.types import Struct
from piker.data._source import Symbol from piker.accounting._mktinfo import Symbol
from piker.brokers._util import ( from piker.brokers._util import (
resproc, resproc,
SymbolNotFound, SymbolNotFound,

View File

@ -48,7 +48,7 @@ from piker.accounting import (
open_pps, open_pps,
get_likely_pair, get_likely_pair,
) )
from piker.data._source import ( from piker.accounting._mktinfo import (
Symbol, Symbol,
digits_to_dec, digits_to_dec,
) )

View File

@ -23,7 +23,7 @@ from typing import Optional
from bidict import bidict from bidict import bidict
from ..data._source import Symbol from ..accounting._mktinfo import Symbol
from ..data.types import Struct from ..data.types import Struct
from ..accounting import Position from ..accounting import Position

View File

@ -27,6 +27,7 @@ import trio
import tractor import tractor
from tractor.trionics import broadcast_receiver from tractor.trionics import broadcast_receiver
from ..accounting._mktinfo import unpack_fqsn
from ..log import get_logger from ..log import get_logger
from ..data.types import Struct from ..data.types import Struct
from ..service import maybe_open_emsd from ..service import maybe_open_emsd
@ -228,7 +229,6 @@ async def open_ems(
# ready for order commands # ready for order commands
book = get_orders() book = get_orders()
from ..data._source import unpack_fqsn
broker, symbol, suffix = unpack_fqsn(fqsn) broker, symbol, suffix = unpack_fqsn(fqsn)
async with maybe_open_emsd(broker) as portal: async with maybe_open_emsd(broker) as portal:

View File

@ -43,7 +43,7 @@ import tractor
from ..log import get_logger from ..log import get_logger
from ..data._normalize import iterticks from ..data._normalize import iterticks
from ..data._source import ( from ..accounting._mktinfo import (
unpack_fqsn, unpack_fqsn,
mk_fqsn, mk_fqsn,
float_digits, float_digits,
@ -521,7 +521,6 @@ class Router(Struct):
none already exists. none already exists.
''' '''
from ..data._source import unpack_fqsn
broker, symbol, suffix = unpack_fqsn(fqsn) broker, symbol, suffix = unpack_fqsn(fqsn)
async with ( async with (

View File

@ -29,7 +29,7 @@ from typing import (
from msgspec import field from msgspec import field
from ..data._source import Symbol from ..accounting._mktinfo import Symbol
from ..data.types import Struct from ..data.types import Struct

View File

@ -38,7 +38,7 @@ import tractor
from .. import data from .. import data
from ..data.types import Struct from ..data.types import Struct
from ..data._source import Symbol from ..accounting._mktinfo import Symbol
from ..accounting import ( from ..accounting import (
Position, Position,
Transaction, Transaction,

View File

@ -28,8 +28,12 @@ from bidict import bidict
import numpy as np import numpy as np
from .types import Struct from .types import Struct
# from numba import from_dtype from ..accounting._mktinfo import (
# mkfqsn,
unpack_fqsn,
# digits_to_dec,
float_digits,
)
ohlc_fields = [ ohlc_fields = [
('time', float), ('time', float),
@ -50,6 +54,7 @@ base_ohlc_dtype = np.dtype(ohlc_fields)
# TODO: for now need to construct this manually for readonly arrays, see # TODO: for now need to construct this manually for readonly arrays, see
# https://github.com/numba/numba/issues/4511 # https://github.com/numba/numba/issues/4511
# from numba import from_dtype
# numba_ohlc_dtype = from_dtype(base_ohlc_dtype) # numba_ohlc_dtype = from_dtype(base_ohlc_dtype)
# map time frame "keys" to seconds values # map time frame "keys" to seconds values
@ -64,47 +69,6 @@ tf_in_1s = bidict({
}) })
def mk_fqsn(
provider: str,
symbol: str,
) -> str:
'''
Generate a "fully qualified symbol name" which is
a reverse-hierarchical cross broker/provider symbol
'''
return '.'.join([symbol, provider]).lower()
def float_digits(
value: float,
) -> int:
'''
Return the number of precision digits read from a float value.
'''
if value == 0:
return 0
return int(-Decimal(str(value)).as_tuple().exponent)
def digits_to_dec(
ndigits: int,
) -> Decimal:
'''
Return the minimum float value for an input integer value.
eg. 3 -> 0.001
'''
if ndigits == 0:
return Decimal('0')
return Decimal('0.' + '0'*(ndigits-1) + '1')
def ohlc_zeros(length: int) -> np.ndarray: def ohlc_zeros(length: int) -> np.ndarray:
"""Construct an OHLC field formatted structarray. """Construct an OHLC field formatted structarray.
@ -115,223 +79,6 @@ def ohlc_zeros(length: int) -> np.ndarray:
return np.zeros(length, dtype=base_ohlc_dtype) return np.zeros(length, dtype=base_ohlc_dtype)
def unpack_fqsn(fqsn: str) -> tuple[str, str, str]:
'''
Unpack a fully-qualified-symbol-name to ``tuple``.
'''
venue = ''
suffix = ''
# TODO: probably reverse the order of all this XD
tokens = fqsn.split('.')
if len(tokens) < 3:
# probably crypto
symbol, broker = tokens
return (
broker,
symbol,
'',
)
elif len(tokens) > 3:
symbol, venue, suffix, broker = tokens
else:
symbol, venue, broker = tokens
suffix = ''
# head, _, broker = fqsn.rpartition('.')
# symbol, _, suffix = head.rpartition('.')
return (
broker,
'.'.join([symbol, venue]),
suffix,
)
class MktPair(Struct, frozen=True):
src: str # source asset name being used to buy
src_type: str # source asset's financial type/classification name
# ^ specifies a "class" of financial instrument
# egs. stock, futer, option, bond etc.
dst: str # destination asset name being bought
dst_type: str # destination asset's financial type/classification name
price_tick: float # minimum price increment value increment
price_tick_digits: int # required decimal digits for above
size_tick: float # minimum size (aka vlm) increment value increment
size_tick_digits: int # required decimal digits for above
venue: str | None = None # market venue provider name
expiry: str | None = None # for derivs, expiry datetime parseable str
# for derivs, info describing contract, egs.
# strike price, call or put, swap type, exercise model, etc.
contract_info: str | None = None
@classmethod
def from_msg(
self,
msg: dict[str, Any],
) -> MktPair:
'''
Constructor for a received msg-dict normally received over IPC.
'''
...
# fqa, fqma, .. etc. see issue:
# https://github.com/pikers/piker/issues/467
@property
def fqsn(self) -> str:
'''
Return the fully qualified market (endpoint) name for the
pair of transacting assets.
'''
...
# TODO: rework the below `Symbol` (which was originally inspired and
# derived from stuff in quantdom) into a simpler, ipc msg ready, market
# endpoint meta-data container type as per the drafted interace above.
class Symbol(Struct):
'''
I guess this is some kinda container thing for dealing with
all the different meta-data formats from brokers?
'''
key: str
tick_size: float = 0.01
lot_tick_size: float = 0.0 # "volume" precision as min step value
tick_size_digits: int = 2
lot_size_digits: int = 0
suffix: str = ''
broker_info: dict[str, dict[str, Any]] = {}
@classmethod
def from_broker_info(
cls,
broker: str,
symbol: str,
info: dict[str, Any],
suffix: str = '',
) -> Symbol:
tick_size = info.get('price_tick_size', 0.01)
lot_size = info.get('lot_tick_size', 0.0)
return Symbol(
key=symbol,
tick_size=tick_size,
lot_tick_size=lot_size,
tick_size_digits=float_digits(tick_size),
lot_size_digits=float_digits(lot_size),
suffix=suffix,
broker_info={broker: info},
)
@classmethod
def from_fqsn(
cls,
fqsn: str,
info: dict[str, Any],
) -> Symbol:
broker, key, suffix = unpack_fqsn(fqsn)
return cls.from_broker_info(
broker,
key,
info=info,
suffix=suffix,
)
@property
def type_key(self) -> str:
return list(self.broker_info.values())[0]['asset_type']
@property
def brokers(self) -> list[str]:
return list(self.broker_info.keys())
def nearest_tick(self, value: float) -> float:
'''
Return the nearest tick value based on mininum increment.
'''
mult = 1 / self.tick_size
return round(value * mult) / mult
def front_feed(self) -> tuple[str, str]:
'''
Return the "current" feed key for this symbol.
(i.e. the broker + symbol key in a tuple).
'''
return (
list(self.broker_info.keys())[0],
self.key,
)
def tokens(self) -> tuple[str]:
broker, key = self.front_feed()
if self.suffix:
return (key, self.suffix, broker)
else:
return (key, broker)
@property
def fqsn(self) -> str:
return '.'.join(self.tokens()).lower()
def front_fqsn(self) -> str:
'''
fqsn = "fully qualified symbol name"
Basically the idea here is for all client-ish code (aka programs/actors
that ask the provider agnostic layers in the stack for data) should be
able to tell which backend / venue / derivative each data feed/flow is
from by an explicit string key of the current form:
<instrumentname>.<venue>.<suffixwithmetadata>.<brokerbackendname>
TODO: I have thoughts that we should actually change this to be
more like an "attr lookup" (like how the web should have done
urls, but marketting peeps ruined it etc. etc.):
<broker>.<venue>.<instrumentname>.<suffixwithmetadata>
'''
tokens = self.tokens()
fqsn = '.'.join(map(str.lower, tokens))
return fqsn
def quantize_size(
self,
size: float,
) -> Decimal:
'''
Truncate input ``size: float`` using ``Decimal``
and ``.lot_size_digits``.
'''
digits = self.lot_size_digits
return Decimal(size).quantize(
Decimal(f'1.{"0".ljust(digits, "0")}'),
rounding=ROUND_HALF_EVEN
)
def _nan_to_closest_num(array: np.ndarray): def _nan_to_closest_num(array: np.ndarray):
"""Return interpolated values instead of NaN. """Return interpolated values instead of NaN.

View File

@ -70,11 +70,11 @@ from ._sharedmem import (
) )
from .ingest import get_ingestormod from .ingest import get_ingestormod
from .types import Struct from .types import Struct
from ._source import ( from ..accounting._mktinfo import (
base_iohlc_dtype,
Symbol, Symbol,
unpack_fqsn, unpack_fqsn,
) )
from ._source import base_iohlc_dtype
from ..ui import _search from ..ui import _search
from ._sampling import ( from ._sampling import (
open_sample_stream, open_sample_stream,

View File

@ -30,10 +30,10 @@ import tractor
import pendulum import pendulum
import numpy as np import numpy as np
from .types import Struct from ..accounting._mktinfo import (
from ._source import (
Symbol, Symbol,
) )
from .types import Struct
from ._sharedmem import ( from ._sharedmem import (
attach_shm_array, attach_shm_array,
ShmArray, ShmArray,

View File

@ -45,7 +45,7 @@ from ..data._sampling import (
_default_delay_s, _default_delay_s,
open_sample_stream, open_sample_stream,
) )
from ..data._source import Symbol from ..accounting._mktinfo import Symbol
from ._api import ( from ._api import (
Fsp, Fsp,
_load_builtins, _load_builtins,

View File

@ -28,7 +28,7 @@ from ..service import maybe_spawn_brokerd
from . import _event from . import _event
from ._exec import run_qtractor from ._exec import run_qtractor
from ..data.feed import install_brokerd_search from ..data.feed import install_brokerd_search
from ..data._source import unpack_fqsn from ..accounting._mktinfo import unpack_fqsn
from . import _search from . import _search
from ._chart import GodWidget from ._chart import GodWidget
from ..log import get_logger from ..log import get_logger

View File

@ -29,7 +29,7 @@ from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtCore import QPointF from PyQt5.QtCore import QPointF
from . import _pg_overrides as pgo from . import _pg_overrides as pgo
from ..data._source import float_digits from ..accounting._mktinfo import float_digits
from ._label import Label from ._label import Label
from ._style import DpiAwareFont, hcolor, _font from ._style import DpiAwareFont, hcolor, _font
from ._interaction import ChartView from ._interaction import ChartView

View File

@ -68,7 +68,7 @@ from ..data.feed import (
Feed, Feed,
Flume, Flume,
) )
from ..data._source import Symbol from ..accounting._mktinfo import Symbol
from ..log import get_logger from ..log import get_logger
from ._interaction import ChartView from ._interaction import ChartView
from ._forms import FieldsForm from ._forms import FieldsForm

View File

@ -46,7 +46,7 @@ from ..data._sharedmem import (
try_read, try_read,
) )
from ..data.feed import Flume from ..data.feed import Flume
from ..data._source import Symbol from ..accounting._mktinfo import Symbol
from ._chart import ( from ._chart import (
ChartPlotWidget, ChartPlotWidget,
LinkedSplits, LinkedSplits,

View File

@ -42,7 +42,7 @@ from ..clearing._allocate import (
mk_allocator, mk_allocator,
) )
from ._style import _font from ._style import _font
from ..data._source import Symbol from ..accounting._mktinfo import Symbol
from ..data.feed import ( from ..data.feed import (
Feed, Feed,
Flume, Flume,

View File

@ -13,7 +13,7 @@ from piker.data import (
ShmArray, ShmArray,
open_feed, open_feed,
) )
from piker.data._source import ( from piker.accounting._mktinfo import (
unpack_fqsn, unpack_fqsn,
) )