Move all fqsn parsing and `Symbol` to new `accounting._mktinfo

piker/accounting/_ledger.py

@@ -33,9 +33,9 @@ import tomli
 import toml
 
 from .. import config
-from ..data._source import Symbol
 from ..data.types import Struct
 from ..log import get_logger
+from ._mktinfo import Symbol
 
 log = get_logger(__name__)
 

piker/accounting/_mktinfo.py

@@ -0,0 +1,302 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Market (pair) meta-info layer: sane addressing semantics and meta-data
+for cross-provider marketplaces.
+
+We intoduce the concept of,
+
+- a FQMA: fully qualified market address,
+- a sane schema for FQMAs including derivatives,
+- a msg-serializeable description of markets for
+  easy sharing with other pikers B)
+
+'''
+from __future__ import annotations
+from decimal import (
+    Decimal,
+    ROUND_HALF_EVEN,
+)
+from typing import (
+    Any,
+)
+
+from ..data.types import Struct
+
+
+class MktPair(Struct, frozen=True):
+
+    src: str  # source asset name being used to buy
+    src_type: str  # source asset's financial type/classification name
+    # ^ specifies a "class" of financial instrument
+    # egs. stock, futer, option, bond etc.
+
+    dst: str  # destination asset name being bought
+    dst_type: str  # destination asset's financial type/classification name
+
+    price_tick: float  # minimum price increment value increment
+    price_tick_digits: int  # required decimal digits for above
+
+    size_tick: float  # minimum size (aka vlm) increment value increment
+
+    # size_tick_digits: int  # required decimal digits for above
+    @property
+    def size_tick_digits(self) -> int:
+        return self.size_tick
+
+    venue: str | None = None  # market venue provider name
+    expiry: str | None = None  # for derivs, expiry datetime parseable str
+
+    # for derivs, info describing contract, egs.
+    # strike price, call or put, swap type, exercise model, etc.
+    contract_info: str | None = None
+
+    @classmethod
+    def from_msg(
+        self,
+        msg: dict[str, Any],
+
+    ) -> MktPair:
+        '''
+        Constructor for a received msg-dict normally received over IPC.
+
+        '''
+        ...
+
+    # fqa, fqma, .. etc. see issue:
+    # https://github.com/pikers/piker/issues/467
+    @property
+    def fqsn(self) -> str:
+        '''
+        Return the fully qualified market (endpoint) name for the
+        pair of transacting assets.
+
+        '''
+        ...
+
+
+def mk_fqsn(
+    provider: str,
+    symbol: str,
+
+) -> str:
+    '''
+    Generate a "fully qualified symbol name" which is
+    a reverse-hierarchical cross broker/provider symbol
+
+    '''
+    return '.'.join([symbol, provider]).lower()
+
+
+def float_digits(
+    value: float,
+) -> int:
+    '''
+    Return the number of precision digits read from a float value.
+
+    '''
+    if value == 0:
+        return 0
+
+    return int(-Decimal(str(value)).as_tuple().exponent)
+
+
+def digits_to_dec(
+    ndigits: int,
+) -> Decimal:
+    '''
+    Return the minimum float value for an input integer value.
+
+    eg. 3 -> 0.001
+
+    '''
+    if ndigits == 0:
+        return Decimal('0')
+
+    return Decimal('0.' + '0'*(ndigits-1) + '1')
+
+
+def unpack_fqsn(fqsn: str) -> tuple[str, str, str]:
+    '''
+    Unpack a fully-qualified-symbol-name to ``tuple``.
+
+    '''
+    venue = ''
+    suffix = ''
+
+    # TODO: probably reverse the order of all this XD
+    tokens = fqsn.split('.')
+    if len(tokens) < 3:
+        # probably crypto
+        symbol, broker = tokens
+        return (
+            broker,
+            symbol,
+            '',
+        )
+
+    elif len(tokens) > 3:
+        symbol, venue, suffix, broker = tokens
+    else:
+        symbol, venue, broker = tokens
+        suffix = ''
+
+    # head, _, broker = fqsn.rpartition('.')
+    # symbol, _, suffix = head.rpartition('.')
+    return (
+        broker,
+        '.'.join([symbol, venue]),
+        suffix,
+    )
+
+# TODO: rework the below `Symbol` (which was originally inspired and
+# derived from stuff in quantdom) into a simpler, ipc msg ready, market
+# endpoint meta-data container type as per the drafted interace above.
+class Symbol(Struct):
+    '''
+    I guess this is some kinda container thing for dealing with
+    all the different meta-data formats from brokers?
+
+    '''
+    key: str
+    tick_size: float = 0.01
+    lot_tick_size: float = 0.0  # "volume" precision as min step value
+    tick_size_digits: int = 2
+    lot_size_digits: int = 0
+    suffix: str = ''
+    broker_info: dict[str, dict[str, Any]] = {}
+
+    @classmethod
+    def from_broker_info(
+        cls,
+        broker: str,
+        symbol: str,
+        info: dict[str, Any],
+        suffix: str = '',
+
+    ) -> Symbol:
+
+        tick_size = info.get('price_tick_size', 0.01)
+        lot_size = info.get('lot_tick_size', 0.0)
+
+        return Symbol(
+            key=symbol,
+
+            tick_size=tick_size,
+            lot_tick_size=lot_size,
+
+            tick_size_digits=float_digits(tick_size),
+            lot_size_digits=float_digits(lot_size),
+
+            suffix=suffix,
+            broker_info={broker: info},
+        )
+
+    @classmethod
+    def from_fqsn(
+        cls,
+        fqsn: str,
+        info: dict[str, Any],
+
+    ) -> Symbol:
+        broker, key, suffix = unpack_fqsn(fqsn)
+        return cls.from_broker_info(
+            broker,
+            key,
+            info=info,
+            suffix=suffix,
+        )
+
+    @property
+    def type_key(self) -> str:
+        return list(self.broker_info.values())[0]['asset_type']
+
+    @property
+    def brokers(self) -> list[str]:
+        return list(self.broker_info.keys())
+
+    def nearest_tick(self, value: float) -> float:
+        '''
+        Return the nearest tick value based on mininum increment.
+
+        '''
+        mult = 1 / self.tick_size
+        return round(value * mult) / mult
+
+    def front_feed(self) -> tuple[str, str]:
+        '''
+        Return the "current" feed key for this symbol.
+
+        (i.e. the broker + symbol key in a tuple).
+
+        '''
+        return (
+            list(self.broker_info.keys())[0],
+            self.key,
+        )
+
+    def tokens(self) -> tuple[str]:
+        broker, key = self.front_feed()
+        if self.suffix:
+            return (key, self.suffix, broker)
+        else:
+            return (key, broker)
+
+    @property
+    def fqsn(self) -> str:
+        return '.'.join(self.tokens()).lower()
+
+    def front_fqsn(self) -> str:
+        '''
+        fqsn = "fully qualified symbol name"
+
+        Basically the idea here is for all client-ish code (aka programs/actors
+        that ask the provider agnostic layers in the stack for data) should be
+        able to tell which backend / venue / derivative each data feed/flow is
+        from by an explicit string key of the current form:
+
+        <instrumentname>.<venue>.<suffixwithmetadata>.<brokerbackendname>
+
+        TODO: I have thoughts that we should actually change this to be
+        more like an "attr lookup" (like how the web should have done
+        urls, but marketting peeps ruined it etc. etc.):
+
+        <broker>.<venue>.<instrumentname>.<suffixwithmetadata>
+
+        '''
+        tokens = self.tokens()
+        fqsn = '.'.join(map(str.lower, tokens))
+        return fqsn
+
+    def quantize_size(
+        self,
+        size: float,
+
+    ) -> Decimal:
+        '''
+        Truncate input ``size: float`` using ``Decimal``
+        and ``.lot_size_digits``.
+
+        '''
+        digits = self.lot_size_digits
+        return Decimal(size).quantize(
+            Decimal(f'1.{"0".ljust(digits, "0")}'),
+            rounding=ROUND_HALF_EVEN
+        )
+
+

piker/accounting/_pos.py

@@ -43,10 +43,13 @@ from ._ledger import (
     iter_by_dt,
     open_trade_ledger,
 )
+from ._mktinfo import (
+    Symbol,
+    unpack_fqsn,
+)
 from .. import config
 from ..brokers import get_brokermod
 from ..clearing._messages import BrokerdPosition, Status
-from ..data._source import Symbol, unpack_fqsn
 from ..data.types import Struct
 from ..log import get_logger
 
@@ -154,6 +157,7 @@ class Position(Struct):
             inline_table['tid'] = tid
             toml_clears_list.append(inline_table)
 
+
         d['clears'] = toml_clears_list
 
         return fqsn, d

piker/brokers/ib/api.py

@@ -644,7 +644,7 @@ class Client:
         # fqsn parsing stage
         # ------------------
         if '.ib' in pattern:
-            from ..data._source import unpack_fqsn
+            from ..accounting._mktinfo import unpack_fqsn
             _, symbol, expiry = unpack_fqsn(pattern)
 
         else:

piker/brokers/ib/broker.py

@@ -70,7 +70,7 @@ from piker.clearing._messages import (
     BrokerdFill,
     BrokerdError,
 )
-from piker.data._source import (
+from piker.accounting._mktinfo import (
     Symbol,
     float_digits,
 )

piker/brokers/kraken/api.py

@@ -42,7 +42,7 @@ import trio
 
 from piker import config
 from piker.data.types import Struct
-from piker.data._source import Symbol
+from piker.accounting._mktinfo import Symbol
 from piker.brokers._util import (
     resproc,
     SymbolNotFound,

piker/brokers/kraken/broker.py

@@ -48,7 +48,7 @@ from piker.accounting import (
     open_pps,
     get_likely_pair,
 )
-from piker.data._source import (
+from piker.accounting._mktinfo import (
     Symbol,
     digits_to_dec,
 )

piker/clearing/_allocate.py

@@ -23,7 +23,7 @@ from typing import Optional
 
 from bidict import bidict
 
-from ..data._source import Symbol
+from ..accounting._mktinfo import Symbol
 from ..data.types import Struct
 from ..accounting import Position
 

piker/clearing/_client.py

@@ -27,6 +27,7 @@ import trio
 import tractor
 from tractor.trionics import broadcast_receiver
 
+from ..accounting._mktinfo import unpack_fqsn
 from ..log import get_logger
 from ..data.types import Struct
 from ..service import maybe_open_emsd
@@ -228,7 +229,6 @@ async def open_ems(
     # ready for order commands
     book = get_orders()
 
-    from ..data._source import unpack_fqsn
     broker, symbol, suffix = unpack_fqsn(fqsn)
 
     async with maybe_open_emsd(broker) as portal:

piker/clearing/_ems.py

@@ -43,7 +43,7 @@ import tractor
 
 from ..log import get_logger
 from ..data._normalize import iterticks
-from ..data._source import (
+from ..accounting._mktinfo import (
     unpack_fqsn,
     mk_fqsn,
     float_digits,
@@ -521,7 +521,6 @@ class Router(Struct):
         none already exists.
 
         '''
-        from ..data._source import unpack_fqsn
         broker, symbol, suffix = unpack_fqsn(fqsn)
 
         async with (

piker/clearing/_messages.py

@@ -29,7 +29,7 @@ from typing import (
 
 from msgspec import field
 
-from ..data._source import Symbol
+from ..accounting._mktinfo import Symbol
 from ..data.types import Struct
 
 

piker/clearing/_paper_engine.py

@@ -38,7 +38,7 @@ import tractor
 
 from .. import data
 from ..data.types import Struct
-from ..data._source import Symbol
+from ..accounting._mktinfo import Symbol
 from ..accounting import (
     Position,
     Transaction,

piker/data/_source.py

@@ -28,8 +28,12 @@ from bidict import bidict
 import numpy as np
 
 from .types import Struct
-# from numba import from_dtype
+from ..accounting._mktinfo import (
-
+    # mkfqsn,
+    unpack_fqsn,
+    # digits_to_dec,
+    float_digits,
+)
 
 ohlc_fields = [
     ('time', float),
@@ -50,6 +54,7 @@ base_ohlc_dtype = np.dtype(ohlc_fields)
 
 # TODO: for now need to construct this manually for readonly arrays, see
 # https://github.com/numba/numba/issues/4511
+# from numba import from_dtype
 # numba_ohlc_dtype = from_dtype(base_ohlc_dtype)
 
 # map time frame "keys" to seconds values
@@ -64,47 +69,6 @@ tf_in_1s = bidict({
 })
 
 
-def mk_fqsn(
-    provider: str,
-    symbol: str,
-
-) -> str:
-    '''
-    Generate a "fully qualified symbol name" which is
-    a reverse-hierarchical cross broker/provider symbol
-
-    '''
-    return '.'.join([symbol, provider]).lower()
-
-
-def float_digits(
-    value: float,
-) -> int:
-    '''
-    Return the number of precision digits read from a float value.
-
-    '''
-    if value == 0:
-        return 0
-
-    return int(-Decimal(str(value)).as_tuple().exponent)
-
-
-def digits_to_dec(
-    ndigits: int,
-) -> Decimal:
-    '''
-    Return the minimum float value for an input integer value.
-
-    eg. 3 -> 0.001
-
-    '''
-    if ndigits == 0:
-        return Decimal('0')
-
-    return Decimal('0.' + '0'*(ndigits-1) + '1')
-
-
 def ohlc_zeros(length: int) -> np.ndarray:
     """Construct an OHLC field formatted structarray.
 
@@ -115,223 +79,6 @@ def ohlc_zeros(length: int) -> np.ndarray:
     return np.zeros(length, dtype=base_ohlc_dtype)
 
 
-def unpack_fqsn(fqsn: str) -> tuple[str, str, str]:
-    '''
-    Unpack a fully-qualified-symbol-name to ``tuple``.
-
-    '''
-    venue = ''
-    suffix = ''
-
-    # TODO: probably reverse the order of all this XD
-    tokens = fqsn.split('.')
-    if len(tokens) < 3:
-        # probably crypto
-        symbol, broker = tokens
-        return (
-            broker,
-            symbol,
-            '',
-        )
-
-    elif len(tokens) > 3:
-        symbol, venue, suffix, broker = tokens
-    else:
-        symbol, venue, broker = tokens
-        suffix = ''
-
-    # head, _, broker = fqsn.rpartition('.')
-    # symbol, _, suffix = head.rpartition('.')
-    return (
-        broker,
-        '.'.join([symbol, venue]),
-        suffix,
-    )
-
-
-class MktPair(Struct, frozen=True):
-
-    src: str  # source asset name being used to buy
-    src_type: str  # source asset's financial type/classification name
-    # ^ specifies a "class" of financial instrument
-    # egs. stock, futer, option, bond etc.
-
-    dst: str  # destination asset name being bought
-    dst_type: str  # destination asset's financial type/classification name
-
-    price_tick: float  # minimum price increment value increment
-    price_tick_digits: int  # required decimal digits for above
-
-    size_tick: float  # minimum size (aka vlm) increment value increment
-    size_tick_digits: int  # required decimal digits for above
-
-    venue: str | None = None  # market venue provider name
-    expiry: str | None = None  # for derivs, expiry datetime parseable str
-
-    # for derivs, info describing contract, egs.
-    # strike price, call or put, swap type, exercise model, etc.
-    contract_info: str | None = None
-
-    @classmethod
-    def from_msg(
-        self,
-        msg: dict[str, Any],
-
-    ) -> MktPair:
-        '''
-        Constructor for a received msg-dict normally received over IPC.
-
-        '''
-        ...
-
-    # fqa, fqma, .. etc. see issue:
-    # https://github.com/pikers/piker/issues/467
-    @property
-    def fqsn(self) -> str:
-        '''
-        Return the fully qualified market (endpoint) name for the
-        pair of transacting assets.
-
-        '''
-        ...
-
-
-# TODO: rework the below `Symbol` (which was originally inspired and
-# derived from stuff in quantdom) into a simpler, ipc msg ready, market
-# endpoint meta-data container type as per the drafted interace above.
-class Symbol(Struct):
-    '''
-    I guess this is some kinda container thing for dealing with
-    all the different meta-data formats from brokers?
-
-    '''
-    key: str
-    tick_size: float = 0.01
-    lot_tick_size: float = 0.0  # "volume" precision as min step value
-    tick_size_digits: int = 2
-    lot_size_digits: int = 0
-    suffix: str = ''
-    broker_info: dict[str, dict[str, Any]] = {}
-
-    @classmethod
-    def from_broker_info(
-        cls,
-        broker: str,
-        symbol: str,
-        info: dict[str, Any],
-        suffix: str = '',
-
-    ) -> Symbol:
-
-        tick_size = info.get('price_tick_size', 0.01)
-        lot_size = info.get('lot_tick_size', 0.0)
-
-        return Symbol(
-            key=symbol,
-
-            tick_size=tick_size,
-            lot_tick_size=lot_size,
-
-            tick_size_digits=float_digits(tick_size),
-            lot_size_digits=float_digits(lot_size),
-
-            suffix=suffix,
-            broker_info={broker: info},
-        )
-
-    @classmethod
-    def from_fqsn(
-        cls,
-        fqsn: str,
-        info: dict[str, Any],
-
-    ) -> Symbol:
-        broker, key, suffix = unpack_fqsn(fqsn)
-        return cls.from_broker_info(
-            broker,
-            key,
-            info=info,
-            suffix=suffix,
-        )
-
-    @property
-    def type_key(self) -> str:
-        return list(self.broker_info.values())[0]['asset_type']
-
-    @property
-    def brokers(self) -> list[str]:
-        return list(self.broker_info.keys())
-
-    def nearest_tick(self, value: float) -> float:
-        '''
-        Return the nearest tick value based on mininum increment.
-
-        '''
-        mult = 1 / self.tick_size
-        return round(value * mult) / mult
-
-    def front_feed(self) -> tuple[str, str]:
-        '''
-        Return the "current" feed key for this symbol.
-
-        (i.e. the broker + symbol key in a tuple).
-
-        '''
-        return (
-            list(self.broker_info.keys())[0],
-            self.key,
-        )
-
-    def tokens(self) -> tuple[str]:
-        broker, key = self.front_feed()
-        if self.suffix:
-            return (key, self.suffix, broker)
-        else:
-            return (key, broker)
-
-    @property
-    def fqsn(self) -> str:
-        return '.'.join(self.tokens()).lower()
-
-    def front_fqsn(self) -> str:
-        '''
-        fqsn = "fully qualified symbol name"
-
-        Basically the idea here is for all client-ish code (aka programs/actors
-        that ask the provider agnostic layers in the stack for data) should be
-        able to tell which backend / venue / derivative each data feed/flow is
-        from by an explicit string key of the current form:
-
-        <instrumentname>.<venue>.<suffixwithmetadata>.<brokerbackendname>
-
-        TODO: I have thoughts that we should actually change this to be
-        more like an "attr lookup" (like how the web should have done
-        urls, but marketting peeps ruined it etc. etc.):
-
-        <broker>.<venue>.<instrumentname>.<suffixwithmetadata>
-
-        '''
-        tokens = self.tokens()
-        fqsn = '.'.join(map(str.lower, tokens))
-        return fqsn
-
-    def quantize_size(
-        self,
-        size: float,
-
-    ) -> Decimal:
-        '''
-        Truncate input ``size: float`` using ``Decimal``
-        and ``.lot_size_digits``.
-
-        '''
-        digits = self.lot_size_digits
-        return Decimal(size).quantize(
-            Decimal(f'1.{"0".ljust(digits, "0")}'),
-            rounding=ROUND_HALF_EVEN
-        )
-
-
 def _nan_to_closest_num(array: np.ndarray):
     """Return interpolated values instead of NaN.
 

piker/data/feed.py

@@ -70,11 +70,11 @@ from ._sharedmem import (
 )
 from .ingest import get_ingestormod
 from .types import Struct
-from ._source import (
+from ..accounting._mktinfo import (
-    base_iohlc_dtype,
     Symbol,
     unpack_fqsn,
 )
+from ._source import base_iohlc_dtype
 from ..ui import _search
 from ._sampling import (
     open_sample_stream,

piker/data/flows.py

@@ -30,10 +30,10 @@ import tractor
 import pendulum
 import numpy as np
 
-from .types import Struct
+from ..accounting._mktinfo import (
-from ._source import (
     Symbol,
 )
+from .types import Struct
 from ._sharedmem import (
     attach_shm_array,
     ShmArray,

piker/fsp/_engine.py

@@ -45,7 +45,7 @@ from ..data._sampling import (
     _default_delay_s,
     open_sample_stream,
 )
-from ..data._source import Symbol
+from ..accounting._mktinfo import Symbol
 from ._api import (
     Fsp,
     _load_builtins,

piker/ui/_app.py

@@ -28,7 +28,7 @@ from ..service import maybe_spawn_brokerd
 from . import _event
 from ._exec import run_qtractor
 from ..data.feed import install_brokerd_search
-from ..data._source import unpack_fqsn
+from ..accounting._mktinfo import unpack_fqsn
 from . import _search
 from ._chart import GodWidget
 from ..log import get_logger

piker/ui/_axes.py

@@ -29,7 +29,7 @@ from PyQt5 import QtCore, QtGui, QtWidgets
 from PyQt5.QtCore import QPointF
 
 from . import _pg_overrides as pgo
-from ..data._source import float_digits
+from ..accounting._mktinfo import float_digits
 from ._label import Label
 from ._style import DpiAwareFont, hcolor, _font
 from ._interaction import ChartView

piker/ui/_chart.py

@@ -68,7 +68,7 @@ from ..data.feed import (
     Feed,
     Flume,
 )
-from ..data._source import Symbol
+from ..accounting._mktinfo import Symbol
 from ..log import get_logger
 from ._interaction import ChartView
 from ._forms import FieldsForm

piker/ui/_fsp.py

@@ -46,7 +46,7 @@ from ..data._sharedmem import (
     try_read,
 )
 from ..data.feed import Flume
-from ..data._source import Symbol
+from ..accounting._mktinfo import Symbol
 from ._chart import (
     ChartPlotWidget,
     LinkedSplits,

piker/ui/order_mode.py

@@ -42,7 +42,7 @@ from ..clearing._allocate import (
     mk_allocator,
 )
 from ._style import _font
-from ..data._source import Symbol
+from ..accounting._mktinfo import Symbol
 from ..data.feed import (
     Feed,
     Flume,

tests/test_feeds.py

@@ -13,7 +13,7 @@ from piker.data import (
     ShmArray,
     open_feed,
 )
-from piker.data._source import (
+from piker.accounting._mktinfo import (
     unpack_fqsn,
 )