Bleh, move `.data.types` back up to top level pkg

Since it's depended on by `.data` stuff as well as pretty much
everything else, makes more sense to expose it as a top level module
(and maybe eventually as a subpkg as we add to it).

piker/accounting/__init__.py

@@ -50,6 +50,8 @@ from ._allocate import (
     mk_allocator,
     Allocator,
 )
+from .models import CostModel
+
 
 log = get_logger(__name__)
 
@@ -57,6 +59,7 @@ __all__ = [
     'Account',
     'Allocator',
     'Asset',
+    'CostModel',
     'MktPair',
     'Position',
     'Symbol',

piker/accounting/_allocate.py

@@ -25,7 +25,7 @@ from bidict import bidict
 
 from ._pos import Position
 from . import MktPair
-from ..data.types import Struct
+from piker.types import Struct
 
 
 _size_units = bidict({

piker/accounting/_ledger.py

@@ -37,8 +37,8 @@ from pendulum import (
 )
 import tomli_w  # for fast ledger writing
 
-from .. import config
+from piker.types import Struct
-from ..data.types import Struct
+from piker import config
 from ..log import get_logger
 from .calc import (
     iter_by_dt,

piker/accounting/_mktinfo.py

@@ -36,7 +36,7 @@ from typing import (
     Literal,
 )
 
-from ..data.types import Struct
+from piker.types import Struct
 
 
 # TODO: make these literals..

piker/accounting/_pos.py

@@ -58,8 +58,8 @@ from .. import config
 from ..clearing._messages import (
     BrokerdPosition,
 )
-from ..data.types import Struct
+from piker.types import Struct
-from ..data._symcache import SymbologyCache
+from piker.data._symcache import SymbologyCache
 from ..log import get_logger
 
 log = get_logger(__name__)

piker/accounting/calc.py

@@ -422,7 +422,12 @@ def ledger_to_dfs(
 
     #     fdf = df.filter(pred)
 
-    # break up into a frame per mkt / fqme
+    # TODO: originally i had tried just using a plain ol' groupby
+    # + agg here but the issue was re-inserting to the src frame.
+    # however, learning more about `polars` seems like maybe we can
+    # use `.over()`?
+    # https://pola-rs.github.io/polars/py-polars/html/reference/expressions/api/polars.Expr.over.html#polars.Expr.over
+    # => CURRENTLY we break up into a frame per mkt / fqme
     dfs: dict[str, pl.DataFrame] = ldf.partition_by(
         'bs_mktid',
         as_dict=True,
@@ -435,14 +440,9 @@ def ledger_to_dfs(
 
         # covert to lazy form (since apparently we might need it
         # eventually ...)
-        df = dfs[key]
+        df: pl.DataFrame = dfs[key]
 
-        ldf = df.lazy()
+        ldf: pl.LazyFrame = df.lazy()
-        # TODO: pass back the current `Position` object loaded from
-        # the account as well? Would provide incentive to do all
-        # this ledger loading inside a new async open_account().
-        # bs_mktid: str = df[0]['bs_mktid']
-        # pos: Position = acnt.pps[bs_mktid]
 
         df = dfs[key] = ldf.with_columns([
 
@@ -688,4 +688,10 @@ def ledger_to_dfs(
             last_ppu: float = ppu
             last_cumsize: float = cumsize
 
+    # TODO?: pass back the current `Position` object loaded from
+    # the account as well? Would provide incentive to do all
+    # this ledger loading inside a new async open_account().
+    # bs_mktid: str = df[0]['bs_mktid']
+    # pos: Position = acnt.pps[bs_mktid]
+
     return dfs

piker/brokers/binance/api.py

@@ -53,7 +53,7 @@ from piker.accounting import (
     Asset,
     digits_to_dec,
 )
-from piker.data.types import Struct
+from piker.types import Struct
 from piker.data import def_iohlcv_fields
 from piker.brokers import (
     resproc,

piker/brokers/binance/feed.py

@@ -58,7 +58,7 @@ from piker.accounting import (
     MktPair,
     unpack_fqme,
 )
-from piker.data.types import Struct
+from piker.types import Struct
 from piker.data.validate import FeedInit
 from piker.data._web_bs import (
     open_autorecon_ws,

piker/brokers/binance/venues.py

@@ -26,7 +26,7 @@ from decimal import Decimal
 
 from msgspec import field
 
-from piker.data.types import Struct
+from piker.types import Struct
 
 
 # API endpoint paths by venue / sub-API

piker/brokers/ib/broker.py

@@ -49,6 +49,7 @@ from ib_insync.objects import (
 )
 
 from piker import config
+from piker.types import Struct
 from piker.accounting import (
     Position,
     Transaction,
@@ -62,7 +63,6 @@ from piker.accounting import (
 from piker.data import (
     open_symcache,
     SymbologyCache,
-    Struct,
 )
 from piker.clearing._messages import (
     Order,

piker/brokers/ib/ledger.py

@@ -39,8 +39,8 @@ from ib_insync.objects import (
     CommissionReport,
 )
 
+from piker.types import Struct
 from piker.data import (
-    Struct,
     SymbologyCache,
 )
 from piker.accounting import (

piker/brokers/kraken/feed.py

@@ -46,7 +46,7 @@ from piker.brokers._util import (
     DataThrottle,
     DataUnavailable,
 )
-from piker.data.types import Struct
+from piker.types import Struct
 from piker.data.validate import FeedInit
 from piker.data._web_bs import open_autorecon_ws, NoBsWs
 from .api import (

piker/brokers/kraken/ledger.py

@@ -34,9 +34,9 @@ from piker.accounting import (
     TransactionLedger,
     # MktPair,
 )
+from piker.types import Struct
 from piker.data import (
     SymbologyCache,
-    Struct,
 )
 from .api import (
     log,

piker/brokers/kraken/symbols.py

@@ -33,7 +33,7 @@ from piker.brokers import (
     open_cached_client,
     SymbolNotFound,
 )
-from piker.data.types import Struct
+from piker.types import Struct
 from piker.accounting._mktinfo import (
     Asset,
     MktPair,

piker/brokers/kucoin.py

@@ -64,7 +64,7 @@ from piker._cacheables import (
 )
 from piker.log import get_logger
 from piker.data.validate import FeedInit
-from piker.data.types import Struct
+from piker.types import Struct
 from piker.data import def_iohlcv_fields
 from piker.data._web_bs import (
     open_autorecon_ws,

piker/clearing/__init__.py

@@ -27,13 +27,28 @@ from ._ems import (
     open_brokerd_dialog,
 )
 from ._util import OrderDialogs
+from ._messages import(
+    Order,
+    Status,
+    Cancel,
+
+    # TODO: deprecate these and replace end-2-end with
+    # client-side-dialog set above B)
+    # https://github.com/pikers/piker/issues/514
+    BrokerdPosition
+)
 
 
 __all__ = [
+    'FeeModel',
     'open_ems',
     'OrderClient',
     'open_brokerd_dialog',
     'OrderDialogs',
+    'Order',
+    'Status',
+    'Cancel',
+    'BrokerdPosition'
 
 ]
 

piker/clearing/_client.py

@@ -30,7 +30,7 @@ from tractor.trionics import broadcast_receiver
 from ._util import (
     log,  # sub-sys logger
 )
-from ..data.types import Struct
+from piker.types import Struct
 from ..service import maybe_open_emsd
 from ._messages import (
     Order,

piker/clearing/_ems.py

@@ -51,9 +51,9 @@ from ..accounting._mktinfo import (
     unpack_fqme,
     dec_digits,
 )
+from piker.types import Struct
 from ..ui._notify import notify_from_ems_status_msg
 from ..data import iterticks
-from ..data.types import Struct
 from ._messages import (
     Order,
     Status,

piker/clearing/_messages.py

@@ -28,7 +28,7 @@ from typing import (
 
 from msgspec import field
 
-from ..data.types import Struct
+from piker.types import Struct
 
 
 # TODO: a composite for tracking msg flow on 2-legged

piker/clearing/_paper_engine.py

@@ -40,21 +40,22 @@ import tractor
 
 from piker.brokers import get_brokermod
 from piker.accounting import (
-    Position,
     Account,
+    CostModel,
+    MktPair,
+    Position,
     Transaction,
     TransactionLedger,
-    open_trade_ledger,
     open_account,
-    MktPair,
+    open_trade_ledger,
     unpack_fqme,
 )
 from piker.data import (
-    open_feed,
-    iterticks,
     Struct,
-    open_symcache,
     SymbologyCache,
+    iterticks,
+    open_feed,
+    open_symcache,
 )
 from ._util import (
     log,  # sub-sys logger
@@ -83,6 +84,7 @@ class PaperBoi(Struct):
     ems_trades_stream: tractor.MsgStream
     acnt: Account
     ledger: TransactionLedger
+    fees: CostModel
 
     # map of paper "live" orders which be used
     # to simulate fills based on paper engine settings

piker/clearing/_util.py

@@ -25,7 +25,7 @@ from ..log import (
     get_logger,
     get_console_log,
 )
-from piker.data.types import Struct
+from piker.types import Struct
 subsys: str = 'piker.clearing'
 
 log = get_logger(subsys)

piker/data/__init__.py

@@ -44,7 +44,7 @@ from ._symcache import (
     open_symcache,
     get_symcache,
 )
-from .types import Struct
+from ._sampling import open_sample_stream
 
 
 __all__: list[str] = [
@@ -60,7 +60,8 @@ __all__: list[str] = [
     'def_iohlcv_fields',
     'def_ohlcv_fields',
     'open_symcache',
+    'open_sample_stream',
     'get_symcache',
     'SymbologyCache',
-    'Struct',
+    'types',
 ]

piker/data/_sharedmem.py

@@ -34,7 +34,7 @@ import tractor
 
 from ._util import log
 from ._source import def_iohlcv_fields
-from .types import Struct
+from piker.types import Struct
 
 
 def cuckoff_mantracker():

piker/data/_symcache.py

@@ -47,11 +47,11 @@ from msgspec import field
 
 from piker.log import get_logger
 from piker import config
+from piker.types import Struct
 from piker.brokers import (
     open_cached_client,
     get_brokermod,
 )
-from .types import Struct
 
 if TYPE_CHECKING:
     from ..accounting import (

piker/data/_web_bs.py

@@ -50,8 +50,8 @@ from trio_websocket._impl import (
     ConnectionTimeout,
 )
 
+from piker.types import Struct
 from ._util import log
-from .types import Struct
 
 
 class NoBsWs:

piker/data/feed.py

@@ -50,15 +50,21 @@ from tractor.trionics import (
     gather_contexts,
 )
 
-from ..brokers import get_brokermod
+from piker.accounting import (
-from ..calc import humanize
+    MktPair,
+    unpack_fqme,
+)
+from piker.types import Struct
+from piker.brokers import get_brokermod
+from piker.service import (
+    maybe_spawn_brokerd,
+)
+from piker.ui import _search
+from piker.calc import humanize
 from ._util import (
     log,
     get_console_log,
 )
-from ..service import (
-    maybe_spawn_brokerd,
-)
 from .flows import Flume
 from .validate import (
     FeedInit,
@@ -68,12 +74,6 @@ from .history import (
     manage_history,
 )
 from .ingest import get_ingestormod
-from .types import Struct
-from ..accounting import (
-    MktPair,
-    unpack_fqme,
-)
-from ..ui import _search
 from ._sampling import (
     sample_and_broadcast,
     uniform_rate_send,

piker/data/flows.py

@@ -30,8 +30,7 @@ import tractor
 import pendulum
 import numpy as np
 
-from ..accounting import MktPair
+from piker.types import Struct
-from .types import Struct
 from ._sharedmem import (
     attach_shm_array,
     ShmArray,
@@ -39,7 +38,7 @@ from ._sharedmem import (
 )
 
 if TYPE_CHECKING:
-    # from pyqtgraph import PlotItem
+    from ..accounting import MktPair
     from .feed import Feed
 
 
@@ -189,6 +188,7 @@ class Flume(Struct):
 
         '''
         mkt_msg = msg.pop('mkt')
+        from ..accounting import MktPair  # cycle otherwise..
         mkt = MktPair.from_msg(mkt_msg)
         return cls(mkt=mkt, **msg)
 

piker/data/validate.py

@@ -18,6 +18,7 @@ Data feed synchronization protocols, init msgs, and general
 data-provider-backend-agnostic schema definitions.
 
 '''
+from __future__ import annotations
 from decimal import Decimal
 from pprint import pformat
 from types import ModuleType
@@ -28,8 +29,8 @@ from typing import (
 
 from msgspec import field
 
-from .types import Struct
+from piker.types import Struct
-from ..accounting import (
+from piker.accounting import (
     Asset,
     MktPair,
 )

piker/fsp/__init__.py

@@ -22,9 +22,25 @@ from typing import AsyncIterator
 
 import numpy as np
 
+from ._api import (
+    maybe_mk_fsp_shm,
+    Fsp,
+)
 from ._engine import cascade
+from ._volume import (
+    dolla_vlm,
+    flow_rates,
+    tina_vwap,
+)
 
-__all__ = ['cascade']
+__all__: list[str] = [
+    'cascade',
+    'maybe_mk_fsp_shm',
+    'Fsp',
+    'dolla_vlm',
+    'flow_rates',
+    'tina_vwap',
+]
 
 
 async def latency(

piker/ui/_display.py

@@ -36,25 +36,27 @@ import pyqtgraph as pg
 from msgspec import field
 
 # from .. import brokers
-from ..accounting import (
+from piker.accounting import (
     MktPair,
 )
-from ..data import (
+from piker.types import Struct
+from piker.data import (
     open_feed,
     Feed,
     Flume,
+    open_sample_stream,
+    ShmArray,
 )
-from ..data.ticktools import (
+from piker.data.ticktools import (
     _tick_groups,
     _auction_ticks,
 )
-from ..data.types import Struct
+from piker.toolz import (
-from ..data._sharedmem import (
+    pg_profile_enabled,
-    ShmArray,
+    ms_slower_then,
-)
+    Profiler,
-from ..data._sampling import (
-    open_sample_stream,
 )
+from piker.log import get_logger
 # from ..data._source import tf_in_1s
 from ._axes import YAxisLabel
 from ._chart import (
@@ -79,12 +81,6 @@ from .order_mode import (
     open_order_mode,
     OrderMode,
 )
-from ..toolz import (
-    pg_profile_enabled,
-    ms_slower_then,
-    Profiler,
-)
-from ..log import get_logger
 
 if TYPE_CHECKING:
     from ._interaction import ChartView

piker/ui/_editors.py

@@ -42,10 +42,10 @@ from pyqtgraph import functions as fn
 from PyQt5.QtCore import QPointF
 import numpy as np
 
+from piker.types import Struct
 from ._style import hcolor, _font
 from ._lines import LevelLine
 from ..log import get_logger
-from ..data.types import Struct
 
 if TYPE_CHECKING:
     from ._chart import GodWidget

piker/ui/_event.py

@@ -30,7 +30,7 @@ from PyQt5.QtWidgets import (
     QGraphicsSceneMouseEvent as gs_mouse,
 )
 
-from ..data.types import Struct
+from piker.types import Struct
 
 
 MOUSE_EVENTS = {

piker/ui/_fsp.py

@@ -36,16 +36,27 @@ from tractor.trionics import maybe_open_context
 import trio
 from trio_typing import TaskStatus
 
-from piker.data.types import Struct
+from piker.accounting import MktPair
-from ._axes import PriceAxis
+from piker.fsp import (
-from ..calc import humanize
+    cascade,
-from ..data._sharedmem import (
+    maybe_mk_fsp_shm,
+    Fsp,
+    dolla_vlm,
+    flow_rates,
+)
+from piker.data import (
+    Flume,
     ShmArray,
+)
+from piker.data._sharedmem import (
     _Token,
     try_read,
 )
-from ..data.feed import Flume
+from piker.log import get_logger
-from ..accounting import MktPair
+from piker.toolz import Profiler
+from piker.types import Struct
+from ._axes import PriceAxis
+from ..calc import humanize
 from ._chart import (
     ChartPlotWidget,
     LinkedSplits,
@@ -55,18 +66,6 @@ from ._forms import (
     mk_form,
     open_form_input_handling,
 )
-from ..fsp._api import (
-    maybe_mk_fsp_shm,
-    Fsp,
-)
-from ..fsp import cascade
-from ..fsp._volume import (
-    # tina_vwap,
-    dolla_vlm,
-    flow_rates,
-)
-from ..log import get_logger
-from ..toolz import Profiler
 
 log = get_logger(__name__)
 

piker/ui/_position.py

@@ -30,40 +30,34 @@ from typing import (
     TYPE_CHECKING,
 )
 
-
 # from PyQt5.QtWidgets import QStyle
 # from PyQt5.QtGui import (
 #     QIcon, QPixmap, QColor
 # )
 from pyqtgraph import functions as fn
 
+from piker.calc import (
+    humanize,
+    pnl,
+    puterize,
+)
+from piker.accounting import (
+    Allocator,
+    Position,
+    MktPair,
+)
+from piker.accounting._mktinfo import _derivs
+from piker.types import Struct
+from piker.data import (
+    iterticks,
+    Feed,
+    Flume,
+)
 from ._annotate import LevelMarker
 from ._anchors import (
     pp_tight_and_right,  # wanna keep it straight in the long run
     gpath_pin,
 )
-from ..calc import (
-    humanize,
-    pnl,
-    puterize,
-)
-from ..accounting import (
-    Allocator,
-    MktPair,
-)
-from ..accounting import (
-    Position,
-)
-from ..accounting._mktinfo import (
-    _derivs,
-)
-
-from ..data import (
-    iterticks,
-    Feed,
-    Flume,
-)
-from ..data.types import Struct
 from ._label import Label
 from ._lines import LevelLine, order_line
 from ._style import _font

piker/ui/order_mode.py

@@ -36,25 +36,29 @@ import tractor
 import trio
 from PyQt5.QtCore import Qt
 
-from .. import config
+from piker import config
-from ..accounting import (
+from piker.accounting import (
     Allocator,
     Position,
     mk_allocator,
     MktPair,
     Symbol,
 )
-from ..clearing._client import (
+from piker.clearing import (
     open_ems,
     OrderClient,
 )
-from ._style import _font
+from piker.clearing._messages import (
-from ..data.feed import (
+    Order,
+    Status,
+    BrokerdPosition,
+)
+from piker.data import (
     Feed,
     Flume,
 )
-from ..data.types import Struct
+from piker.types import Struct
-from ..log import get_logger
+from piker.log import get_logger
 from ._editors import LineEditor, ArrowEditor
 from ._lines import order_line, LevelLine
 from ._position import (
@@ -63,14 +67,7 @@ from ._position import (
 )
 from ._forms import FieldsForm
 from ._window import MultiStatus
-from ..clearing._messages import (
+from ._style import _font
-    # Cancel,
-    Order,
-    Status,
-    # BrokerdOrder,
-    # BrokerdStatus,
-    BrokerdPosition,
-)
 from ._forms import open_form_input_handling
 from ._notify import notify_from_ems_status_msg
 

piker/ui/view_mode.py

@@ -30,7 +30,7 @@ import numpy as np
 import pendulum
 import pyqtgraph as pg
 
-from ..data.types import Struct
+from piker.types import Struct
 from ..data._timeseries import slice_from_time
 from ..log import get_logger
 from ..toolz import Profiler