Add Plot

Here is the `plot_graph()` that is in char of the bars, scatter and vertical line plot items.

Also all the necessary code  for the graph to be shown.

examples/max_pain.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python
+from decimal import (
+    Decimal,
+)
+import trio
+import tractor
+from datetime import datetime
+from pprint import pformat
+from piker.brokers.deribit.api import (
+    get_client,
+    maybe_open_oi_feed,
+)
+import sys
+import pyqtgraph as pg
+from PyQt6 import QtCore
+from pyqtgraph import ScatterPlotItem, InfiniteLine
+from PyQt6.QtWidgets import QApplication
+
+def check_if_complete(
+        oi: dict[str, dict[str, Decimal | None]]
+    ) -> bool:
+    return all(
+        oi[strike]['C'] is not None
+        and
+        oi[strike]['P'] is not None for strike in oi
+    )
+
+
+async def max_pain_daemon(
+) -> None:
+    oi_by_strikes: dict[str, dict[str, Decimal | None]]
+    instruments: list[Symbol] = []
+    expiry_dates: list[str]
+    expiry_date: str
+    currency: str = 'btc'
+    kind: str = 'option'
+
+    async with get_client(
+    ) as client:
+        expiry_dates: list[str] = await client.get_expiration_dates(
+            currency=currency,
+            kind=kind
+        )
+
+        print(f'Available expiration dates for {currency}-{kind}:')
+        print(f'{expiry_dates}')
+        expiry_date = input('Please enter a valid expiration date: ').upper()
+        print('Starting little daemon...')
+
+        oi_by_strikes: dict[str, dict[str, Decimal]]
+        instruments = await client.get_instruments(
+            expiry_date=expiry_date,
+        )
+        oi_by_strikes = client.get_strikes_dict(instruments)
+
+
+    def get_total_intrinsic_values(
+        oi_by_strikes: dict[str, dict[str, Decimal]]
+    ) -> dict[str, dict[str, Decimal]]:
+        call_cash: Decimal = Decimal(0)
+        put_cash: Decimal = Decimal(0)
+        intrinsic_values: dict[str, dict[str, Decimal]] = {}
+        closes: list = sorted(Decimal(close) for close in oi_by_strikes)
+
+        for strike, oi in oi_by_strikes.items():
+            s = Decimal(strike)
+            call_cash = sum(max(0, (s - c) * oi_by_strikes[str(c)]['C']) for c in closes)
+            put_cash = sum(max(0, (c - s) * oi_by_strikes[str(c)]['P']) for c in closes)
+
+            intrinsic_values[strike] = {
+                'C': call_cash,
+                'P': put_cash,
+                'total': call_cash + put_cash,
+            }
+
+        return intrinsic_values
+
+    def get_intrinsic_value_and_max_pain(
+        intrinsic_values: dict[str, dict[str, Decimal]]
+        ):
+        # We meed to find the lowest value, so we start at
+        # infinity to ensure that, and the max_pain must be
+        # an amount greater than zero.
+        total_intrinsic_value: Decimal = Decimal('Infinity')
+        max_pain: Decimal = Decimal(0)
+
+        for strike, oi in oi_by_strikes.items():
+            s = Decimal(strike)
+            if intrinsic_values[strike]['total'] < total_intrinsic_value:
+                total_intrinsic_value = intrinsic_values[strike]['total']
+                max_pain = s
+
+        return total_intrinsic_value, max_pain
+
+    def plot_graph(
+        oi_by_strikes: dict[str, dict[str, Decimal]],
+        plot,
+    ):
+        """Update the bar graph with new open interest data."""
+        plot.clear()
+
+        intrinsic_values = get_total_intrinsic_values(oi_by_strikes)
+
+        for strike_str in sorted(oi_by_strikes, key=lambda x: int(x)):
+            strike = int(strike_str)
+            calls_val = float(oi_by_strikes[strike_str]['C'])
+            puts_val  = float(oi_by_strikes[strike_str]['P'])
+
+            bar_c = pg.BarGraphItem(
+                x=[strike - 100],
+                height=[calls_val],
+                width=200,
+                pen='w',
+                brush=(0, 0, 255, 150)
+            )
+            plot.addItem(bar_c)
+
+            bar_p = pg.BarGraphItem(
+                x=[strike + 100],
+                height=[puts_val],
+                width=200,
+                pen='w',
+                brush=(255, 0, 0, 150)
+            )
+            plot.addItem(bar_p)
+
+            total_val = float(intrinsic_values[strike_str]['total']) / 100000
+
+            scatter_iv = ScatterPlotItem(
+                x=[strike],
+                y=[total_val],
+                pen=pg.mkPen(color=(0, 255, 0), width=2),
+                brush=pg.mkBrush(0, 255, 0, 150),
+                size=3,
+                symbol='o'
+            )
+            plot.addItem(scatter_iv)
+
+        _, max_pain = get_intrinsic_value_and_max_pain(intrinsic_values)
+
+        vertical_line = InfiniteLine(
+            pos=max_pain,
+            angle=90,
+            pen=pg.mkPen(color='yellow', width=1, style=QtCore.Qt.PenStyle.DotLine),
+            label=f'Max pain: {max_pain:,.0f}',
+            labelOpts={
+                'position': 0.85,
+                'color': 'yellow',
+                'movable': True
+            }
+        )
+        plot.addItem(vertical_line)
+
+    def update_oi_by_strikes(msg: tuple):
+        nonlocal oi_by_strikes
+        if 'oi' == msg[0]:
+            strike_price = msg[1]['strike_price']
+            option_type = msg[1]['option_type']
+            open_interest = msg[1]['open_interest']
+            oi_by_strikes.setdefault(
+                strike_price, {}
+            ).update(
+                {option_type: open_interest}
+            )
+
+    def get_max_pain(
+        oi_by_strikes: dict[str, dict[str, Decimal]]
+    ) -> dict[str, str | Decimal]:
+        '''
+        This method requires only the strike_prices and oi for call
+        and puts, the closes list are the same as the strike_prices
+        the idea is to sum all the calls and puts cash for each strike
+        and the ITM strikes from that strike, the lowest value is what we
+        are looking for the intrinsic value.
+
+        '''
+
+        nonlocal timestamp
+
+        intrinsic_values = get_total_intrinsic_values(oi_by_strikes)
+
+        total_intrinsic_value, max_pain = get_intrinsic_value_and_max_pain(intrinsic_values)
+
+        return {
+            'timestamp': timestamp,
+            'expiry_date': expiry_date,
+            'total_intrinsic_value': total_intrinsic_value,
+            'max_pain': max_pain,
+        }
+
+    async with maybe_open_oi_feed(
+        instruments,
+    ) as oi_feed:
+        # Initialize QApplication
+        app = QApplication(sys.argv)
+
+        win = pg.GraphicsLayoutWidget(show=True)
+        win.setWindowTitle('Calls (blue) vs Puts (red)')
+
+        plot = win.addPlot(title='OI by Strikes')
+        plot.showGrid(x=True, y=True)
+        print('Plot initialized...')
+
+        async for msg in oi_feed:
+
+            update_oi_by_strikes(msg)
+            if check_if_complete(oi_by_strikes):
+                if 'oi' == msg[0]:
+                    timestamp = msg[1]['timestamp']
+                    max_pain = get_max_pain(oi_by_strikes)
+                    intrinsic_values = get_total_intrinsic_values(oi_by_strikes)
+
+                    # graph here
+                    plot_graph(oi_by_strikes, plot)
+
+                    print('-----------------------------------------------')
+                    print(f'timestamp:             {datetime.fromtimestamp(max_pain['timestamp'])}')
+                    print(f'expiry_date:           {max_pain['expiry_date']}')
+                    print(f'max_pain:              {max_pain['max_pain']:,.0f}')
+                    print(f'total intrinsic value: {max_pain['total_intrinsic_value']:,.0f}')
+                    print('-----------------------------------------------')
+
+            # Process GUI events to keep the window responsive
+            app.processEvents()
+
+
+async def main():
+
+    async with tractor.open_nursery() as n:
+
+        p: tractor.Portal = await n.start_actor(
+            'max_pain_daemon',
+            enable_modules=[__name__],
+            infect_asyncio=True,
+        )
+        await p.run(max_pain_daemon)
+
+if __name__ == '__main__':
+    trio.run(main)

examples/max_pain_readme.rst

@@ -0,0 +1,19 @@
+## Max Pain Calculation for Deribit Options
+
+This feature, which calculates the max pain point for options traded on the Deribit exchange using cryptofeed library.
+
+- Functions in the api module for fetching options data from Deribit. [commit](https://pikers.dev/pikers/piker/commit/da55856dd2876291f55a06eb0561438a912d8241)
+
+- Compute the max pain point based on open interest data using deribit's api. [commit](https://pikers.dev/pikers/piker/commit/0d9d6e15ba0edeb662ec97f7599dd66af3046b94)
+
+### How to test it?
+
+**Before start:** in order to get this working with `uv`,  you **must** use my `tractor` [fork](https://pikers.dev/ntorres/tractor/src/branch/aio_abandons) and this branch: `aio_abandons`, the reason is that I cherry-pick the `uv_migration` that guille made, for some reason that a didn't dive into, in my system y need tractor using `uv` too. quite hacky I guess.
+
+1. `uv lock`
+
+2. `uv run --no-dev python examples/max_pain.py`
+
+3. A message should be display, enter one of the expiration date available.
+
+4. The script should be up and running.

piker/brokers/__init__.py

@@ -51,6 +51,7 @@ __brokers__: list[str] = [
     'ib',
     'kraken',
     'kucoin',
+    'deribit',
 
     # broken but used to work
     # 'questrade',
@@ -61,7 +62,6 @@ __brokers__: list[str] = [
     # wstrade
     # iex
 
-    # deribit
     # bitso
 ]
 

piker/brokers/deribit/__init__.py

@@ -25,6 +25,7 @@ from .api import (
     get_client,
 )
 from .feed import (
+    get_mkt_info,
     open_history_client,
     open_symbol_search,
     stream_quotes,
@@ -34,15 +35,20 @@ from .feed import (
     # open_trade_dialog,
     # norm_trade_records,
 # )
+from .venues import (
+    OptionPair,
+)
 
 log = get_logger(__name__)
 
 __all__ = [
     'get_client',
 #    'trades_dialogue',
+    'get_mkt_info',
     'open_history_client',
     'open_symbol_search',
     'stream_quotes',
+    'OptionPair',
 #    'norm_trade_records',
 ]
 

piker/brokers/deribit/feed.py

@@ -18,38 +18,59 @@
 Deribit backend.
 
 '''
+from __future__ import annotations
 from contextlib import asynccontextmanager as acm
 from datetime import datetime
-from typing import Any, Optional, Callable
+from typing import (
+    # Any,
+    # Optional,
+    Callable,
+)
+# from pprint import pformat
 import time
 
+import cryptofeed
 import trio
 from trio_typing import TaskStatus
-import pendulum
-from rapidfuzz import process as fuzzy
+from pendulum import (
+    from_timestamp,
+)
 import numpy as np
 import tractor
 
-from piker.brokers import open_cached_client
-from piker.log import get_logger, get_console_log
-from piker.data import ShmArray
-from piker.brokers._util import (
-    BrokerError,
+from piker.accounting import (
+    Asset,
+    MktPair,
+    unpack_fqme,
+)
+from piker.brokers import (
+    open_cached_client,
+    NoData,
     DataUnavailable,
 )
-
-from cryptofeed import FeedHandler
-from cryptofeed.defines import (
-    DERIBIT, L1_BOOK, TRADES, OPTION, CALL, PUT
+from piker._cacheables import (
+    async_lifo_cache,
 )
-from cryptofeed.symbols import Symbol
+from piker.log import (
+    get_logger,
+    mk_repr,
+)
+from piker.data.validate import FeedInit
+
 
 from .api import (
-    Client, Trade,
-    get_config,
-    str_to_cb_sym, piker_sym_to_cb_sym, cb_sym_to_deribit_inst,
+    Client,
+    # get_config,
+    piker_sym_to_cb_sym,
+    cb_sym_to_deribit_inst,
+    str_to_cb_sym,
     maybe_open_price_feed
 )
+from .venues import (
+    Pair,
+    OptionPair,
+    Trade,
+)
 
 _spawn_kwargs = {
     'infect_asyncio': True,
@@ -64,90 +85,215 @@ async def open_history_client(
     mkt: MktPair,
 ) -> tuple[Callable, int]:
 
-    fnstrument: str = mkt.bs_fqme
     # TODO implement history getter for the new storage layer.
     async with open_cached_client('deribit') as client:
 
+        pair: OptionPair = client._pairs[mkt.dst.name]
+        # XXX NOTE, the cuckers use ms !!!
+        creation_time_s: int = pair.creation_timestamp/1000
+
         async def get_ohlc(
-            end_dt: Optional[datetime] = None,
-            start_dt: Optional[datetime] = None,
+            timeframe: float,
+            end_dt: datetime | None = None,
+            start_dt: datetime | None = None,
 
         ) -> tuple[
             np.ndarray,
             datetime,  # start
             datetime,  # end
         ]:
+            if timeframe != 60:
+                raise DataUnavailable('Only 1m bars are supported')
 
-            array = await client.bars(
-                instrument,
+            array: np.ndarray = await client.bars(
+                mkt,
                 start_dt=start_dt,
                 end_dt=end_dt,
             )
             if len(array) == 0:
-                raise DataUnavailable
+                if (
+                    end_dt is None
+                ):
+                    raise DataUnavailable(
+                        'No history seems to exist yet?\n\n'
+                        f'{mkt}'
+                    )
+                elif (
+                    end_dt
+                    and
+                    end_dt.timestamp() < creation_time_s
+                ):
+                    # the contract can't have history
+                    # before it was created.
+                    pair_type_str: str = type(pair).__name__
+                    create_dt: datetime = from_timestamp(creation_time_s)
+                    raise DataUnavailable(
+                        f'No history prior to\n'
+                        f'`{pair_type_str}.creation_timestamp: int = '
+                        f'{pair.creation_timestamp}\n\n'
+                        f'------ deribit sux ------\n'
+                        f'WHICH IN "NORMAL PEOPLE WHO USE EPOCH TIME" form is,\n'
+                        f'creation_time_s: {creation_time_s}\n'
+                        f'create_dt: {create_dt}\n'
+                    )
+                raise NoData(
+                    f'No frame for {start_dt} -> {end_dt}\n'
+                )
 
-            start_dt = pendulum.from_timestamp(array[0]['time'])
-            end_dt = pendulum.from_timestamp(array[-1]['time'])
+            start_dt = from_timestamp(array[0]['time'])
+            end_dt = from_timestamp(array[-1]['time'])
+
+            times = array['time']
+            if not times.any():
+                raise ValueError(
+                    'Bad frame with null-times?\n\n'
+                    f'{times}'
+                )
+
+            if end_dt is None:
+                inow: int = round(time.time())
+                if (inow - times[-1]) > 60:
+                    await tractor.pause()
 
             return array, start_dt, end_dt
 
-        yield get_ohlc, {'erlangs': 3, 'rate': 3}
+        yield (
+            get_ohlc,
+            {  # backfill config
+                'erlangs': 3,
+                'rate': 3,
+            }
+        )
+
+
+@async_lifo_cache()
+async def get_mkt_info(
+    fqme: str,
+
+) -> tuple[MktPair, Pair|OptionPair] | None:
+
+    # uppercase since kraken bs_mktid is always upper
+    if 'deribit' not in fqme.lower():
+        fqme += '.deribit'
+
+    mkt_mode: str = ''
+    broker, mkt_ep, venue, expiry = unpack_fqme(fqme)
+
+    # NOTE: we always upper case all tokens to be consistent with
+    # binance's symbology style for pairs, like `BTCUSDT`, but in
+    # theory we could also just keep things lower case; as long as
+    # we're consistent and the symcache matches whatever this func
+    # returns, always!
+    expiry: str = expiry.upper()
+    venue: str = venue.upper()
+    # venue_lower: str = venue.lower()
+
+    mkt_mode: str = 'option'
+
+    async with open_cached_client(
+        'deribit',
+    ) as client:
+
+        assets: dict[str, Asset] = await client.get_assets()
+        pair_str: str = mkt_ep.lower()
+
+        pair: Pair = await client.exch_info(
+            sym=pair_str,
+        )
+        mkt_mode = pair.venue
+        client.mkt_mode = mkt_mode
+
+        dst: Asset | None = assets.get(pair.bs_dst_asset)
+        src: Asset | None = assets.get(pair.bs_src_asset)
+
+        mkt = MktPair(
+            dst=dst,
+            src=src,
+            price_tick=pair.price_tick,
+            size_tick=pair.size_tick,
+            bs_mktid=pair.symbol,
+            venue=mkt_mode,
+            broker='deribit',
+            _atype=mkt_mode,
+            _fqme_without_src=True,
+
+            # expiry=pair.expiry,
+            # XXX TODO, currently we don't use it since it's
+            # already "described" in the `OptionPair.symbol: str`
+            # and if we slap in the ISO repr it's kinda hideous..
+            # -[ ] figure out the best either std
+        )
+        return mkt, pair
 
 
 async def stream_quotes(
-
     send_chan: trio.abc.SendChannel,
     symbols: list[str],
     feed_is_live: trio.Event,
-    loglevel: str = None,
 
     # startup sync
     task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
 
 ) -> None:
-    # XXX: required to propagate ``tractor`` loglevel to piker logging
-    get_console_log(loglevel or tractor.current_actor().loglevel)
+    '''
+    Open a live quote stream for the market set defined by `symbols`.
 
-    sym = symbols[0]
+    Internally this starts a `cryptofeed.FeedHandler` inside an `asyncio`-side
+    task and relays through L1 and `Trade` msgs here to our `trio.Task`.
+
+    '''
+    sym = symbols[0].split('.')[0]
+    init_msgs: list[FeedInit] = []
+
+    # multiline nested `dict` formatter (since rn quote-msgs are
+    # just that).
+    pfmt: Callable[[str], str] = mk_repr(
+        # so we can see `deribit`'s delightfully mega-long bs fields..
+        maxstring=100,
+    )
 
     async with (
         open_cached_client('deribit') as client,
         send_chan as send_chan
     ):
+        mkt: MktPair
+        pair: Pair
+        mkt, pair = await get_mkt_info(sym)
 
-        init_msgs = {
-            # pass back token, and bool, signalling if we're the writer
-            # and that history has been written
-            sym: {
-                'symbol_info': {
-                    'asset_type': 'option',
-                    'price_tick_size': 0.0005
-                },
-                'shm_write_opts': {'sum_tick_vml': False},
-                'fqsn': sym,
-            },
-        }
+        # build out init msgs according to latest spec
+        init_msgs.append(
+            FeedInit(
+                mkt_info=mkt,
+            )
+        )
+        # build `cryptofeed` feed-handle
+        cf_sym: cryptofeed.Symbol = piker_sym_to_cb_sym(sym)
 
-        nsym = piker_sym_to_cb_sym(sym)
+        from_cf: tractor.to_asyncio.LinkedTaskChannel
+        async with maybe_open_price_feed(sym) as from_cf:
 
-        async with maybe_open_price_feed(sym) as stream:
+            # load the "last trades" summary
+            last_trades_res: cryptofeed.LastTradesResult = await client.last_trades(
+                cb_sym_to_deribit_inst(cf_sym),
+                count=1,
+            )
+            last_trades: list[Trade] = last_trades_res.trades
 
-            cache = await client.cache_symbols()
+            # TODO, do we even need this or will the above always
+            # work?
+            # if not last_trades:
+            #     await tractor.pause()
+            #     async for typ, quote in from_cf:
+            #         if typ == 'trade':
+            #             last_trade = Trade(**(quote['data']))
+            #             break
 
-            last_trades = (await client.last_trades(
-                cb_sym_to_deribit_inst(nsym), count=1)).trades
+            # else:
+            last_trade = Trade(
+                **(last_trades[0])
+            )
 
-            if len(last_trades) == 0:
-                last_trade = None
-                async for typ, quote in stream:
-                    if typ == 'trade':
-                        last_trade = Trade(**(quote['data']))
-                        break
-
-            else:
-                last_trade = Trade(**(last_trades[0]))
-
-            first_quote = {
+            first_quote: dict = {
                 'symbol': sym,
                 'last': last_trade.price,
                 'brokerd_ts': last_trade.timestamp,
@@ -158,13 +304,84 @@ async def stream_quotes(
                     'broker_ts': last_trade.timestamp
                 }]
             }
-            task_status.started((init_msgs,  first_quote))
+            task_status.started((
+                init_msgs,
+                first_quote,
+            ))
 
             feed_is_live.set()
 
-            async for typ, quote in stream:
-                topic = quote['symbol']
-                await send_chan.send({topic: quote})
+            # NOTE XXX, static for now!
+            # => since this only handles ONE mkt feed at a time we
+            # don't need a lookup table to map interleaved quotes
+            # from multiple possible mkt-pairs
+            topic: str = mkt.bs_fqme
+
+            # deliver until cancelled
+            async for typ, ref in from_cf:
+                match typ:
+                    case 'trade':
+                        trade: cryptofeed.types.Trade = ref
+
+                        # TODO, re-impl this according to teh ideal
+                        # fqme for opts that we choose!!
+                        bs_fqme: str = cb_sym_to_deribit_inst(
+                            str_to_cb_sym(trade.symbol)
+                        ).lower()
+
+                        piker_quote: dict = {
+                            'symbol': bs_fqme,
+                            'last': trade.price,
+                            'broker_ts': time.time(),
+                            # ^TODO, name this `brokerd/datad_ts` and
+                            # use `time.time_ns()` ??
+                            'ticks': [{
+                                'type': 'trade',
+                                'price': float(trade.price),
+                                'size': float(trade.amount),
+                                'broker_ts': trade.timestamp,
+                            }],
+                        }
+                        log.info(
+                            f'deribit {typ!r} quote for {sym!r}\n\n'
+                            f'{trade}\n\n'
+                            f'{pfmt(piker_quote)}\n'
+                        )
+
+                    case 'l1':
+                        book: cryptofeed.types.L1Book = ref
+
+                        # TODO, so this is where we can possibly change things
+                        # and instead lever the `MktPair.bs_fqme: str` output?
+                        bs_fqme: str = cb_sym_to_deribit_inst(
+                            str_to_cb_sym(book.symbol)
+                        ).lower()
+
+                        piker_quote: dict = {
+                            'symbol': bs_fqme,
+                            'ticks': [
+
+                                {'type': 'bid',
+                                 'price': float(book.bid_price),
+                                 'size': float(book.bid_size)},
+
+                                {'type': 'bsize',
+                                 'price': float(book.bid_price),
+                                 'size': float(book.bid_size),},
+
+                                {'type': 'ask',
+                                 'price': float(book.ask_price),
+                                 'size': float(book.ask_size),},
+
+                                {'type': 'asize',
+                                 'price': float(book.ask_price),
+                                 'size': float(book.ask_size),}
+                            ]
+                        }
+
+                await send_chan.send({
+                    topic: piker_quote,
+                })
 
 
 @tractor.context
@@ -174,12 +391,21 @@ async def open_symbol_search(
     async with open_cached_client('deribit') as client:
 
         # load all symbols locally for fast search
-        cache = await client.cache_symbols()
+        # cache = client._pairs
         await ctx.started()
 
         async with ctx.open_stream() as stream:
-
+            pattern: str
             async for pattern in stream:
-                # repack in dict form
-                await stream.send(
-                    await client.search_symbols(pattern))
+
+                # NOTE: pattern fuzzy-matching is done within
+                # the methd impl.
+                pairs: dict[str, Pair] = await client.search_symbols(
+                    pattern,
+                )
+                # repack in fqme-keyed table
+                byfqme: dict[str, Pair] = {}
+                for pair in pairs.values():
+                    byfqme[pair.bs_fqme] = pair
+
+                await stream.send(byfqme)

piker/brokers/deribit/venues.py

@@ -0,0 +1,196 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+Per market data-type definitions and schemas types.
+
+"""
+from __future__ import annotations
+import pendulum
+from typing import (
+    Literal,
+    Optional,
+)
+from decimal import Decimal
+
+from piker.types import Struct
+
+
+# API endpoint paths by venue / sub-API
+_domain: str = 'deribit.com'
+_url = f'https://www.{_domain}'
+
+# WEBsocketz
+_ws_url: str = f'wss://www.{_domain}/ws/api/v2'
+
+# test nets
+_testnet_ws_url: str = f'wss://test.{_domain}/ws/api/v2'
+
+MarketType = Literal[
+    'option'
+]
+
+
+def get_api_eps(venue: MarketType) -> tuple[str, str]:
+    '''
+    Return API ep root paths per venue.
+
+    '''
+    return {
+        'option': (
+            _ws_url,
+        ),
+    }[venue]
+
+
+class Pair(Struct, frozen=True, kw_only=True):
+
+    symbol: str
+
+    # src
+    quote_currency: str # 'BTC'
+
+    # dst
+    base_currency: str # "BTC",
+
+    tick_size: float # 0.0001 # [{'above_price': 0.005, 'tick_size': 0.0005}]
+    tick_size_steps: list[dict[str, float]] 
+
+    @property
+    def price_tick(self) -> Decimal:
+        return Decimal(str(self.tick_size_steps[0]['above_price']))
+
+    @property
+    def size_tick(self) -> Decimal:
+        return Decimal(str(self.tick_size))
+
+    @property
+    def bs_fqme(self) -> str:
+        return f'{self.symbol}'
+
+    @property
+    def bs_mktid(self) -> str:
+        return f'{self.symbol}.{self.venue}'
+
+
+class OptionPair(Pair, frozen=True):
+
+    taker_commission: float # 0.0003
+    strike: float # 5000.0
+    settlement_period: str # 'day'
+    settlement_currency: str # "BTC",
+    rfq: bool # false
+    price_index: str # 'btc_usd'
+    option_type: str # 'call'
+    min_trade_amount: float # 0.1
+    maker_commission: float # 0.0003
+    kind: str # 'option'
+    is_active: bool # true
+    instrument_type: str # 'reversed'
+    instrument_name: str # 'BTC-1SEP24-55000-C'
+    instrument_id: int # 364671
+    expiration_timestamp: int # 1725177600000
+    creation_timestamp: int # 1724918461000
+    counter_currency: str # 'USD' 
+    contract_size: float # '1.0'
+    block_trade_tick_size: float # '0.0001'
+    block_trade_min_trade_amount: int # '25'
+    block_trade_commission: float # '0.003'
+
+    # NOTE: see `.data._symcache.SymbologyCache.load()` for why
+    ns_path: str = 'piker.brokers.deribit:OptionPair'
+
+    # TODO, impl this without the MM:SS part of
+    # the `'THH:MM:SS..'` etc..
+    @property
+    def expiry(self) -> str:
+        iso_date = pendulum.from_timestamp(
+            self.expiration_timestamp / 1000
+        ).isoformat()
+        return iso_date 
+
+    @property
+    def venue(self) -> str:
+        return f'{self.instrument_type}_option'
+
+    @property
+    def bs_fqme(self) -> str:
+        return f'{self.symbol}'
+
+    @property
+    def bs_src_asset(self) -> str:
+        return f'{self.quote_currency}'
+
+    @property
+    def bs_dst_asset(self) -> str:
+        return f'{self.symbol}'
+
+
+PAIRTYPES: dict[MarketType, Pair] = {
+    'option': OptionPair,
+}
+
+
+class JSONRPCResult(Struct):
+    id: int
+    usIn: int
+    usOut: int
+    usDiff: int
+    testnet: bool
+    jsonrpc: str = '2.0'
+    error: Optional[dict] = None
+    result: Optional[list[dict]] = None
+
+
+class JSONRPCChannel(Struct):
+    method: str
+    params: dict
+    jsonrpc: str = '2.0'
+
+
+class KLinesResult(Struct):
+    low: list[float]
+    cost: list[float]
+    high: list[float]
+    open: list[float]
+    close: list[float]
+    ticks: list[int]
+    status: str
+    volume: list[float]
+
+
+class Trade(Struct):
+    iv: float
+    price: float
+    amount: float
+    trade_id: str
+    contracts: float
+    direction: str
+    trade_seq: int
+    timestamp: int
+    mark_price: float
+    index_price: float
+    tick_direction: int
+    instrument_name: str
+    combo_id: Optional[str] = '',
+    combo_trade_id: Optional[int] = 0,
+    block_trade_id: Optional[str] = '',
+    block_trade_leg_count: Optional[int] = 0,
+
+
+class LastTradesResult(Struct):
+    trades: list[Trade]
+    has_more: bool

piker/data/_web_bs.py

@@ -273,7 +273,7 @@ async def _reconnect_forever(
                 nobsws._connected.set()
                 await trio.sleep_forever()
         except HandshakeError:
-            log.exception(f'Retrying connection')
+            log.exception('Retrying connection')
 
         # ws & nursery block ends
 
@@ -359,8 +359,8 @@ async def open_autorecon_ws(
 
 
 '''
-JSONRPC response-request style machinery for transparent multiplexing of msgs
-over a NoBsWs.
+JSONRPC response-request style machinery for transparent multiplexing
+of msgs over a `NoBsWs`.
 
 '''
 
@@ -377,43 +377,82 @@ async def open_jsonrpc_session(
     url: str,
     start_id: int = 0,
     response_type: type = JSONRPCResult,
-    request_type: Optional[type] = None,
-    request_hook: Optional[Callable] = None,
-    error_hook: Optional[Callable] = None,
+    msg_recv_timeout: float = float('inf'),
+    # ^NOTE, since only `deribit` is using this jsonrpc stuff atm
+    # and options mkts are generally "slow moving"..
+    #
+    # FURTHER if we break the underlying ws connection then since we
+    # don't pass a `fixture` to the task that manages `NoBsWs`, i.e.
+    # `_reconnect_forever()`, the jsonrpc "transport pipe" get's
+    # broken and never restored with wtv init sequence is required to
+    # re-establish a working req-resp session.
+
 ) -> Callable[[str, dict], dict]:
+    '''
+    Init a json-RPC-over-websocket connection to the provided `url`.
+
+    A `json_rpc: Callable[[str, dict], dict` is delivered to the
+    caller for sending requests and a bg-`trio.Task` handles
+    processing of response msgs including error reporting/raising in
+    the parent/caller task.
+
+    '''
+    # NOTE, store all request msgs so we can raise errors on the
+    # caller side!
+    req_msgs: dict[int, dict] = {}
 
     async with (
-        trio.open_nursery() as n,
-        open_autorecon_ws(url) as ws
+        trio.open_nursery() as tn,
+        open_autorecon_ws(
+            url=url,
+            msg_recv_timeout=msg_recv_timeout,
+        ) as ws
     ):
-        rpc_id: Iterable = count(start_id)
+        rpc_id: Iterable[int] = count(start_id)
         rpc_results: dict[int, dict] = {}
 
-        async def json_rpc(method: str, params: dict) -> dict:
+        async def json_rpc(
+            method: str,
+            params: dict,
+        ) -> dict:
             '''
             perform a json rpc call and wait for the result, raise exception in
             case of error field present on response
             '''
+            nonlocal req_msgs
+
+            req_id: int = next(rpc_id)
             msg = {
                 'jsonrpc': '2.0',
-                'id': next(rpc_id),
+                'id': req_id,
                 'method': method,
                 'params': params
             }
             _id = msg['id']
 
-            rpc_results[_id] = {
+            result = rpc_results[_id] = {
                 'result': None,
-                'event': trio.Event()
+                'error': None,
+                'event': trio.Event(),  # signal caller resp arrived
             }
+            req_msgs[_id] = msg
 
             await ws.send_msg(msg)
 
+            # wait for reponse before unblocking requester code
             await rpc_results[_id]['event'].wait()
 
-            ret = rpc_results[_id]['result']
+            if (maybe_result := result['result']):
+                ret = maybe_result
+                del rpc_results[_id]
 
-            del rpc_results[_id]
+            else:
+                err = result['error']
+                raise Exception(
+                    f'JSONRPC request failed\n'
+                    f'req: {msg}\n'
+                    f'resp: {err}\n'
+                )
 
             if ret.error is not None:
                 raise Exception(json.dumps(ret.error, indent=4))
@@ -428,6 +467,7 @@ async def open_jsonrpc_session(
             the server side.
 
             '''
+            nonlocal req_msgs
             async for msg in ws:
                 match msg:
                     case {
@@ -451,19 +491,28 @@ async def open_jsonrpc_session(
                         'params': _,
                     }:
                         log.debug(f'Recieved\n{msg}')
-                        if request_hook:
-                            await request_hook(request_type(**msg))
 
                     case {
                         'error': error
                     }:
-                        log.warning(f'Recieved\n{error}')
-                        if error_hook:
-                            await error_hook(response_type(**msg))
+                        # retreive orig request msg, set error
+                        # response in original "result" msg,
+                        # THEN FINALLY set the event to signal caller
+                        # to raise the error in the parent task.
+                        req_id: int = error['id']
+                        req_msg: dict = req_msgs[req_id]
+                        result: dict = rpc_results[req_id]
+                        result['error'] = error
+                        result['event'].set()
+                        log.error(
+                            f'JSONRPC request failed\n'
+                            f'req: {req_msg}\n'
+                            f'resp: {error}\n'
+                        )
 
                     case _:
                         log.warning(f'Unhandled JSON-RPC msg!?\n{msg}')
 
-        n.start_soon(recv_task)
+        tn.start_soon(recv_task)
         yield json_rpc
-        n.cancel_scope.cancel()
+        tn.cancel_scope.cancel()

piker/log.py

@@ -18,7 +18,11 @@
 Log like a forester!
 """
 import logging
+import reprlib
 import json
+from typing import (
+    Callable,
+)
 
 import tractor
 from pygments import (
@@ -84,3 +88,27 @@ def colorize_json(
         # likeable styles: algol_nu, tango, monokai
         formatters.TerminalTrueColorFormatter(style=style)
     )
+
+
+def mk_repr(
+    **repr_kws,
+) -> Callable[[str], str]:
+    '''
+    Allocate and deliver a `repr.Repr` instance with provided input
+    settings using the std-lib's `reprlib` mod,
+     * https://docs.python.org/3/library/reprlib.html
+
+    ------ Ex. ------
+    An up to 6-layer-nested `dict` as multi-line:
+    - https://stackoverflow.com/a/79102479
+    - https://docs.python.org/3/library/reprlib.html#reprlib.Repr.maxlevel
+
+    '''
+    def_kws: dict[str, int] = dict(
+        indent=2,
+        maxlevel=6,  # recursion levels
+        maxstring=66,  # match editor line-len limit
+    )
+    def_kws |= repr_kws
+    reprr = reprlib.Repr(**def_kws)
+    return reprr.repr

piker/storage/cli.py

@@ -386,6 +386,8 @@ def ldshm(
             open_annot_ctl() as actl,
         ):
             shm_df: pl.DataFrame | None = None
+            tf2aids: dict[float, dict] = {}
+
             for (
                 shmfile,
                 shm,
@@ -526,16 +528,17 @@ def ldshm(
                             new_df,
                             step_gaps,
                         )
-
                         # last chance manual overwrites in REPL
-                        await tractor.pause()
+                        # await tractor.pause()
                         assert aids
+                        tf2aids[period_s] = aids
 
                 else:
                     # allow interaction even when no ts problems.
-                    await tractor.pause()
-                    # assert not diff
+                    assert not diff
 
+            await tractor.pause()
+            log.info('Exiting TSP shm anal-izer!')
 
             if shm_df is None:
                 log.error(

piker/storage/nativedb.py

@@ -161,7 +161,13 @@ class NativeStorageClient:
 
     def index_files(self):
         for path in self._datadir.iterdir():
-            if path.name in {'borked', 'expired',}:
+            if (
+                path.is_dir()
+                or
+                '.parquet' not in str(path)
+                # or
+                # path.name in {'borked', 'expired',}
+            ):
                 continue
 
             key: str = path.name.rstrip('.parquet')

piker/tsp/__init__.py

@@ -44,8 +44,10 @@ import trio
 from trio_typing import TaskStatus
 import tractor
 from pendulum import (
+    Interval,
     DateTime,
     Duration,
+    duration as mk_duration,
     from_timestamp,
 )
 import numpy as np
@@ -214,7 +216,8 @@ async def maybe_fill_null_segments(
         # pair, immediately stop backfilling?
         if (
             start_dt
-            and end_dt < start_dt
+            and
+            end_dt < start_dt
         ):
             await tractor.pause()
             break
@@ -262,6 +265,7 @@ async def maybe_fill_null_segments(
         except tractor.ContextCancelled:
             # log.exception
             await tractor.pause()
+            raise
 
     null_segs_detected.set()
     # RECHECK for more null-gaps
@@ -349,7 +353,7 @@ async def maybe_fill_null_segments(
 
 async def start_backfill(
     get_hist,
-    frame_types: dict[str, Duration] | None,
+    def_frame_duration: Duration,
     mod: ModuleType,
     mkt: MktPair,
     shm: ShmArray,
@@ -379,22 +383,23 @@ async def start_backfill(
         update_start_on_prepend: bool = False
         if backfill_until_dt is None:
 
-            # TODO: drop this right and just expose the backfill
-            # limits inside a [storage] section in conf.toml?
-            # when no tsdb "last datum" is provided, we just load
-            # some near-term history.
-            # periods = {
-            #     1: {'days': 1},
-            #     60: {'days': 14},
-            # }
-
-            # do a decently sized backfill and load it into storage.
+            # TODO: per-provider default history-durations?
+            # -[ ] inside the `open_history_client()` config allow
+            #    declaring the history duration limits instead of
+            #    guessing and/or applying the same limits to all?
+            #
+            # -[ ] allow declaring (default) per-provider backfill
+            #     limits inside a [storage] sub-section in conf.toml?
+            #
+            # NOTE, when no tsdb "last datum" is provided, we just
+            # load some near-term history by presuming a "decently
+            # large" 60s duration limit and a much shorter 1s range.
             periods = {
                 1: {'days': 2},
                 60: {'years': 6},
             }
             period_duration: int = periods[timeframe]
-            update_start_on_prepend = True
+            update_start_on_prepend: bool = True
 
             # NOTE: manually set the "latest" datetime which we intend to
             # backfill history "until" so as to adhere to the history
@@ -416,7 +421,6 @@ async def start_backfill(
                 f'backfill_until_dt: {backfill_until_dt}\n'
                 f'last_start_dt: {last_start_dt}\n'
             )
-
             try:
                 (
                     array,
@@ -426,71 +430,114 @@ async def start_backfill(
                     timeframe,
                     end_dt=last_start_dt,
                 )
-
             except NoData as _daterr:
-                # 3 cases:
-                # - frame in the middle of a legit venue gap
-                # - history actually began at the `last_start_dt`
-                # - some other unknown error (ib blocking the
-                #   history bc they don't want you seeing how they
-                #   cucked all the tinas..)
-                if dur := frame_types.get(timeframe):
-                    # decrement by a frame's worth of duration and
-                    # retry a few times.
-                    last_start_dt.subtract(
-                        seconds=dur.total_seconds()
+                orig_last_start_dt: datetime = last_start_dt
+                gap_report: str = (
+                    f'EMPTY FRAME for `end_dt: {last_start_dt}`?\n'
+                    f'{mod.name} -> tf@fqme: {timeframe}@{mkt.fqme}\n'
+                    f'last_start_dt: {orig_last_start_dt}\n\n'
+                    f'bf_until: {backfill_until_dt}\n'
+                )
+                # EMPTY FRAME signal with 3 (likely) causes:
+                #
+                # 1. range contains legit gap in venue history
+                # 2. history actually (edge case) **began** at the
+                #    value `last_start_dt`
+                # 3. some other unknown error (ib blocking the
+                #    history-query bc they don't want you seeing how
+                #    they cucked all the tinas.. like with options
+                #    hist)
+                #
+                if def_frame_duration:
+                    # decrement by a duration's (frame) worth of time
+                    # as maybe indicated by the backend to see if we
+                    # can get older data before this possible
+                    # "history gap".
+                    last_start_dt: datetime = last_start_dt.subtract(
+                        seconds=def_frame_duration.total_seconds()
                     )
-                    log.warning(
-                        f'{mod.name} -> EMPTY FRAME for end_dt?\n'
-                        f'tf@fqme: {timeframe}@{mkt.fqme}\n'
-                        'bf_until <- last_start_dt:\n'
-                        f'{backfill_until_dt} <- {last_start_dt}\n'
-                        f'Decrementing `end_dt` by {dur} and retry..\n'
+                    gap_report += (
+                        f'Decrementing `end_dt` and retrying with,\n'
+                        f'def_frame_duration: {def_frame_duration}\n'
+                        f'(new) last_start_dt: {last_start_dt}\n'
                     )
+                    log.warning(gap_report)
+                    # skip writing to shm/tsdb and try the next
+                    # duration's worth of prior history.
                     continue
 
-            # broker says there never was or is no more history to pull
-            except DataUnavailable:
-                log.warning(
-                    f'NO-MORE-DATA in range?\n'
-                    f'`{mod.name}` halted history:\n'
-                    f'tf@fqme: {timeframe}@{mkt.fqme}\n'
-                    'bf_until <- last_start_dt:\n'
-                    f'{backfill_until_dt} <- {last_start_dt}\n'
-                )
+                else:
+                    # await tractor.pause()
+                    raise DataUnavailable(gap_report)
 
-                # ugh, what's a better way?
-                # TODO: fwiw, we probably want a way to signal a throttle
-                # condition (eg. with ib) so that we can halt the
-                # request loop until the condition is resolved?
-                if timeframe > 1:
-                    await tractor.pause()
+            # broker says there never was or is no more history to pull
+            except DataUnavailable as due:
+                message: str = due.args[0]
+                log.warning(
+                    f'Provider {mod.name!r} halted backfill due to,\n\n'
+
+                    f'{message}\n'
+
+                    f'fqme: {mkt.fqme}\n'
+                    f'timeframe: {timeframe}\n'
+                    f'last_start_dt: {last_start_dt}\n'
+                    f'bf_until: {backfill_until_dt}\n'
+                )
+                # UGH: what's a better way?
+                # TODO: backends are responsible for being correct on
+                # this right!?
+                # -[ ] in the `ib` case we could maybe offer some way
+                #     to halt the request loop until the condition is
+                #     resolved or should the backend be entirely in
+                #     charge of solving such faults? yes, right?
                 return
 
+            time: np.ndarray = array['time']
             assert (
-                array['time'][0]
+                time[0]
                 ==
                 next_start_dt.timestamp()
             )
 
-            diff = last_start_dt - next_start_dt
-            frame_time_diff_s = diff.seconds
+            assert time[-1] == next_end_dt.timestamp()
+
+            expected_dur: Interval = last_start_dt - next_start_dt
 
             # frame's worth of sample-period-steps, in seconds
             frame_size_s: float = len(array) * timeframe
-            expected_frame_size_s: float = frame_size_s + timeframe
-            if frame_time_diff_s > expected_frame_size_s:
-
+            recv_frame_dur: Duration = (
+                from_timestamp(array[-1]['time'])
+                -
+                from_timestamp(array[0]['time'])
+            )
+            if (
+                (lt_frame := (recv_frame_dur < expected_dur))
+                or
+                (null_frame := (frame_size_s == 0))
+                # ^XXX, should NEVER hit now!
+            ):
                 # XXX: query result includes a start point prior to our
                 # expected "frame size" and thus is likely some kind of
                 # history gap (eg. market closed period, outage, etc.)
                 # so just report it to console for now.
+                if lt_frame:
+                    reason = 'Possible GAP (or first-datum)'
+                else:
+                    assert null_frame
+                    reason = 'NULL-FRAME'
+
+                missing_dur: Interval = expected_dur.end - recv_frame_dur.end
                 log.warning(
-                    'GAP DETECTED:\n'
-                    f'last_start_dt: {last_start_dt}\n'
-                    f'diff: {diff}\n'
-                    f'frame_time_diff_s: {frame_time_diff_s}\n'
+                    f'{timeframe}s-series {reason} detected!\n'
+                    f'fqme: {mkt.fqme}\n'
+                    f'last_start_dt: {last_start_dt}\n\n'
+                    f'recv interval: {recv_frame_dur}\n'
+                    f'expected interval: {expected_dur}\n\n'
+
+                    f'Missing duration of history of {missing_dur.in_words()!r}\n'
+                    f'{missing_dur}\n'
                 )
+                # await tractor.pause()
 
             to_push = diff_history(
                 array,
@@ -565,22 +612,27 @@ async def start_backfill(
             # long-term storage.
             if (
                 storage is not None
-                and write_tsdb
+                and
+                write_tsdb
             ):
                 log.info(
                     f'Writing {ln} frame to storage:\n'
                     f'{next_start_dt} -> {last_start_dt}'
                 )
 
-                # always drop the src asset token for
+                # NOTE, always drop the src asset token for
                 # non-currency-pair like market types (for now)
+                #
+                # THAT IS, for now our table key schema is NOT
+                # including the dst[/src] source asset token. SO,
+                # 'tsla.nasdaq.ib' over 'tsla/usd.nasdaq.ib' for
+                # historical reasons ONLY.
                 if mkt.dst.atype not in {
                     'crypto',
                     'crypto_currency',
                     'fiat',  # a "forex pair"
+                    'perpetual_future',  # stupid "perps" from cex land
                 }:
-                    # for now, our table key schema is not including
-                    # the dst[/src] source asset token.
                     col_sym_key: str = mkt.get_fqme(
                         delim_char='',
                         without_src=True,
@@ -685,7 +737,7 @@ async def back_load_from_tsdb(
         last_tsdb_dt
         and latest_start_dt
     ):
-        backfilled_size_s = (
+        backfilled_size_s: Duration = (
             latest_start_dt - last_tsdb_dt
         ).seconds
         # if the shm buffer len is not large enough to contain
@@ -908,6 +960,8 @@ async def tsdb_backfill(
             f'{pformat(config)}\n'
         )
 
+        # concurrently load the provider's most-recent-frame AND any
+        # pre-existing tsdb history already saved in `piker` storage.
         dt_eps: list[DateTime, DateTime] = []
         async with trio.open_nursery() as tn:
             tn.start_soon(
@@ -918,7 +972,6 @@ async def tsdb_backfill(
                 timeframe,
                 config,
             )
-
             tsdb_entry: tuple = await load_tsdb_hist(
                 storage,
                 mkt,
@@ -947,6 +1000,25 @@ async def tsdb_backfill(
                 mr_end_dt,
             ) = dt_eps
 
+            first_frame_dur_s: Duration = (mr_end_dt - mr_start_dt).seconds
+            calced_frame_size: Duration = mk_duration(
+                seconds=first_frame_dur_s,
+            )
+            # NOTE, attempt to use the backend declared default frame
+            # sizing (as allowed by their time-series query APIs) and
+            # if not provided try to construct a default from the
+            # first frame received above.
+            def_frame_durs: dict[
+                int,
+                Duration,
+            ]|None = config.get('frame_types', None)
+            if def_frame_durs:
+                def_frame_size: Duration = def_frame_durs[timeframe]
+                assert def_frame_size == calced_frame_size
+            else:
+                # use what we calced from first frame above.
+                def_frame_size = calced_frame_size
+
             # NOTE: when there's no offline data, there's 2 cases:
             # - data backend doesn't support timeframe/sample
             #   period (in which case `dt_eps` should be `None` and
@@ -977,7 +1049,7 @@ async def tsdb_backfill(
                     partial(
                         start_backfill,
                         get_hist=get_hist,
-                        frame_types=config.get('frame_types', None),
+                        def_frame_duration=def_frame_size,
                         mod=mod,
                         mkt=mkt,
                         shm=shm,

piker/tsp/_anal.py

@@ -616,6 +616,18 @@ def detect_price_gaps(
     # ])
     ...
 
+# TODO: probably just use the null_segs impl above?
+def detect_vlm_gaps(
+    df: pl.DataFrame,
+    col: str = 'volume',
+
+) -> pl.DataFrame:
+
+    vnull: pl.DataFrame = w_dts.filter(
+        pl.col(col) == 0
+    )
+    return vnull
+
 
 def dedupe(
     src_df: pl.DataFrame,
@@ -626,7 +638,6 @@ def dedupe(
 
 ) -> tuple[
     pl.DataFrame,  # with dts
-    pl.DataFrame,  # gaps
     pl.DataFrame,  # with deduplicated dts (aka gap/repeat removal)
     int,  # len diff between input and deduped
 ]:
@@ -639,19 +650,22 @@ def dedupe(
     '''
     wdts: pl.DataFrame = with_dts(src_df)
 
-    # maybe sort on any time field
-    if sort:
-        wdts = wdts.sort(by='time')
-        # TODO: detect out-of-order segments which were corrected!
-        # -[ ] report in log msg
-        # -[ ] possibly return segment sections which were moved?
+    deduped = wdts
 
     # remove duplicated datetime samples/sections
     deduped: pl.DataFrame = wdts.unique(
-        subset=['dt'],
+        # subset=['dt'],
+        subset=['time'],
         maintain_order=True,
     )
 
+    # maybe sort on any time field
+    if sort:
+        deduped = deduped.sort(by='time')
+        # TODO: detect out-of-order segments which were corrected!
+        # -[ ] report in log msg
+        # -[ ] possibly return segment sections which were moved?
+
     diff: int = (
         wdts.height
         -

pyproject.toml

@@ -18,22 +18,6 @@
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
-# ------ - ------
-
-[tool.ruff.lint]
-# https://docs.astral.sh/ruff/settings/#lint_ignore
-ignore = []
-
-# https://docs.astral.sh/ruff/settings/#lint_per-file-ignores
-"piker/ui/qt.py" = [
-  "E402",
-  'F401',  # unused imports (without __all__ or blah as blah)
-  # "F841", # unused variable rules
-]
-# ignore-init-module-imports = false
-
-# ------ - ------
-
 [project]
 name = "piker"
 version = "0.1.0a0dev0"
@@ -103,15 +87,23 @@ uis = [
     "pyqt6 >=6.7.0, <7.0.0",
     "pyqtgraph",
 
-    # ------ - ------
+    # for consideration,
+    # - 'visidata'
 
     # TODO: add an `--only daemon` group for running non-ui / pikerd
     # service tree in distributed mode B)
     # https://docs.astral.sh/uv/concepts/projects/dependencies/#optional-dependencies
-    # [project.optional-dependencies]
 ]
 
 [dependency-groups]
+# TODO: a toolset that makes debugging a `pikerd` service (tree) easy
+# to hack on directly using more or less the local env:
+# - xonsh + xxh
+# - rsyscall + pdbp
+# - actor runtime control console like BEAM/OTP
+#
+# console ehancements and eventually remote debugging extras/helpers.
+# use `uv --dev` to enable
 dev = [
     "pytest >=6.0.0, <7.0.0",
     "elasticsearch >=8.9.0, <9.0.0",
@@ -119,13 +111,7 @@ dev = [
     "prompt-toolkit ==3.0.40",
     "cython >=3.0.0, <4.0.0",
     "greenback >=1.1.1, <2.0.0",
-    # console ehancements and eventually remote debugging
-    # extras/helpers.
-    # TODO: add a toolset that makes debugging a `pikerd` service
-    # (tree) easy to hack on directly using more or less the local env:
-    # - xonsh + xxh
-    # - rsyscall + pdbp
-    # - actor runtime control console like BEAM/OTP
+    "ruff>=0.9.6",
 ]
 
 [project.scripts]

ruff.toml

@@ -0,0 +1,93 @@
+# from default `ruff.toml` @
+# https://docs.astral.sh/ruff/configuration/
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+]
+
+# Same as Black.
+line-length = 88
+indent-width = 4
+
+# Assume Python 3.9
+target-version = "py312"
+
+# ------ - ------
+# TODO, stop warnings around `anext()` builtin use?
+# tool.ruff.target-version = "py310"
+
+
+[lint]
+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E4", "E7", "E9", "F"]
+ignore = []
+ignore-init-module-imports = false
+
+[lint.per-file-ignores]
+"piker/ui/qt.py" = [
+  "E402",
+  'F401',  # unused imports (without __all__ or blah as blah)
+  # "F841", # unused variable rules
+]
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[format]
+# Use single quotes in `ruff format`.
+quote-style = "single"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"
+
+# Enable auto-formatting of code examples in docstrings. Markdown,
+# reStructuredText code/literal blocks and doctests are all supported.
+#
+# This is currently disabled by default, but it is planned for this
+# to be opt-out in the future.
+docstring-code-format = false
+
+# Set the line length limit used when formatting code snippets in
+# docstrings.
+#
+# This only has an effect when the `docstring-code-format` setting is
+# enabled.
+docstring-code-line-length = "dynamic"

uv.lock

@@ -708,6 +708,7 @@ dev = [
     { name = "greenback" },
     { name = "prompt-toolkit" },
     { name = "pytest" },
+    { name = "ruff" },
     { name = "xonsh" },
 ]
 
@@ -739,7 +740,7 @@ requires-dist = [
     { name = "tomli", specifier = ">=2.0.1,<3.0.0" },
     { name = "tomli-w", specifier = ">=1.0.0,<2.0.0" },
     { name = "tomlkit", git = "https://github.com/pikers/tomlkit.git?branch=piker_pin" },
-    { name = "tractor", directory = "../tractor" },
+    { name = "tractor", editable = "../tractor" },
     { name = "trio", specifier = ">=0.24,<0.25" },
     { name = "trio-util", specifier = ">=0.7.0,<0.8.0" },
     { name = "trio-websocket", specifier = ">=0.10.3,<0.11.0" },
@@ -754,6 +755,7 @@ dev = [
     { name = "greenback", specifier = ">=1.1.1,<2.0.0" },
     { name = "prompt-toolkit", specifier = "==3.0.40" },
     { name = "pytest", specifier = ">=6.0.0,<7.0.0" },
+    { name = "ruff", specifier = ">=0.9.6" },
     { name = "xonsh", specifier = ">=0.14.2,<0.15.0" },
 ]
 
@@ -1073,6 +1075,31 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/19/71/39c7c0d87f8d4e6c020a393182060eaefeeae6c01dab6a84ec346f2567df/rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90", size = 242424 },
 ]
 
+[[package]]
+name = "ruff"
+version = "0.9.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2a/e1/e265aba384343dd8ddd3083f5e33536cd17e1566c41453a5517b5dd443be/ruff-0.9.6.tar.gz", hash = "sha256:81761592f72b620ec8fa1068a6fd00e98a5ebee342a3642efd84454f3031dca9", size = 3639454 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/76/e3/3d2c022e687e18cf5d93d6bfa2722d46afc64eaa438c7fbbdd603b3597be/ruff-0.9.6-py3-none-linux_armv6l.whl", hash = "sha256:2f218f356dd2d995839f1941322ff021c72a492c470f0b26a34f844c29cdf5ba", size = 11714128 },
+    { url = "https://files.pythonhosted.org/packages/e1/22/aff073b70f95c052e5c58153cba735748c9e70107a77d03420d7850710a0/ruff-0.9.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b908ff4df65dad7b251c9968a2e4560836d8f5487c2f0cc238321ed951ea0504", size = 11682539 },
+    { url = "https://files.pythonhosted.org/packages/75/a7/f5b7390afd98a7918582a3d256cd3e78ba0a26165a467c1820084587cbf9/ruff-0.9.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b109c0ad2ececf42e75fa99dc4043ff72a357436bb171900714a9ea581ddef83", size = 11132512 },
+    { url = "https://files.pythonhosted.org/packages/a6/e3/45de13ef65047fea2e33f7e573d848206e15c715e5cd56095589a7733d04/ruff-0.9.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1de4367cca3dac99bcbd15c161404e849bb0bfd543664db39232648dc00112dc", size = 11929275 },
+    { url = "https://files.pythonhosted.org/packages/7d/f2/23d04cd6c43b2e641ab961ade8d0b5edb212ecebd112506188c91f2a6e6c/ruff-0.9.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac3ee4d7c2c92ddfdaedf0bf31b2b176fa7aa8950efc454628d477394d35638b", size = 11466502 },
+    { url = "https://files.pythonhosted.org/packages/b5/6f/3a8cf166f2d7f1627dd2201e6cbc4cb81f8b7d58099348f0c1ff7b733792/ruff-0.9.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5dc1edd1775270e6aa2386119aea692039781429f0be1e0949ea5884e011aa8e", size = 12676364 },
+    { url = "https://files.pythonhosted.org/packages/f5/c4/db52e2189983c70114ff2b7e3997e48c8318af44fe83e1ce9517570a50c6/ruff-0.9.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:4a091729086dffa4bd070aa5dab7e39cc6b9d62eb2bef8f3d91172d30d599666", size = 13335518 },
+    { url = "https://files.pythonhosted.org/packages/66/44/545f8a4d136830f08f4d24324e7db957c5374bf3a3f7a6c0bc7be4623a37/ruff-0.9.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1bbc6808bf7b15796cef0815e1dfb796fbd383e7dbd4334709642649625e7c5", size = 12823287 },
+    { url = "https://files.pythonhosted.org/packages/c5/26/8208ef9ee7431032c143649a9967c3ae1aae4257d95e6f8519f07309aa66/ruff-0.9.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:589d1d9f25b5754ff230dce914a174a7c951a85a4e9270613a2b74231fdac2f5", size = 14592374 },
+    { url = "https://files.pythonhosted.org/packages/31/70/e917781e55ff39c5b5208bda384fd397ffd76605e68544d71a7e40944945/ruff-0.9.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc61dd5131742e21103fbbdcad683a8813be0e3c204472d520d9a5021ca8b217", size = 12500173 },
+    { url = "https://files.pythonhosted.org/packages/84/f5/e4ddee07660f5a9622a9c2b639afd8f3104988dc4f6ba0b73ffacffa9a8c/ruff-0.9.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5e2d9126161d0357e5c8f30b0bd6168d2c3872372f14481136d13de9937f79b6", size = 11906555 },
+    { url = "https://files.pythonhosted.org/packages/f1/2b/6ff2fe383667075eef8656b9892e73dd9b119b5e3add51298628b87f6429/ruff-0.9.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:68660eab1a8e65babb5229a1f97b46e3120923757a68b5413d8561f8a85d4897", size = 11538958 },
+    { url = "https://files.pythonhosted.org/packages/3c/db/98e59e90de45d1eb46649151c10a062d5707b5b7f76f64eb1e29edf6ebb1/ruff-0.9.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c4cae6c4cc7b9b4017c71114115db0445b00a16de3bcde0946273e8392856f08", size = 12117247 },
+    { url = "https://files.pythonhosted.org/packages/ec/bc/54e38f6d219013a9204a5a2015c09e7a8c36cedcd50a4b01ac69a550b9d9/ruff-0.9.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:19f505b643228b417c1111a2a536424ddde0db4ef9023b9e04a46ed8a1cb4656", size = 12554647 },
+    { url = "https://files.pythonhosted.org/packages/a5/7d/7b461ab0e2404293c0627125bb70ac642c2e8d55bf590f6fce85f508f1b2/ruff-0.9.6-py3-none-win32.whl", hash = "sha256:194d8402bceef1b31164909540a597e0d913c0e4952015a5b40e28c146121b5d", size = 9949214 },
+    { url = "https://files.pythonhosted.org/packages/ee/30/c3cee10f915ed75a5c29c1e57311282d1a15855551a64795c1b2bbe5cf37/ruff-0.9.6-py3-none-win_amd64.whl", hash = "sha256:03482d5c09d90d4ee3f40d97578423698ad895c87314c4de39ed2af945633caa", size = 10999914 },
+    { url = "https://files.pythonhosted.org/packages/e8/a8/d71f44b93e3aa86ae232af1f2126ca7b95c0f515ec135462b3e1f351441c/ruff-0.9.6-py3-none-win_arm64.whl", hash = "sha256:0e2bb706a2be7ddfea4a4af918562fdc1bcb16df255e5fa595bbd800ce322a5a", size = 10177499 },
+]
+
 [[package]]
 name = "shellingham"
 version = "1.5.4"
@@ -1188,7 +1215,7 @@ source = { git = "https://github.com/pikers/tomlkit.git?branch=piker_pin#8e0239a
 [[package]]
 name = "tractor"
 version = "0.1.0a6.dev0"
-source = { directory = "../tractor" }
+source = { editable = "../tractor" }
 dependencies = [
     { name = "colorlog" },
     { name = "msgspec" },