piker/brokers/__init__.py

@@ -51,7 +51,6 @@ __brokers__: list[str] = [
     'ib',
     'kraken',
     'kucoin',
-    'deribit',
 
     # broken but used to work
     # 'questrade',
@@ -62,6 +61,7 @@ __brokers__: list[str] = [
     # wstrade
     # iex
 
+    # deribit
     # bitso
 ]
 

piker/brokers/deribit/__init__.py

@@ -25,7 +25,6 @@ from .api import (
     get_client,
 )
 from .feed import (
-    get_mkt_info,
     open_history_client,
     open_symbol_search,
     stream_quotes,
@@ -35,20 +34,15 @@ from .feed import (
     # open_trade_dialog,
     # norm_trade_records,
 # )
-from .venues import (
-    OptionPair,
-)
 
 log = get_logger(__name__)
 
 __all__ = [
     'get_client',
 #    'trades_dialogue',
-    'get_mkt_info',
     'open_history_client',
     'open_symbol_search',
     'stream_quotes',
-    'OptionPair',
 #    'norm_trade_records',
 ]
 

piker/brokers/deribit/feed.py

@@ -18,59 +18,38 @@
 Deribit backend.
 
 '''
-from __future__ import annotations
 from contextlib import asynccontextmanager as acm
 from datetime import datetime
-from typing import (
+from typing import Any, Optional, Callable
-    # Any,
-    # Optional,
-    Callable,
-)
-# from pprint import pformat
 import time
 
-import cryptofeed
 import trio
 from trio_typing import TaskStatus
-from pendulum import (
+import pendulum
-    from_timestamp,
+from rapidfuzz import process as fuzzy
-)
 import numpy as np
 import tractor
 
-from piker.accounting import (
+from piker.brokers import open_cached_client
-    Asset,
+from piker.log import get_logger, get_console_log
-    MktPair,
+from piker.data import ShmArray
-    unpack_fqme,
+from piker.brokers._util import (
-)
+    BrokerError,
-from piker.brokers import (
-    open_cached_client,
-    NoData,
     DataUnavailable,
 )
-from piker._cacheables import (
-    async_lifo_cache,
-)
-from piker.log import (
-    get_logger,
-    mk_repr,
-)
-from piker.data.validate import FeedInit
 
+from cryptofeed import FeedHandler
+from cryptofeed.defines import (
+    DERIBIT, L1_BOOK, TRADES, OPTION, CALL, PUT
+)
+from cryptofeed.symbols import Symbol
 
 from .api import (
-    Client,
+    Client, Trade,
-    # get_config,
+    get_config,
-    piker_sym_to_cb_sym,
+    str_to_cb_sym, piker_sym_to_cb_sym, cb_sym_to_deribit_inst,
-    cb_sym_to_deribit_inst,
-    str_to_cb_sym,
     maybe_open_price_feed
 )
-from .venues import (
-    Pair,
-    OptionPair,
-    Trade,
-)
 
 _spawn_kwargs = {
     'infect_asyncio': True,
@@ -85,215 +64,90 @@ async def open_history_client(
     mkt: MktPair,
 ) -> tuple[Callable, int]:
 
+    fnstrument: str = mkt.bs_fqme
     # TODO implement history getter for the new storage layer.
     async with open_cached_client('deribit') as client:
 
-        pair: OptionPair = client._pairs[mkt.dst.name]
-        # XXX NOTE, the cuckers use ms !!!
-        creation_time_s: int = pair.creation_timestamp/1000
-
         async def get_ohlc(
-            timeframe: float,
+            end_dt: Optional[datetime] = None,
-            end_dt: datetime | None = None,
+            start_dt: Optional[datetime] = None,
-            start_dt: datetime | None = None,
 
         ) -> tuple[
             np.ndarray,
             datetime,  # start
             datetime,  # end
         ]:
-            if timeframe != 60:
-                raise DataUnavailable('Only 1m bars are supported')
 
-            array: np.ndarray = await client.bars(
+            array = await client.bars(
-                mkt,
+                instrument,
                 start_dt=start_dt,
                 end_dt=end_dt,
             )
             if len(array) == 0:
-                if (
+                raise DataUnavailable
-                    end_dt is None
-                ):
-                    raise DataUnavailable(
-                        'No history seems to exist yet?\n\n'
-                        f'{mkt}'
-                    )
-                elif (
-                    end_dt
-                    and
-                    end_dt.timestamp() < creation_time_s
-                ):
-                    # the contract can't have history
-                    # before it was created.
-                    pair_type_str: str = type(pair).__name__
-                    create_dt: datetime = from_timestamp(creation_time_s)
-                    raise DataUnavailable(
-                        f'No history prior to\n'
-                        f'`{pair_type_str}.creation_timestamp: int = '
-                        f'{pair.creation_timestamp}\n\n'
-                        f'------ deribit sux ------\n'
-                        f'WHICH IN "NORMAL PEOPLE WHO USE EPOCH TIME" form is,\n'
-                        f'creation_time_s: {creation_time_s}\n'
-                        f'create_dt: {create_dt}\n'
-                    )
-                raise NoData(
-                    f'No frame for {start_dt} -> {end_dt}\n'
-                )
 
-            start_dt = from_timestamp(array[0]['time'])
+            start_dt = pendulum.from_timestamp(array[0]['time'])
-            end_dt = from_timestamp(array[-1]['time'])
+            end_dt = pendulum.from_timestamp(array[-1]['time'])
-
-            times = array['time']
-            if not times.any():
-                raise ValueError(
-                    'Bad frame with null-times?\n\n'
-                    f'{times}'
-                )
-
-            if end_dt is None:
-                inow: int = round(time.time())
-                if (inow - times[-1]) > 60:
-                    await tractor.pause()
 
             return array, start_dt, end_dt
 
-        yield (
+        yield get_ohlc, {'erlangs': 3, 'rate': 3}
-            get_ohlc,
-            {  # backfill config
-                'erlangs': 3,
-                'rate': 3,
-            }
-        )
-
-
-@async_lifo_cache()
-async def get_mkt_info(
-    fqme: str,
-
-) -> tuple[MktPair, Pair|OptionPair] | None:
-
-    # uppercase since kraken bs_mktid is always upper
-    if 'deribit' not in fqme.lower():
-        fqme += '.deribit'
-
-    mkt_mode: str = ''
-    broker, mkt_ep, venue, expiry = unpack_fqme(fqme)
-
-    # NOTE: we always upper case all tokens to be consistent with
-    # binance's symbology style for pairs, like `BTCUSDT`, but in
-    # theory we could also just keep things lower case; as long as
-    # we're consistent and the symcache matches whatever this func
-    # returns, always!
-    expiry: str = expiry.upper()
-    venue: str = venue.upper()
-    # venue_lower: str = venue.lower()
-
-    mkt_mode: str = 'option'
-
-    async with open_cached_client(
-        'deribit',
-    ) as client:
-
-        assets: dict[str, Asset] = await client.get_assets()
-        pair_str: str = mkt_ep.lower()
-
-        pair: Pair = await client.exch_info(
-            sym=pair_str,
-        )
-        mkt_mode = pair.venue
-        client.mkt_mode = mkt_mode
-
-        dst: Asset | None = assets.get(pair.bs_dst_asset)
-        src: Asset | None = assets.get(pair.bs_src_asset)
-
-        mkt = MktPair(
-            dst=dst,
-            src=src,
-            price_tick=pair.price_tick,
-            size_tick=pair.size_tick,
-            bs_mktid=pair.symbol,
-            venue=mkt_mode,
-            broker='deribit',
-            _atype=mkt_mode,
-            _fqme_without_src=True,
-
-            # expiry=pair.expiry,
-            # XXX TODO, currently we don't use it since it's
-            # already "described" in the `OptionPair.symbol: str`
-            # and if we slap in the ISO repr it's kinda hideous..
-            # -[ ] figure out the best either std
-        )
-        return mkt, pair
 
 
 async def stream_quotes(
+
     send_chan: trio.abc.SendChannel,
     symbols: list[str],
     feed_is_live: trio.Event,
+    loglevel: str = None,
 
     # startup sync
     task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
 
 ) -> None:
-    '''
+    # XXX: required to propagate ``tractor`` loglevel to piker logging
-    Open a live quote stream for the market set defined by `symbols`.
+    get_console_log(loglevel or tractor.current_actor().loglevel)
 
-    Internally this starts a `cryptofeed.FeedHandler` inside an `asyncio`-side
+    sym = symbols[0]
-    task and relays through L1 and `Trade` msgs here to our `trio.Task`.
-
-    '''
-    sym = symbols[0].split('.')[0]
-    init_msgs: list[FeedInit] = []
-
-    # multiline nested `dict` formatter (since rn quote-msgs are
-    # just that).
-    pfmt: Callable[[str], str] = mk_repr(
-        # so we can see `deribit`'s delightfully mega-long bs fields..
-        maxstring=100,
-    )
 
     async with (
         open_cached_client('deribit') as client,
         send_chan as send_chan
     ):
-        mkt: MktPair
-        pair: Pair
-        mkt, pair = await get_mkt_info(sym)
 
-        # build out init msgs according to latest spec
+        init_msgs = {
-        init_msgs.append(
+            # pass back token, and bool, signalling if we're the writer
-            FeedInit(
+            # and that history has been written
-                mkt_info=mkt,
+            sym: {
-            )
+                'symbol_info': {
-        )
+                    'asset_type': 'option',
-        # build `cryptofeed` feed-handle
+                    'price_tick_size': 0.0005
-        cf_sym: cryptofeed.Symbol = piker_sym_to_cb_sym(sym)
+                },
+                'shm_write_opts': {'sum_tick_vml': False},
+                'fqsn': sym,
+            },
+        }
 
-        from_cf: tractor.to_asyncio.LinkedTaskChannel
+        nsym = piker_sym_to_cb_sym(sym)
-        async with maybe_open_price_feed(sym) as from_cf:
 
-            # load the "last trades" summary
+        async with maybe_open_price_feed(sym) as stream:
-            last_trades_res: cryptofeed.LastTradesResult = await client.last_trades(
-                cb_sym_to_deribit_inst(cf_sym),
-                count=1,
-            )
-            last_trades: list[Trade] = last_trades_res.trades
 
-            # TODO, do we even need this or will the above always
+            cache = await client.cache_symbols()
-            # work?
-            # if not last_trades:
-            #     await tractor.pause()
-            #     async for typ, quote in from_cf:
-            #         if typ == 'trade':
-            #             last_trade = Trade(**(quote['data']))
-            #             break
 
-            # else:
+            last_trades = (await client.last_trades(
-            last_trade = Trade(
+                cb_sym_to_deribit_inst(nsym), count=1)).trades
-                **(last_trades[0])
-            )
 
-            first_quote: dict = {
+            if len(last_trades) == 0:
+                last_trade = None
+                async for typ, quote in stream:
+                    if typ == 'trade':
+                        last_trade = Trade(**(quote['data']))
+                        break
+
+            else:
+                last_trade = Trade(**(last_trades[0]))
+
+            first_quote = {
                 'symbol': sym,
                 'last': last_trade.price,
                 'brokerd_ts': last_trade.timestamp,
@@ -304,84 +158,13 @@ async def stream_quotes(
                     'broker_ts': last_trade.timestamp
                 }]
             }
-            task_status.started((
+            task_status.started((init_msgs,  first_quote))
-                init_msgs,
-                first_quote,
-            ))
 
             feed_is_live.set()
 
-            # NOTE XXX, static for now!
+            async for typ, quote in stream:
-            # => since this only handles ONE mkt feed at a time we
+                topic = quote['symbol']
-            # don't need a lookup table to map interleaved quotes
+                await send_chan.send({topic: quote})
-            # from multiple possible mkt-pairs
-            topic: str = mkt.bs_fqme
-
-            # deliver until cancelled
-            async for typ, ref in from_cf:
-                match typ:
-                    case 'trade':
-                        trade: cryptofeed.types.Trade = ref
-
-                        # TODO, re-impl this according to teh ideal
-                        # fqme for opts that we choose!!
-                        bs_fqme: str = cb_sym_to_deribit_inst(
-                            str_to_cb_sym(trade.symbol)
-                        ).lower()
-
-                        piker_quote: dict = {
-                            'symbol': bs_fqme,
-                            'last': trade.price,
-                            'broker_ts': time.time(),
-                            # ^TODO, name this `brokerd/datad_ts` and
-                            # use `time.time_ns()` ??
-                            'ticks': [{
-                                'type': 'trade',
-                                'price': float(trade.price),
-                                'size': float(trade.amount),
-                                'broker_ts': trade.timestamp,
-                            }],
-                        }
-                        log.info(
-                            f'deribit {typ!r} quote for {sym!r}\n\n'
-                            f'{trade}\n\n'
-                            f'{pfmt(piker_quote)}\n'
-                        )
-
-                    case 'l1':
-                        book: cryptofeed.types.L1Book = ref
-
-                        # TODO, so this is where we can possibly change things
-                        # and instead lever the `MktPair.bs_fqme: str` output?
-                        bs_fqme: str = cb_sym_to_deribit_inst(
-                            str_to_cb_sym(book.symbol)
-                        ).lower()
-
-                        piker_quote: dict = {
-                            'symbol': bs_fqme,
-                            'ticks': [
-
-                                {'type': 'bid',
-                                 'price': float(book.bid_price),
-                                 'size': float(book.bid_size)},
-
-                                {'type': 'bsize',
-                                 'price': float(book.bid_price),
-                                 'size': float(book.bid_size),},
-
-                                {'type': 'ask',
-                                 'price': float(book.ask_price),
-                                 'size': float(book.ask_size),},
-
-                                {'type': 'asize',
-                                 'price': float(book.ask_price),
-                                 'size': float(book.ask_size),}
-                            ]
-                        }
-
-                await send_chan.send({
-                    topic: piker_quote,
-                })
 
 
 @tractor.context
@@ -391,21 +174,12 @@ async def open_symbol_search(
     async with open_cached_client('deribit') as client:
 
         # load all symbols locally for fast search
-        # cache = client._pairs
+        cache = await client.cache_symbols()
         await ctx.started()
 
         async with ctx.open_stream() as stream:
-            pattern: str
+
             async for pattern in stream:
-
+                # repack in dict form
-                # NOTE: pattern fuzzy-matching is done within
+                await stream.send(
-                # the methd impl.
+                    await client.search_symbols(pattern))
-                pairs: dict[str, Pair] = await client.search_symbols(
-                    pattern,
-                )
-                # repack in fqme-keyed table
-                byfqme: dict[str, Pair] = {}
-                for pair in pairs.values():
-                    byfqme[pair.bs_fqme] = pair
-
-                await stream.send(byfqme)

piker/brokers/deribit/venues.py

@@ -1,196 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for pikers)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-"""
-Per market data-type definitions and schemas types.
-
-"""
-from __future__ import annotations
-import pendulum
-from typing import (
-    Literal,
-    Optional,
-)
-from decimal import Decimal
-
-from piker.types import Struct
-
-
-# API endpoint paths by venue / sub-API
-_domain: str = 'deribit.com'
-_url = f'https://www.{_domain}'
-
-# WEBsocketz
-_ws_url: str = f'wss://www.{_domain}/ws/api/v2'
-
-# test nets
-_testnet_ws_url: str = f'wss://test.{_domain}/ws/api/v2'
-
-MarketType = Literal[
-    'option'
-]
-
-
-def get_api_eps(venue: MarketType) -> tuple[str, str]:
-    '''
-    Return API ep root paths per venue.
-
-    '''
-    return {
-        'option': (
-            _ws_url,
-        ),
-    }[venue]
-
-
-class Pair(Struct, frozen=True, kw_only=True):
-
-    symbol: str
-
-    # src
-    quote_currency: str # 'BTC'
-
-    # dst
-    base_currency: str # "BTC",
-
-    tick_size: float # 0.0001 # [{'above_price': 0.005, 'tick_size': 0.0005}]
-    tick_size_steps: list[dict[str, float]] 
-
-    @property
-    def price_tick(self) -> Decimal:
-        return Decimal(str(self.tick_size_steps[0]['above_price']))
-
-    @property
-    def size_tick(self) -> Decimal:
-        return Decimal(str(self.tick_size))
-
-    @property
-    def bs_fqme(self) -> str:
-        return f'{self.symbol}'
-
-    @property
-    def bs_mktid(self) -> str:
-        return f'{self.symbol}.{self.venue}'
-
-
-class OptionPair(Pair, frozen=True):
-
-    taker_commission: float # 0.0003
-    strike: float # 5000.0
-    settlement_period: str # 'day'
-    settlement_currency: str # "BTC",
-    rfq: bool # false
-    price_index: str # 'btc_usd'
-    option_type: str # 'call'
-    min_trade_amount: float # 0.1
-    maker_commission: float # 0.0003
-    kind: str # 'option'
-    is_active: bool # true
-    instrument_type: str # 'reversed'
-    instrument_name: str # 'BTC-1SEP24-55000-C'
-    instrument_id: int # 364671
-    expiration_timestamp: int # 1725177600000
-    creation_timestamp: int # 1724918461000
-    counter_currency: str # 'USD' 
-    contract_size: float # '1.0'
-    block_trade_tick_size: float # '0.0001'
-    block_trade_min_trade_amount: int # '25'
-    block_trade_commission: float # '0.003'
-
-    # NOTE: see `.data._symcache.SymbologyCache.load()` for why
-    ns_path: str = 'piker.brokers.deribit:OptionPair'
-
-    # TODO, impl this without the MM:SS part of
-    # the `'THH:MM:SS..'` etc..
-    @property
-    def expiry(self) -> str:
-        iso_date = pendulum.from_timestamp(
-            self.expiration_timestamp / 1000
-        ).isoformat()
-        return iso_date 
-
-    @property
-    def venue(self) -> str:
-        return f'{self.instrument_type}_option'
-
-    @property
-    def bs_fqme(self) -> str:
-        return f'{self.symbol}'
-
-    @property
-    def bs_src_asset(self) -> str:
-        return f'{self.quote_currency}'
-
-    @property
-    def bs_dst_asset(self) -> str:
-        return f'{self.symbol}'
-
-
-PAIRTYPES: dict[MarketType, Pair] = {
-    'option': OptionPair,
-}
-
-
-class JSONRPCResult(Struct):
-    id: int
-    usIn: int
-    usOut: int
-    usDiff: int
-    testnet: bool
-    jsonrpc: str = '2.0'
-    error: Optional[dict] = None
-    result: Optional[list[dict]] = None
-
-
-class JSONRPCChannel(Struct):
-    method: str
-    params: dict
-    jsonrpc: str = '2.0'
-
-
-class KLinesResult(Struct):
-    low: list[float]
-    cost: list[float]
-    high: list[float]
-    open: list[float]
-    close: list[float]
-    ticks: list[int]
-    status: str
-    volume: list[float]
-
-
-class Trade(Struct):
-    iv: float
-    price: float
-    amount: float
-    trade_id: str
-    contracts: float
-    direction: str
-    trade_seq: int
-    timestamp: int
-    mark_price: float
-    index_price: float
-    tick_direction: int
-    instrument_name: str
-    combo_id: Optional[str] = '',
-    combo_trade_id: Optional[int] = 0,
-    block_trade_id: Optional[str] = '',
-    block_trade_leg_count: Optional[int] = 0,
-
-
-class LastTradesResult(Struct):
-    trades: list[Trade]
-    has_more: bool

piker/log.py

@@ -18,11 +18,7 @@
 Log like a forester!
 """
 import logging
-import reprlib
 import json
-from typing import (
-    Callable,
-)
 
 import tractor
 from pygments import (
@@ -88,27 +84,3 @@ def colorize_json(
         # likeable styles: algol_nu, tango, monokai
         formatters.TerminalTrueColorFormatter(style=style)
     )
-
-
-def mk_repr(
-    **repr_kws,
-) -> Callable[[str], str]:
-    '''
-    Allocate and deliver a `repr.Repr` instance with provided input
-    settings using the std-lib's `reprlib` mod,
-     * https://docs.python.org/3/library/reprlib.html
-
-    ------ Ex. ------
-    An up to 6-layer-nested `dict` as multi-line:
-    - https://stackoverflow.com/a/79102479
-    - https://docs.python.org/3/library/reprlib.html#reprlib.Repr.maxlevel
-
-    '''
-    def_kws: dict[str, int] = dict(
-        indent=2,
-        maxlevel=6,  # recursion levels
-        maxstring=66,  # match editor line-len limit
-    )
-    def_kws |= repr_kws
-    reprr = reprlib.Repr(**def_kws)
-    return reprr.repr