Nix fix for using `poetry run` instead `poetry shell`:

Since the last poetry update (2.0.0) the `poetry shell` command has been move out to a plugin, so now we use `poetry run`
this change broke the environment for the poetry to work.

Here are the missing deps that has been added for this to work aagain.

default.nix

@@ -1,46 +1,127 @@
 with (import <nixpkgs> {});
-with python312Packages;
 let
   glibStorePath = lib.getLib glib;
-  qtpyStorePath = lib.getLib qtpy;
-  pyqt6StorePath = lib.getLib pyqt6;
-  pyqt6SipStorePath = lib.getLib pyqt6-sip;
+  zstdStorePath = lib.getLib zstd;
+  dbusStorePath = lib.getLib dbus;
+  libGLStorePath = lib.getLib libGL;
+  freetypeStorePath = lib.getLib freetype;
   qt6baseStorePath = lib.getLib qt6.qtbase;
-  rapidfuzzStorePath = lib.getLib rapidfuzz;
-  qdarkstyleStorePath = lib.getLib qdarkstyle;
+  fontconfigStorePath = lib.getLib fontconfig;
+  libxkbcommonStorePath = lib.getLib libxkbcommon;
+  xcbutilcursorStorePath = lib.getLib xcb-util-cursor;
+
+  qtpyStorePath = lib.getLib python312Packages.qtpy;
+  pyqt6StorePath = lib.getLib python312Packages.pyqt6;
+  pyqt6SipStorePath = lib.getLib python312Packages.pyqt6-sip;
+  rapidfuzzStorePath = lib.getLib python312Packages.rapidfuzz;
+  qdarkstyleStorePath = lib.getLib python312Packages.qdarkstyle;
+
+  xorgLibX11StorePath = lib.getLib xorg.libX11;
+  xorgLibxcbStorePath = lib.getLib xorg.libxcb;
+  xorgxcbutilwmStorePath = lib.getLib xorg.xcbutilwm;
+  xorgxcbutilimageStorePath = lib.getLib xorg.xcbutilimage;
+  xorgxcbutilerrorsStorePath = lib.getLib xorg.xcbutilerrors;
+  xorgxcbutilkeysymsStorePath = lib.getLib xorg.xcbutilkeysyms;
+  xorgxcbutilrenderutilStorePath = lib.getLib xorg.xcbutilrenderutil;
 in
 stdenv.mkDerivation {
   name = "piker-qt6-poetry-shell";
   buildInputs = [
     # System requirements.
     glib
+    dbus
+    zstd
+    libGL
+    freetype
     qt6.qtbase
     libgcc.lib
+    fontconfig
+    libxkbcommon
+
+    # Xorg requirements
+    xcb-util-cursor
+    xorg.libxcb
+    xorg.libX11
+    xorg.xcbutilwm
+    xorg.xcbutilimage
+    xorg.xcbutilerrors
+    xorg.xcbutilkeysyms
+    xorg.xcbutilrenderutil
 
     # Python requirements.
     python312Full
-    poetry-core
-    qdarkstyle
-    rapidfuzz
-    pyqt6
-    qtpy
+    python312Packages.poetry-core
+    python312Packages.qdarkstyle
+    python312Packages.rapidfuzz
+    python312Packages.pyqt6
+    python312Packages.qtpy
   ];
   src = null;
   shellHook = ''
     set -e
 
-    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${libgcc.lib}/lib:${glibStorePath}/lib
-
     # Set the Qt plugin path
     # export QT_DEBUG_PLUGINS=1
+    
+    QTBASE_PATH="${qt6baseStorePath}/lib"
+    QT_PLUGIN_PATH="$QTBASE_PATH/qt-6/plugins"
+    QT_QPA_PLATFORM_PLUGIN_PATH="$QT_PLUGIN_PATH/platforms"
 
-    QTBASE_PATH="${qt6baseStorePath}"
-    echo "qtbase path:    $QTBASE_PATH"
-    echo ""
-    export QT_PLUGIN_PATH="$QTBASE_PATH/lib/qt-6/plugins"
-    export QT_QPA_PLATFORM_PLUGIN_PATH="$QT_PLUGIN_PATH/platforms"
-    echo "qt plugin path: $QT_PLUGIN_PATH"
-    echo ""
+    LIB_GCC_PATH="${libgcc.lib}/lib"
+    GLIB_PATH="${glibStorePath}/lib"
+    ZSTD_PATH="${zstdStorePath}/lib"
+    DBUS_PATH="${dbusStorePath}/lib"
+    LIBGL_PATH="${libGLStorePath}/lib"
+    FREETYPE_PATH="${freetypeStorePath}/lib"
+    FONTCONFIG_PATH="${fontconfigStorePath}/lib"
+    LIB_XKB_COMMON_PATH="${libxkbcommonStorePath}/lib"
+
+    XCB_UTIL_CURSOR_PATH="${xcbutilcursorStorePath}/lib"
+    XORG_LIB_X11_PATH="${xorgLibX11StorePath}/lib"
+    XORG_LIB_XCB_PATH="${xorgLibxcbStorePath}/lib"
+    XORG_XCB_UTIL_IMAGE_PATH="${xorgxcbutilimageStorePath}/lib"
+    XORG_XCB_UTIL_WM_PATH="${xorgxcbutilwmStorePath}/lib"
+    XORG_XCB_UTIL_RENDER_UTIL_PATH="${xorgxcbutilrenderutilStorePath}/lib"
+    XORG_XCB_UTIL_KEYSYMS_PATH="${xorgxcbutilkeysymsStorePath}/lib"
+    XORG_XCB_UTIL_ERRORS_PATH="${xorgxcbutilerrorsStorePath}/lib"
+
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$QTBASE_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$QT_PLUGIN_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$QT_QPA_PLATFORM_PLUGIN_PATH"
+
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$LIB_GCC_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$DBUS_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GLIB_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$ZSTD_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$LIBGL_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$FONTCONFIG_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$FREETYPE_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$LIB_XKB_COMMON_PATH"
+
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$XCB_UTIL_CURSOR_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$XORG_LIB_X11_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$XORG_LIB_XCB_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$XORG_XCB_UTIL_IMAGE_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$XORG_XCB_UTIL_WM_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$XORG_XCB_UTIL_RENDER_UTIL_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$XORG_XCB_UTIL_KEYSYMS_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$XORG_XCB_UTIL_ERRORS_PATH"
+
+    export LD_LIBRARY_PATH
+
+    RPDFUZZ_PATH="${rapidfuzzStorePath}/lib/python3.12/site-packages"
+    QDRKSTYLE_PATH="${qdarkstyleStorePath}/lib/python3.12/site-packages"
+    QTPY_PATH="${qtpyStorePath}/lib/python3.12/site-packages"
+    PYQT6_PATH="${pyqt6StorePath}/lib/python3.12/site-packages"
+    PYQT6_SIP_PATH="${pyqt6SipStorePath}/lib/python3.12/site-packages"
+
+    PATCH="$PATCH:$RPDFUZZ_PATH"
+    PATCH="$PATCH:$QDRKSTYLE_PATH"
+    PATCH="$PATCH:$QTPY_PATH"
+    PATCH="$PATCH:$PYQT6_PATH"
+    PATCH="$PATCH:$PYQT6_SIP_PATH"
+
+    export PATCH
 
     # Maybe create venv & install deps
     poetry install --with uis
@@ -48,28 +129,6 @@ stdenv.mkDerivation {
     # Use pyqt6 from System, patch activate script
     ACTIVATE_SCRIPT_PATH="$(poetry env info --path)/bin/activate"
 
-    export RPDFUZZ_PATH="${rapidfuzzStorePath}/lib/python3.12/site-packages"
-    export QDRKSTYLE_PATH="${qdarkstyleStorePath}/lib/python3.12/site-packages"
-    export QTPY_PATH="${qtpyStorePath}/lib/python3.12/site-packages"
-    export PYQT6_PATH="${pyqt6StorePath}/lib/python3.12/site-packages"
-    export PYQT6_SIP_PATH="${pyqt6SipStorePath}/lib/python3.12/site-packages"
-    echo "rapidfuzz at:   $RPDFUZZ_PATH"
-    echo "qdarkstyle at:  $QDRKSTYLE_PATH"
-    echo "qtpy at:        $QTPY_PATH" 
-    echo "pyqt6 at:       $PYQT6_PATH"
-    echo "pyqt6-sip at:   $PYQT6_SIP_PATH"
-    echo ""
-
-    PATCH="export PYTHONPATH=\""
-
-    PATCH="$PATCH\$RPDFUZZ_PATH"
-    PATCH="$PATCH:\$QDRKSTYLE_PATH"
-    PATCH="$PATCH:\$QTPY_PATH"
-    PATCH="$PATCH:\$PYQT6_PATH"
-    PATCH="$PATCH:\$PYQT6_SIP_PATH"
-
-    PATCH="$PATCH\""
-
     if grep -q "$PATCH" "$ACTIVATE_SCRIPT_PATH"; then
         echo "venv is already patched."
     else
@@ -77,6 +136,5 @@ stdenv.mkDerivation {
         sed -i "\$i$PATCH" $ACTIVATE_SCRIPT_PATH
     fi
 
-    poetry shell
   '';
 }

piker/brokers/deribit/api.py

@@ -55,9 +55,10 @@ from cryptofeed.defines import (
     OPTION, CALL, PUT
 )
 from cryptofeed.symbols import Symbol
-
-# types for managing the cb callbacks.
-# from cryptofeed.types import L1Book
+from cryptofeed.types import (
+    L1Book,
+    Trade,
+)
 from piker.brokers import SymbolNotFound
 from .venues import (
     _ws_url,
@@ -66,9 +67,7 @@ from .venues import (
     Pair,
     OptionPair,
     JSONRPCResult,
-    # JSONRPCChannel,
     KLinesResult,
-    # Trade,
     LastTradesResult,
 )
 from piker.accounting import (
@@ -98,9 +97,17 @@ _spawn_kwargs = {
 }
 
 
-# convert datetime obj timestamp to unixtime in milliseconds
-def deribit_timestamp(when) -> int:
-    return int((when.timestamp() * 1000) + (when.microsecond / 1000))
+def deribit_timestamp(when: datetime) -> int:
+    '''
+    Convert conventional epoch timestamp, in secs, to unixtime in
+    milliseconds.
+
+    '''
+    return int(
+        (when.timestamp() * 1000)
+        +
+        (when.microsecond / 1000)
+    )
 
 
 def str_to_cb_sym(name: str) -> Symbol:
@@ -155,11 +162,28 @@ def piker_sym_to_cb_sym(name: str) -> Symbol:
     )
 
 
-def cb_sym_to_deribit_inst(sym: Symbol):
-    new_expiry_date = get_values_from_cb_normalized_date(sym.expiry_date)
-    otype = 'C' if sym.option_type == CALL else 'P'
+# TODO, instead can't we just lookup the `MktPair` directly
+# and pass it upward to `stream_quotes()`??
+def cb_sym_to_deribit_inst(sym: Symbol) -> str:
+    '''
+    Generate our own internal `str`-repr for a `cryptofeed.Symbol`
+    uniquely from its fields.
 
-    return f'{sym.base}-{new_expiry_date}-{sym.strike_price}-{otype}'
+    This is the equiv of generating a `Pair.fmqe` from `cryptofeed`
+    for now i suppose..?
+
+    '''
+    new_expiry_date = get_values_from_cb_normalized_date(sym.expiry_date)
+    otype = (
+        'C' if sym.option_type == CALL
+        else 'P'
+    )
+    return (
+        f'{sym.base}-'
+        f'{new_expiry_date}-'
+        f'{sym.strike_price}-'
+        f'{otype}'
+    )
 
 
 def get_values_from_cb_normalized_date(expiry_date: str) -> str:
@@ -598,7 +622,7 @@ async def get_client(
 
 
 @acm
-async def open_feed_handler():
+async def open_feed_handler() -> FeedHandler:
     fh = FeedHandler(config=get_config())
     yield fh
     await to_asyncio.run_task(fh.stop_async)
@@ -619,59 +643,37 @@ async def aio_price_feed_relay(
     from_trio: asyncio.Queue,
     to_trio: trio.abc.SendChannel,
 ) -> None:
+    '''
+    Relay price feed quotes from the `cryptofeed.FeedHandler` to
+    the `piker`-side `trio.task` consumers for delivery to consumer
+    sub-actors for various subsystems.
 
+    '''
     async def _trade(
-        data: dict,
+        trade: Trade,  # cryptofeed, NOT ours from `.venues`!
         receipt_timestamp: int,
     ) -> None:
         '''
-        Send `cryptofeed.FeedHandler` quotes to `piker`-side
-        `trio.Task`.
+        Proxy-thru `cryptofeed.FeedHandler` "trades" to `piker`-side.
 
         '''
-        to_trio.send_nowait((
-            'trade', {
-                'symbol': cb_sym_to_deribit_inst(
-                    str_to_cb_sym(data.symbol)).lower(),
-                'last': data,
-                'broker_ts': time.time(),
-                'data': data.to_dict(),
-                'receipt': receipt_timestamp,
-            },
-        ))
+        to_trio.send_nowait(('trade', trade))
 
     async def _l1(
-        data: dict,
+        book: L1Book,
         receipt_timestamp: int,
     ) -> None:
-        to_trio.send_nowait((
-            'l1', {
-               'symbol': cb_sym_to_deribit_inst(
-                    str_to_cb_sym(data.symbol)).lower(),
-                'ticks': [
-                    {
-                        'type': 'bid',
-                        'price': float(data.bid_price),
-                        'size': float(data.bid_size)
-                    },
-                    {
-                        'type': 'bsize',
-                        'price': float(data.bid_price),
-                        'size': float(data.bid_size)
-                    },
-                    {
-                        'type': 'ask',
-                        'price': float(data.ask_price),
-                        'size': float(data.ask_size)
-                    },
-                    {
-                        'type': 'asize',
-                        'price': float(data.ask_price),
-                        'size': float(data.ask_size)
-                    }
-                ]
-            },
-        ))
+        '''
+        Relay-thru "l1 book" updates.
+
+        '''
+
+        to_trio.send_nowait(('l1', book))
+
+        # TODO, make this work!
+        # -[ ] why isn't this working in `tractor.pause_from_sync()`??
+        # breakpoint()
+
     sym: Symbol = piker_sym_to_cb_sym(instrument)
     fh.add_feed(
         DERIBIT,
@@ -685,27 +687,35 @@ async def aio_price_feed_relay(
     if not fh.running:
         fh.run(
             start_loop=False,
-            install_signal_handlers=False)
+            install_signal_handlers=False
+        )
 
     # sync with trio
     to_trio.send_nowait(None)
 
+    # run until cancelled
     await asyncio.sleep(float('inf'))
 
 
 @acm
 async def open_price_feed(
     instrument: str
-) -> trio.abc.ReceiveStream:
-    async with maybe_open_feed_handler() as fh:
-        async with to_asyncio.open_channel_from(
+) -> to_asyncio.LinkedTaskChannel:
+
+    fh: FeedHandler
+    first: None
+    chan: to_asyncio.LinkedTaskChannel
+    async with (
+        maybe_open_feed_handler() as fh,
+        to_asyncio.open_channel_from(
             partial(
                 aio_price_feed_relay,
                 fh,
                 instrument
             )
-        ) as (first, chan):
-            yield chan
+        ) as (first, chan)
+    ):
+        yield chan
 
 
 @acm
@@ -714,6 +724,7 @@ async def maybe_open_price_feed(
 ) -> trio.abc.ReceiveStream:
 
     # TODO: add a predicate to maybe_open_context
+    feed: to_asyncio.LinkedTaskChannel
     async with maybe_open_context(
         acm_func=open_price_feed,
         kwargs={

piker/brokers/deribit/feed.py

@@ -63,6 +63,7 @@ from .api import (
     # get_config,
     piker_sym_to_cb_sym,
     cb_sym_to_deribit_inst,
+    str_to_cb_sym,
     maybe_open_price_feed
 )
 from .venues import (
@@ -237,13 +238,19 @@ async def stream_quotes(
     '''
     Open a live quote stream for the market set defined by `symbols`.
 
+    Internally this starts a `cryptofeed.FeedHandler` inside an `asyncio`-side
+    task and relays through L1 and `Trade` msgs here to our `trio.Task`.
+
     '''
     sym = symbols[0].split('.')[0]
     init_msgs: list[FeedInit] = []
 
     # multiline nested `dict` formatter (since rn quote-msgs are
     # just that).
-    pfmt: Callable[[str], str] = mk_repr()
+    pfmt: Callable[[str], str] = mk_repr(
+        # so we can see `deribit`'s delightfully mega-long bs fields..
+        maxstring=100,
+    )
 
     async with (
         open_cached_client('deribit') as client,
@@ -262,25 +269,31 @@ async def stream_quotes(
         # build `cryptofeed` feed-handle
         cf_sym: cryptofeed.Symbol = piker_sym_to_cb_sym(sym)
 
-        async with maybe_open_price_feed(sym) as stream:
-            last_trades = (
-                await client.last_trades(
-                    cb_sym_to_deribit_inst(cf_sym),
-                    count=1,
-                )
-            ).trades
+        from_cf: tractor.to_asyncio.LinkedTaskChannel
+        async with maybe_open_price_feed(sym) as from_cf:
 
-            if len(last_trades) == 0:
-                last_trade = None
-                async for typ, quote in stream:
-                    if typ == 'trade':
-                        last_trade = Trade(**(quote['data']))
-                        break
+            # load the "last trades" summary
+            last_trades_res: cryptofeed.LastTradesResult = await client.last_trades(
+                cb_sym_to_deribit_inst(cf_sym),
+                count=1,
+            )
+            last_trades: list[Trade] = last_trades_res.trades
 
-            else:
-                last_trade = Trade(**(last_trades[0]))
+            # TODO, do we even need this or will the above always
+            # work?
+            # if not last_trades:
+            #     await tractor.pause()
+            #     async for typ, quote in from_cf:
+            #         if typ == 'trade':
+            #             last_trade = Trade(**(quote['data']))
+            #             break
 
-            first_quote = {
+            # else:
+            last_trade = Trade(
+                **(last_trades[0])
+            )
+
+            first_quote: dict = {
                 'symbol': sym,
                 'last': last_trade.price,
                 'brokerd_ts': last_trade.timestamp,
@@ -305,14 +318,69 @@ async def stream_quotes(
             topic: str = mkt.bs_fqme
 
             # deliver until cancelled
-            async for typ, quote in stream:
-                sym: str = quote['symbol']
-                log.info(
-                    f'deribit {typ!r} quote for {sym!r}\n\n'
-                    f'{pfmt(quote)}\n'
-                )
+            async for typ, ref in from_cf:
+                match typ:
+                    case 'trade':
+                        trade: cryptofeed.types.Trade = ref
+
+                        # TODO, re-impl this according to teh ideal
+                        # fqme for opts that we choose!!
+                        bs_fqme: str = cb_sym_to_deribit_inst(
+                            str_to_cb_sym(trade.symbol)
+                        ).lower()
+
+                        piker_quote: dict = {
+                            'symbol': bs_fqme,
+                            'last': trade.price,
+                            'broker_ts': time.time(),
+                            # ^TODO, name this `brokerd/datad_ts` and
+                            # use `time.time_ns()` ??
+                            'ticks': [{
+                                'type': 'trade',
+                                'price': float(trade.price),
+                                'size': float(trade.amount),
+                                'broker_ts': trade.timestamp,
+                            }],
+                        }
+                        log.info(
+                            f'deribit {typ!r} quote for {sym!r}\n\n'
+                            f'{trade}\n\n'
+                            f'{pfmt(piker_quote)}\n'
+                        )
+
+                    case 'l1':
+                        book: cryptofeed.types.L1Book = ref
+
+                        # TODO, so this is where we can possibly change things
+                        # and instead lever the `MktPair.bs_fqme: str` output?
+                        bs_fqme: str = cb_sym_to_deribit_inst(
+                            str_to_cb_sym(book.symbol)
+                        ).lower()
+
+                        piker_quote: dict = {
+                            'symbol': bs_fqme,
+                            'ticks': [
+
+                                {'type': 'bid',
+                                 'price': float(book.bid_price),
+                                 'size': float(book.bid_size)},
+
+                                {'type': 'bsize',
+                                 'price': float(book.bid_price),
+                                 'size': float(book.bid_size),},
+
+                                {'type': 'ask',
+                                 'price': float(book.ask_price),
+                                 'size': float(book.ask_size),},
+
+                                {'type': 'asize',
+                                 'price': float(book.ask_price),
+                                 'size': float(book.ask_size),}
+                            ]
+                        }
+
                 await send_chan.send({
-                    topic: quote,
+                    topic: piker_quote,
                 })
 
 
@@ -327,7 +395,6 @@ async def open_symbol_search(
         await ctx.started()
 
         async with ctx.open_stream() as stream:
-
             pattern: str
             async for pattern in stream:
 

piker/brokers/deribit/venues.py

@@ -154,6 +154,7 @@ class JSONRPCResult(Struct):
     error: Optional[dict] = None
     result: Optional[list[dict]] = None
 
+
 class JSONRPCChannel(Struct):
     method: str
     params: dict
@@ -170,6 +171,7 @@ class KLinesResult(Struct):
     status: str
     volume: list[float]
 
+
 class Trade(Struct):
     iv: float
     price: float
@@ -188,6 +190,7 @@ class Trade(Struct):
     block_trade_id: Optional[str] = '',
     block_trade_leg_count: Optional[int] = 0,
 
+
 class LastTradesResult(Struct):
     trades: list[Trade]
     has_more: bool

pyproject.toml

@@ -130,6 +130,7 @@ ledger = 'piker.accounting.cli:ledger'
 
 
 [project]
+name="piker"
 keywords=[
   "async",
   "trading",