Merge pull request 'how_to_show_ur_pp: fixes for end-2-end order/position display'

Reviewed-on: https://www.pikers.dev/pikers/piker/pulls/60

README.rst

@@ -93,13 +93,13 @@ bc why install with `python` when you can faster with `rust` ::
     # ^ astral's docs,
     # https://docs.astral.sh/uv/concepts/projects/sync/
 
-include all GUIs ::
+include all GUIs (ex. for charting)::
 
     uv sync --extra uis
 
-AND with all our hacking tools::
+AND with all our hacking tools and WIP integrations::
 
-    uv sync --dev --extra uis
+    uv sync --dev --all-extras 
 
 
 Ensure you can run the root-daemon::
@@ -107,13 +107,21 @@ Ensure you can run the root-daemon::
     uv run pikerd [-l info --pdb]
 
 
-hacky install on nixos
-**********************
+install on nix(os)
+******************
 ``NixOS`` is our core devs' distro of choice for which we offer
-a stringently defined development shell envoirment that can be loaded with::
+a stringently defined development shell envoirment that can currently
+be applied in one of 2 ways::
 
+    # ONLY if running on X11
     nix-shell default.nix
 
+Or if you prefer flakes style and a modern DE::
+
+    # ONLY if also running on Wayland
+    nix develop  # for default bash
+    nix develop -c uv run xonsh  # for @goodboy's preferred sh B)
+
 
 start a chart
 *************

config/brokers.toml

@@ -1,6 +1,5 @@
-################
 # ---- CEXY ----
-################
+
 [binance]
 accounts.paper = 'paper'
 
@@ -13,28 +12,41 @@ accounts.spot = 'spot'
 spot.use_testnet = false
 spot.api_key = ''
 spot.api_secret = ''
+# ------ binance ------
 
 
 [deribit]
+# std assets
 key_id = ''
 key_secret = ''
+# options
+accounts.option = 'option'
+option.use_testnet = false
+option.key_id = ''
+option.key_secret = ''
+# aux logging from `cryptofeed`
+option.log.filename = 'cryptofeed.log'
+option.log.level = 'DEBUG'
+option.log.disabled = true
+# ------ deribit ------
 
 
 [kraken]
 key_descr = ''
 api_key = ''
 secret = ''
+# ------ kraken ------
 
 
 [kucoin]
 key_id = ''
 key_secret = ''
 key_passphrase = ''
+# ------ kucoin ------
 
 
-################
 # -- BROKERZ ---
-################
+
 [questrade]
 refresh_token = ''
 access_token = ''
@@ -42,44 +54,55 @@ api_server = 'https://api06.iq.questrade.com/'
 expires_in = 1800
 token_type = 'Bearer'
 expires_at = 1616095326.355846
+# ------ questrade ------
 
 
 [ib]
+# define the (set of) host-port socketaddrs that
+# brokerd.ib will scan to connect to an API endpoint
+# (ib-gw or ib-tws listening instances)
 hosts = [
     '127.0.0.1',
 ]
-# XXX: the order in which ports will be scanned
-# (by the `brokerd` daemon-actor)
-# is determined # by the line order here.
-# TODO: when we eventually spawn gateways in our
-# container, we can just dynamically allocate these
-# using IBC.
 ports = [
     4002,  # gw
     7497,  # tws
 ]
 
-# XXX: for a paper account the flex web query service
-# is not supported so you have to manually download
-# and XML report and put it in a location that can be
-# accessed by the ``brokerd.ib`` backend code for parsing.
-flex_token = ''
-flex_trades_query_id = ''  # live account
-
-# when clients are being scanned this determines
-# which clients are preferred to be used for data
-# feeds based on the order of account names, if
-# detected as active on an API client.
+# When API endpoints are being scanned durin startup, the order
+# of user-defined-account "names" (as defined below) here
+# determines which py-client connection is given priority to be
+# used for data-feed-requests by according to whichever client
+# connected to an API endpoing which reported the equivalent
+# account number for that name.
 prefer_data_account = [
     'paper',
     'margin',
     'ira',
 ]
 
+# For long-term trades txn (transaction) history
+# processing (i.e your txn ledger with IB) you can
+# (automatically for live accounts) query the FLEX
+# report system for past history.
+#
+# (For paper accounts the web query service
+# is not supported so you have to manually download
+# an XML report and put it in a location that can be
+# accessed by our `brokerd.ib` backend code for parsing).
+#
+flex_token = ''
+flex_trades_query_id = ''  # live account
+
+# define "aliases" (names) for each account number
+# such that the names can be reffed and logged throughout
+# `piker.accounting` subsys and more easily
+# referred to  by the user.
+#
+# These keys will be the set exposed through the order-mode
+# account-selection UI so that numbers are never shown.
 [ib.accounts]
-# the order in which accounts will be selectable
-# in the order mode UI (if found via clients during
-# API-app scanning)when a new symbol is loaded.
-paper = 'XX0000000'
-margin = 'X0000000'
-ira = 'X0000000'
+paper = 'DU0000000'  # <- literal account #
+margin = 'U0000000'
+ira = 'U0000000'
+# ------ ib ------

default.nix

@@ -11,11 +11,12 @@ let
   libxkbcommonStorePath = lib.getLib libxkbcommon;
   xcbutilcursorStorePath = lib.getLib xcb-util-cursor;
 
-  qtpyStorePath = lib.getLib python312Packages.qtpy;
-  pyqt6StorePath = lib.getLib python312Packages.pyqt6;
-  pyqt6SipStorePath = lib.getLib python312Packages.pyqt6-sip;
-  rapidfuzzStorePath = lib.getLib python312Packages.rapidfuzz;
-  qdarkstyleStorePath = lib.getLib python312Packages.qdarkstyle;
+  pypkgs = python313Packages;
+  qtpyStorePath = lib.getLib pypkgs.qtpy;
+  pyqt6StorePath = lib.getLib pypkgs.pyqt6;
+  pyqt6SipStorePath = lib.getLib pypkgs.pyqt6-sip;
+  rapidfuzzStorePath = lib.getLib pypkgs.rapidfuzz;
+  qdarkstyleStorePath = lib.getLib pypkgs.qdarkstyle;
 
   xorgLibX11StorePath = lib.getLib xorg.libX11;
   xorgLibxcbStorePath = lib.getLib xorg.libxcb;
@@ -51,12 +52,12 @@ stdenv.mkDerivation {
     xorg.xcbutilrenderutil
 
     # Python requirements.
-    python312Full
-    python312Packages.uv
-    python312Packages.qdarkstyle
-    python312Packages.rapidfuzz
-    python312Packages.pyqt6
-    python312Packages.qtpy
+    python313
+    uv
+    pypkgs.qdarkstyle
+    pypkgs.rapidfuzz
+    pypkgs.pyqt6
+    pypkgs.qtpy
   ];
   src = null;
   shellHook = ''
@@ -113,11 +114,11 @@ stdenv.mkDerivation {
 
     export LD_LIBRARY_PATH
 
-    RPDFUZZ_PATH="${rapidfuzzStorePath}/lib/python3.12/site-packages"
-    QDRKSTYLE_PATH="${qdarkstyleStorePath}/lib/python3.12/site-packages"
-    QTPY_PATH="${qtpyStorePath}/lib/python3.12/site-packages"
-    PYQT6_PATH="${pyqt6StorePath}/lib/python3.12/site-packages"
-    PYQT6_SIP_PATH="${pyqt6SipStorePath}/lib/python3.12/site-packages"
+    RPDFUZZ_PATH="${rapidfuzzStorePath}/lib/python3.13/site-packages"
+    QDRKSTYLE_PATH="${qdarkstyleStorePath}/lib/python3.13/site-packages"
+    QTPY_PATH="${qtpyStorePath}/lib/python3.13/site-packages"
+    PYQT6_PATH="${pyqt6StorePath}/lib/python3.13/site-packages"
+    PYQT6_SIP_PATH="${pyqt6SipStorePath}/lib/python3.13/site-packages"
 
     PATCH="$PATCH:$RPDFUZZ_PATH"
     PATCH="$PATCH:$QDRKSTYLE_PATH"
@@ -127,8 +128,8 @@ stdenv.mkDerivation {
 
     export PATCH
 
-    # Install deps
-    uv lock
+    # install all dev and extras
+    uv sync --dev --all-extras
 
   '';
 }

dockering/ib/README.rst

@@ -24,9 +24,8 @@ here is an example using ``vncclient`` on ``linux``::
 
     vncviewer localhost:5900
 
-
-now enter the pw you set via an (see second code blob) `.env file`_
-or pw-file according to the `credentials section`_.
+now enter the pw (password) you set via an (see second code blob)
+`.env file`_ or pw-file according to the `credentials section`_.
 
 If you want to change away from their default config see the example
 `docker-compose.yml`-config issue and config-section of the readme,
@@ -39,6 +38,74 @@ If you want to change away from their default config see the example
 .. _credentials section: https://github.com/gnzsnz/ib-gateway-docker?tab=readme-ov-file#credentials
 
 
+Connecting to the API from `piker`
+---------------------------------
+In order to expose the container's API endpoint to the
+`brokerd/datad/ib` actor, we need to add a section to the user's
+`brokers.toml` config (note the below is similar to the repo-shipped
+template file),
+
+.. code:: toml
+
+    [ib]
+    # define the (set of) host-port socketaddrs that
+    # brokerd.ib will scan to connect to an API endpoint
+    # (ib-gw or ib-tws listening instances)
+    hosts = [
+        '127.0.0.1',
+    ]
+    ports = [
+        4002,  # gw
+        7497,  # tws
+    ]
+
+    # When API endpoints are being scanned durin startup, the order
+    # of user-defined-account "names" (as defined below) here
+    # determines which py-client connection is given priority to be
+    # used for data-feed-requests by according to whichever client
+    # connected to an API endpoing which reported the equivalent
+    # account number for that name.
+    prefer_data_account = [
+        'paper',
+        'margin',
+        'ira',
+    ]
+
+    # define "aliases" (names) for each account number
+    # such that the names can be reffed and logged throughout
+    # `piker.accounting` subsys and more easily
+    # referred to  by the user.
+    #
+    # These keys will be the set exposed through the order-mode
+    # account-selection UI so that numbers are never shown.
+    [ib.accounts]
+    paper = 'XX0000000'
+    margin = 'X0000000'
+    ira = 'X0000000'
+
+
+the broker daemon can also connect to the container's VNC server for
+added functionalies including,
+
+- viewing the API endpoint program's GUI for manual interventions,
+- workarounds for historical data throttling using hotkey hacks,
+
+Add a further section to `brokers.toml` which maps each API-ep's
+port to a table of VNC server connection info like,
+
+.. code:: toml
+
+    [ib.vnc_addrs]
+    4002 = {host = 'localhost', port = 5900, pw = 'doggy'}
+
+The `pw = 'doggy'` here ^ should the same value as the particular
+container instances `.env` file setting (when it was run),
+
+.. code:: ini
+
+    VNC_SERVER_PASSWORD='doggy'
+
+
 IF you also want to run ``TWS``
 -------------------------------
 You can also run it containerized,

dockering/ib/docker-compose.yml

@@ -1,10 +1,15 @@
-# rework from the original @
-# https://github.com/waytrade/ib-gateway-docker/blob/master/docker-compose.yml
-version: "3.5"
-
+# a community maintained IB API container!
+#
+# https://github.com/gnzsnz/ib-gateway-docker
+#
+# For piker we (currently) include some minor deviations
+# for some config files in the `volumes` section.
+#
+# See full configuration settings @
+# - https://github.com/gnzsnz/ib-gateway-docker?tab=readme-ov-file#configuration
+# - https://github.com/gnzsnz/ib-gateway-docker/discussions/103
 
 services:
-
   ib_gw_paper:
 
     # apparently java is a mega cukc:
@@ -50,16 +55,22 @@ services:
         target: /root/scripts/run_x11_vnc.sh
         read_only: true
 
-    # NOTE:to fill these out, define an `.env` file in the same dir as
-    # this compose file which looks something like:
-    # TWS_USERID='myuser'
-    # TWS_PASSWORD='guest'
+    # NOTE: an alt method to fill these out is to
+    # define an `.env` file in the same dir as
+    # this compose file.
     environment:
       TWS_USERID: ${TWS_USERID}
+      # TWS_USERID: 'myuser'
       TWS_PASSWORD: ${TWS_PASSWORD}
-      TRADING_MODE: 'paper'
-      VNC_SERVER_PASSWORD: 'doggy'
-      VNC_SERVER_PORT: '3003'
+      # TWS_PASSWORD: 'guest'
+      TRADING_MODE: ${TRADING_MODE}
+      # TRADING_MODE: 'paper'
+      VNC_SERVER_PASSWORD: ${VNC_SERVER_PASSWORD}
+      # VNC_SERVER_PASSWORD: 'doggy'
+
+      # TODO, see if we can get this supported like it
+      # was on the old `waytrade` image?
+      # VNC_SERVER_PORT: '3003'
 
     # ports:
     #   - target: 4002
@@ -76,6 +87,9 @@ services:
       # - "127.0.0.1:4002:4002"
       # - "127.0.0.1:5900:5900"
 
+  # TODO, a masked but working example of dual paper + live
+  # ib-gw instances running in a single app run!
+  #
   # ib_gw_live:
   #   image: waytrade/ib-gateway:1012.2i
   #   restart: no

flake.lock

@@ -1,135 +1,24 @@
 {
   "nodes": {
-    "flake-utils": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1689068808,
-        "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "flake-utils_2": {
-      "inputs": {
-        "systems": "systems_2"
-      },
-      "locked": {
-        "lastModified": 1689068808,
-        "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "nix-github-actions": {
-      "inputs": {
-        "nixpkgs": [
-          "poetry2nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1688870561,
-        "narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=",
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1692174805,
-        "narHash": "sha256-xmNPFDi/AUMIxwgOH/IVom55Dks34u1g7sFKKebxUm0=",
-        "owner": "NixOS",
+        "lastModified": 1765779637,
+        "narHash": "sha256-KJ2wa/BLSrTqDjbfyNx70ov/HdgNBCBBSQP3BIzKnv4=",
+        "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "caac0eb6bdcad0b32cb2522e03e4002c8975c62e",
+        "rev": "1306659b587dc277866c7b69eb97e5f07864d8c4",
         "type": "github"
       },
       "original": {
-        "owner": "NixOS",
+        "owner": "nixos",
         "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
-    "poetry2nix": {
-      "inputs": {
-        "flake-utils": "flake-utils_2",
-        "nix-github-actions": "nix-github-actions",
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1692048894,
-        "narHash": "sha256-cDw03rso2V4CDc3Mll0cHN+ztzysAvdI8pJ7ybbz714=",
-        "ref": "refs/heads/pyqt6",
-        "rev": "b059ad4c3051f45d6c912e17747aae37a9ec1544",
-        "revCount": 2276,
-        "type": "git",
-        "url": "file:///home/lord_fomo/repos/poetry2nix"
-      },
-      "original": {
-        "type": "git",
-        "url": "file:///home/lord_fomo/repos/poetry2nix"
-      }
-    },
     "root": {
       "inputs": {
-        "flake-utils": "flake-utils",
-        "nixpkgs": "nixpkgs",
-        "poetry2nix": "poetry2nix"
-      }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
-    "systems_2": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
+        "nixpkgs": "nixpkgs"
       }
     }
   },

flake.nix

@@ -1,180 +1,103 @@
-# NOTE: to convert to a poetry2nix env like this here are the
-# steps:
-# - install poetry in your system nix config
-# - convert the repo to use poetry using `poetry init`:
-#   https://python-poetry.org/docs/basic-usage/#initialising-a-pre-existing-project
-# - then manually ensuring all deps are converted over:
-# - add this file to the repo and commit it
-# - 
-
-# GROKin tips:
-# - CLI eps are (ostensibly) added via an `entry_points.txt`:
-#   - https://packaging.python.org/en/latest/specifications/entry-points/#file-format
-#   - https://github.com/nix-community/poetry2nix/blob/master/editable.nix#L49
+# An "impure" template thx to `pyproject.nix`,
+# https://pyproject-nix.github.io/pyproject.nix/templates.html#impure
+# https://github.com/pyproject-nix/pyproject.nix/blob/master/templates/impure/flake.nix
 {
-  description = "piker: trading gear for hackers (pkged with poetry2nix)";
+  description = "An impure `piker` overlay using `uv` with Nix(OS)";
 
-  inputs.flake-utils.url = "github:numtide/flake-utils";
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
-
-  # see https://github.com/nix-community/poetry2nix/tree/master#api
-  inputs.poetry2nix = {
-    # url = "github:nix-community/poetry2nix";
-    # url = "github:K900/poetry2nix/qt5-explicit-deps";
-    url = "/home/lord_fomo/repos/poetry2nix";
-
-    inputs.nixpkgs.follows = "nixpkgs";
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
   };
 
-  outputs = {
-    self,
-    nixpkgs,
-    flake-utils,
-    poetry2nix,
-  }:
-    # TODO: build cross-OS and use the `${system}` var thingy..
-    flake-utils.lib.eachDefaultSystem (system:
+  outputs =
+    { nixpkgs, ... }:
     let
-        # use PWD as sources
-        projectDir = ./.;
-        pyproject = ./pyproject.toml;
-        poetrylock = ./poetry.lock;
-
-        # TODO: port to 3.11 and support both versions?
-        python = "python3.10";
-
-        # for more functions and examples.
-        # inherit
-        # (poetry2nix.legacyPackages.${system})
-        # mkPoetryApplication;
-        # pkgs = nixpkgs.legacyPackages.${system};
-
-        pkgs = nixpkgs.legacyPackages.x86_64-linux;
-        lib = pkgs.lib;
-        p2npkgs = poetry2nix.legacyPackages.x86_64-linux;
-
-        # define all pkg overrides per dep, see edgecases.md:
-        # https://github.com/nix-community/poetry2nix/blob/master/docs/edgecases.md
-        # TODO: add these into the json file:
-        # https://github.com/nix-community/poetry2nix/blob/master/overrides/build-systems.json
-        pypkgs-build-requirements = {
-          asyncvnc = [ "setuptools" ];
-          eventkit = [ "setuptools" ];
-          ib-insync = [ "setuptools" "flake8" ];
-          msgspec = [ "setuptools"];
-          pdbp = [ "setuptools" ];
-          pyqt6-sip = [ "setuptools" ];
-          tabcompleter = [ "setuptools" ];
-          tractor = [ "setuptools" ];
-          tricycle = [ "setuptools" ];
-          trio-typing = [ "setuptools" ];
-          trio-util = [ "setuptools" ];
-          xonsh = [ "setuptools" ];
-        };
-
-        # auto-generate override entries
-        p2n-overrides = p2npkgs.defaultPoetryOverrides.extend (self: super:
-          builtins.mapAttrs (package: build-requirements:
-            (builtins.getAttr package super).overridePythonAttrs (old: {
-              buildInputs = (
-                old.buildInputs or [ ]
-              ) ++ (
-                builtins.map (
-                  pkg: if builtins.isString pkg then builtins.getAttr pkg super else pkg
-                  ) build-requirements
-              );
-            })
-          ) pypkgs-build-requirements
-        );
-
-        # override some ahead-of-time compiled extensions
-        # to be built with their wheels.
-        ahot_overrides = p2n-overrides.extend(
-          final: prev: {
-
-            # llvmlite = prev.llvmlite.override {
-            #   preferWheel = false;
-            # };
-
-            # TODO: get this workin with p2n and nixpkgs..
-            # pyqt6 = prev.pyqt6.override {
-            #   preferWheel = true;
-            # };
-
-            # NOTE: this DOESN'T work atm but after a fix
-            # to poetry2nix, it will and actually this line
-            # won't be needed - thanks @k900:
-            # https://github.com/nix-community/poetry2nix/pull/1257
-            pyqt5 = prev.pyqt5.override {
-              # withWebkit = false;
-              preferWheel = true;
-            };
-
-            # see PR from @k900:
-            # https://github.com/nix-community/poetry2nix/pull/1257
-            # pyqt5-qt5 = prev.pyqt5-qt5.override {
-            #   withWebkit = false;
-            #   preferWheel = true;
-            # };
-
-            # TODO: patch in an override for polars to build
-            # from src! See the details likely needed from
-            # the cryptography entry:
-            # https://github.com/nix-community/poetry2nix/blob/master/overrides/default.nix#L426-L435
-            polars = prev.polars.override {
-              preferWheel = true;
-            };
-          }
-      );
-
-      # WHY!? -> output-attrs that `nix develop` scans for:
-      # https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-develop.html#flake-output-attributes
+      inherit (nixpkgs) lib;
+      forAllSystems = lib.genAttrs lib.systems.flakeExposed;
     in
-      rec {
-        packages = {
-          # piker = poetry2nix.legacyPackages.x86_64-linux.mkPoetryEditablePackage {
-          #   editablePackageSources = { piker = ./piker; };
+    {
+      devShells = forAllSystems (
+        system:
+        let
+          pkgs = nixpkgs.legacyPackages.${system};
 
-          piker = p2npkgs.mkPoetryApplication {
-            projectDir = projectDir;
+          # do store-path extractions
+          qt6baseStorePath = lib.getLib pkgs.qt6.qtbase;
+          # ?TODO? can remove below since manual linking not needed?
+          # qt6QtWaylandStorePath = lib.getLib pkgs.qt6.qtwayland;
 
-            # SEE ABOVE for auto-genned input set, override
-            # buncha deps with extras.. like `setuptools` mostly.
-            # TODO: maybe propose a patch to p2n to show that you
-            # can even do this in the edgecases docs?
-            overrides = ahot_overrides;
+          # XXX NOTE XXX, for now we overlay specific pkgs via
+          # a major-version-pinned-`cpython`
+          cpython = "python313";
+          pypkgs = pkgs."${cpython}Packages";
+        in
+        {
+          default = pkgs.mkShell {
 
-            # XXX: won't work on llvmlite..
-            # preferWheels = true;
-          };
-        };
+            packages = with pkgs; [
+              # XXX, ensure sh completions active!
+              bashInteractive
+              bash-completion
 
-        # devShells.default = pkgs.mkShell {
-        #   projectDir = projectDir;
-        #   python = "python3.10";
-        #   overrides = ahot_overrides;
-        #   inputsFrom = [ self.packages.x86_64-linux.piker ];
-        #   packages = packages;
-        #   # packages = [ poetry2nix.packages.${system}.poetry ];
-        # };
+              # dev utils
+              ruff
+              pypkgs.ruff
 
-        # TODO: grok the difference here..
-        # - avoid re-cloning git repos on every develop entry..
-        # - ideally allow hacking on the src code of some deps
-        #   (tractor, pyqtgraph, tomlkit, etc.) WITHOUT having to
-        #   re-install them every time a change is made.
-        # - boot a usable xonsh inside the poetry virtualenv when
-        #   defined via a custom entry point?
-        devShells.default = p2npkgs.mkPoetryEnv {
-        # env = p2npkgs.mkPoetryEnv {
-            projectDir = projectDir;
-            python = pkgs.python310;
-            overrides = ahot_overrides;
-            editablePackageSources = packages;
-              # piker = "./";
-              # tractor = "../tractor/";
-            # };  # wut?
+              qt6.qtwayland
+              qt6.qtbase
+
+              uv
+              python313  # ?TODO^ how to set from `cpython` above?
+              pypkgs.pyqt6
+              pypkgs.pyqt6-sip
+              pypkgs.qtpy
+              pypkgs.qdarkstyle
+              pypkgs.rapidfuzz
+            ];
+
+            shellHook = ''
+              # unmask to debug **this** dev-shell-hook
+              # set -e
+
+              # set qt-base/plugin path(s)
+              QTBASE_PATH="${qt6baseStorePath}/lib"
+              QT_PLUGIN_PATH="${qt6baseStorePath}/lib/qt-6/plugins"
+              QT_QPA_PLATFORM_PLUGIN_PATH="$QT_PLUGIN_PATH/platforms"
+
+              # link in Qt cc lib paths from <nixpkgs>
+              LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$QTBASE_PATH"
+              LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$QT_PLUGIN_PATH"
+              LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$QT_QPA_PLATFORM_PLUGIN_PATH"
+
+              # link-in c++ stdlib for various AOT-ext-pkgs (numpy, etc.)
+              LD_LIBRARY_PATH="${pkgs.stdenv.cc.cc.lib}/lib:$LD_LIBRARY_PATH"
+
+              export LD_LIBRARY_PATH
+
+              # RUNTIME-SETTINGS
+              #
+              # ------ Qt ------
+              # XXX, unmask to debug qt .so linking/loading deats
+              # export QT_DEBUG_PLUGINS=1
+              #
+              # ALSO, for *modern linux* DEs,
+              # - maybe set wayland-mode (TODO, parametrtize this!)
+              #  * a chosen wayland-mode shell-integration
+              export QT_QPA_PLATFORM="wayland"
+              export QT_WAYLAND_SHELL_INTEGRATION="xdg-shell"
+
+              # ------ uv ------
+              # - always use the ./py313/ venv-subdir
+              export UV_PROJECT_ENVIRONMENT="py313"
+              # sync project-env with all extras
+              uv sync --dev --all-extras --no-group lint
+
+              # ------ TIPS ------
+              # NOTE, to launch the py-venv installed `xonsh` (like @goodboy)
+              # run the `nix develop` cmd with,
+              # >> nix develop -c uv run xonsh
+            '';
           };
         }
-    );  # end of .outputs scope
+      );
+    };
 }

piker/accounting/_ledger.py

@@ -40,7 +40,7 @@ import tomli_w  # for fast ledger writing
 
 from piker.types import Struct
 from piker import config
-from ..log import get_logger
+from piker.log import get_logger
 from .calc import (
     iter_by_dt,
 )
@@ -239,7 +239,9 @@ class TransactionLedger(UserDict):
 
         symcache: SymbologyCache = self._symcache
         towrite: dict[str, Any] = {}
-        for tid, txdict in self.tx_sort(self.data.copy()):
+        for tid, txdict in self.tx_sort(
+            self.data.copy()
+        ):
             # write blank-str expiry for non-expiring assets
             if (
                 'expiry' in txdict
@@ -377,7 +379,7 @@ def open_trade_ledger(
         account,
         dirpath=_fp,
     )
-    cpy = ledger_dict.copy()
+    cpy: dict = ledger_dict.copy()
 
     # XXX NOTE: if not provided presume we are being called from
     # sync code and need to maybe run `trio` to generate..
@@ -406,7 +408,13 @@ def open_trade_ledger(
         account=account,
         mod=mod,
         symcache=symcache,
-        tx_sort=getattr(mod, 'tx_sort', tx_sort),
+
+        # NOTE: allow backends to provide custom ledger sorting
+        tx_sort=getattr(
+            mod,
+            'tx_sort',
+            tx_sort,
+        ),
     )
     try:
         yield ledger

piker/accounting/_mktinfo.py

@@ -305,8 +305,8 @@ class MktPair(Struct, frozen=True):
     # config right?
     # src_type: AssetTypeName
 
-    # for derivs, info describing contract, egs.
-    # strike price, call or put, swap type, exercise model, etc.
+    # for derivs, info describing contract, egs. strike price, call
+    # or put, swap type, exercise model, etc.
     contract_info: list[str] | None = None
 
     # TODO: rename to sectype since all of these can

piker/accounting/_pos.py

@@ -30,7 +30,8 @@ from types import ModuleType
 from typing import (
     Any,
     Iterator,
-    Generator
+    Generator,
+    TYPE_CHECKING,
 )
 
 import pendulum
@@ -59,8 +60,10 @@ from ..clearing._messages import (
     BrokerdPosition,
 )
 from piker.types import Struct
+from piker.log import get_logger
+
+if TYPE_CHECKING:
     from piker.data._symcache import SymbologyCache
-from ..log import get_logger
 
 log = get_logger(__name__)
 
@@ -502,6 +505,17 @@ class Account(Struct):
 
         _mktmap_table: dict[str, MktPair] | None = None,
 
+        only_require: list[str]|True = True,
+        # ^list of fqmes that are "required" to be processed from
+        # this ledger pass; we often don't care about others and
+        # definitely shouldn't always error in such cases.
+        # (eg. broker backend loaded that doesn't yet supsport the
+        # symcache but also, inside the paper engine we don't ad-hoc
+        # request `get_mkt_info()` for every symbol in the ledger,
+        # only the one for which we're simulating against).
+        # TODO, not sure if there's a better soln for this, ideally
+        # all backends get symcache support afap i guess..
+
     ) -> dict[str, Position]:
         '''
         Update the internal `.pps[str, Position]` table from input
@@ -544,11 +558,32 @@ class Account(Struct):
                 if _mktmap_table is None:
                     raise
 
+                required: bool = (
+                    only_require is True
+                    or (
+                        only_require is not True
+                        and
+                        fqme in only_require
+                    )
+                )
                 # XXX: caller is allowed to provide a fallback
                 # mktmap table for the case where a new position is
                 # being added and the preloaded symcache didn't
                 # have this entry prior (eg. with frickin IB..)
-                mkt = _mktmap_table[fqme]
+                if (
+                    not (mkt := _mktmap_table.get(fqme))
+                    and
+                    required
+                ):
+                    raise
+
+                elif not required:
+                    continue
+
+                else:
+                    # should be an entry retreived somewhere
+                    assert mkt
+
 
             if not (pos := pps.get(bs_mktid)):
 
@@ -665,7 +700,7 @@ class Account(Struct):
     def write_config(self) -> None:
         '''
         Write the current account state to the user's account TOML file, normally
-        something like ``pps.toml``.
+        something like `pps.toml`.
 
         '''
         # TODO: show diff output?

piker/accounting/calc.py

@@ -268,9 +268,6 @@ def iter_by_dt(
                     (v := tx.get(k))
                 )
             ):
-                # TODO? remove yah?
-                # v = tx[k] if isdict else tx.dt
-
                 # only call parser on the value if not None from
                 # the `parsers` table above (when NOT using
                 # `.get()`), otherwise pass through the value and
@@ -287,24 +284,50 @@ def iter_by_dt(
                 return ret
 
             else:
+                log.debug(
+                    f'Parser-field not found in txn\n'
+                    f'\n'
+                    f'parser-field: {k!r}\n'
+                    f'txn: {tx!r}\n'
+                    f'\n'
+                    f'Trying next..\n'
+                )
                 continue
 
-        # XXX: should never get here..
+        # XXX: we should never really get here bc it means some kinda
+        # bad txn-record (field) data..
+        #
+        # -> set the `debug_mode = True` if you want to trace such
+        # cases from REPL ;)
         else:
-            with maybe_open_crash_handler(pdb=True):
-                raise ValueError(
-                    f'Invalid txn time ??\n'
-                    f'txn-id: {k!r}\n'
-                    f'{k!r}: {v!r}\n'
+            # XXX: we should really never get here..
+            # only if a ledger record has no expected sort(able)
+            # field will we likely hit this.. like with ze IB.
+            # if no sortable field just deliver epoch?
+            log.warning(
+                'No (time) sortable field for TXN:\n'
+                f'{tx!r}\n'
             )
-                    # assert v is not None, f'No valid value for `{k}`!?'
+            report: str = (
+                f'No supported time-field found in txn !?\n'
+                f'\n'
+                f'supported-time-fields: {parsers!r}\n'
+                f'\n'
+                f'txn: {tx!r}\n'
+            )
+            if debug:
+                with maybe_open_crash_handler(
+                    pdb=debug,
+                    raise_on_exit=False,
+                ):
+                    raise ValueError(report)
+            else:
+                log.error(report)
 
             if _invalid is not None:
                 _invalid.append(tx)
                 return from_timestamp(0.)
 
-                # breakpoint()
-
     entry: tuple[str, dict]|Transaction
     invalid: list = []
     for entry in sorted(
@@ -318,8 +341,6 @@ def iter_by_dt(
             log.warning(
                 f'Ignoring txn w invalid timestamp ??\n'
                 f'{pformat(entry)}\n'
-                # f'txn-id: {k!r}\n'
-                # f'{k!r}: {v!r}\n'
             )
             continue
 
@@ -400,7 +421,10 @@ def open_ledger_dfs(
     can update the ledger on exit.
 
     '''
-    with maybe_open_crash_handler(pdb=debug_mode):
+    with maybe_open_crash_handler(
+        pdb=debug_mode,
+        # raise_on_exit=False,
+    ):
         if not ledger:
             import time
             from ._ledger import open_trade_ledger

piker/accounting/cli.py

@@ -300,7 +300,8 @@ def disect(
         assert not df.is_empty()
 
         # muck around in pdbp REPL
-        breakpoint()
+        # tractor.devx.mk_pdb().set_trace()
+        # breakpoint()
 
         # TODO: we REALLY need a better console REPL for this
         # kinda thing..

piker/brokers/__init__.py

@@ -98,13 +98,14 @@ async def open_cached_client(
     If one has not been setup do it and cache it.
 
     '''
-    brokermod = get_brokermod(brokername)
+    brokermod: ModuleType = get_brokermod(brokername)
+
+    # TODO: make abstract or `typing.Protocol`
+    # client: Client
     async with maybe_open_context(
         acm_func=brokermod.get_client,
         kwargs=kwargs,
-
     ) as (cache_hit, client):
-
         if cache_hit:
             log.runtime(f'Reusing existing {client}')
 

piker/brokers/binance/feed.py

@@ -94,13 +94,15 @@ class L1(Struct):
 
 
 # validation type
+# https://developers.binance.com/docs/derivatives/usds-margined-futures/websocket-market-streams/Aggregate-Trade-Streams#response-example
 class AggTrade(Struct, frozen=True):
     e: str  # Event type
     E: int  # Event time
     s: str  # Symbol
     a: int  # Aggregate trade ID
     p: float  # Price
-    q: float  # Quantity
+    q: float  # Quantity with all the market trades
+    nq: float # Normal quantity without the trades involving RPI orders
     f: int  # First trade ID
     l: int  # noqa Last trade ID
     T: int  # Trade time

piker/brokers/binance/venues.py

@@ -104,6 +104,9 @@ class Pair(Struct, frozen=True, kw_only=True):
     # https://developers.binance.com/docs/binance-spot-api-docs#future-changes
     pegInstructionsAllowed: bool = False
 
+    # https://developers.binance.com/docs/binance-spot-api-docs#2025-12-02
+    opoAllowed: bool = False
+
     filters: dict[
         str,
         str | int | float,
@@ -220,7 +223,10 @@ class FutesPair(Pair):
                 assert pair == self.pair  # sanity
                 return f'{expiry}'
 
-            case 'PERPETUAL':
+            case (
+                'PERPETUAL'
+                | 'TRADIFI_PERPETUAL'
+            ):
                 return 'PERP'
 
             case '':
@@ -249,7 +255,10 @@ class FutesPair(Pair):
         margin: str = self.marginAsset
 
         match ctype:
-            case 'PERPETUAL':
+            case (
+                'PERPETUAL'
+                | 'TRADIFI_PERPETUAL'
+            ):
                 return f'{margin}M'
 
             case (

piker/brokers/cli.py

@@ -471,11 +471,15 @@ def search(
 
     '''
     # global opts
-    brokermods = list(config['brokermods'].values())
+    brokermods: list[ModuleType] = list(config['brokermods'].values())
+
+    # TODO: this is coming from the `search --pdb` NOT from
+    # the `piker --pdb` XD ..
+    # -[ ] pull from the parent click ctx's values..dumdum
+    # assert pdb
 
     # define tractor entrypoint
     async def main(func):
-
         async with maybe_open_pikerd(
             loglevel=config['loglevel'],
             debug_mode=pdb,

piker/brokers/core.py

@@ -22,7 +22,9 @@ routines should be primitive data types where possible.
 """
 import inspect
 from types import ModuleType
-from typing import List, Dict, Any, Optional
+from typing import (
+    Any,
+)
 
 import trio
 
@@ -34,8 +36,10 @@ from ..accounting import MktPair
 
 
 async def api(brokername: str, methname: str, **kwargs) -> dict:
-    """Make (proxy through) a broker API call by name and return its result.
-    """
+    '''
+    Make (proxy through) a broker API call by name and return its result.
+
+    '''
     brokermod = get_brokermod(brokername)
     async with brokermod.get_client() as client:
         meth = getattr(client, methname, None)
@@ -62,10 +66,14 @@ async def api(brokername: str, methname: str, **kwargs) -> dict:
 
 async def stocks_quote(
     brokermod: ModuleType,
-    tickers: List[str]
-) -> Dict[str, Dict[str, Any]]:
-    """Return quotes dict for ``tickers``.
-    """
+    tickers: list[str]
+
+) -> dict[str, dict[str, Any]]:
+    '''
+    Return a `dict` of snapshot quotes for the provided input
+    `tickers`: a `list` of fqmes.
+
+    '''
     async with brokermod.get_client() as client:
         return await client.quote(tickers)
 
@@ -74,13 +82,15 @@ async def stocks_quote(
 async def option_chain(
     brokermod: ModuleType,
     symbol: str,
-    date: Optional[str] = None,
-) -> Dict[str, Dict[str, Dict[str, Any]]]:
-    """Return option chain for ``symbol`` for ``date``.
+    date: str|None = None,
+) -> dict[str, dict[str, dict[str, Any]]]:
+    '''
+    Return option chain for ``symbol`` for ``date``.
 
     By default all expiries are returned. If ``date`` is provided
     then contract quotes for that single expiry are returned.
-    """
+
+    '''
     async with brokermod.get_client() as client:
         if date:
             id = int((await client.tickers2ids([symbol]))[symbol])
@@ -98,7 +108,7 @@ async def option_chain(
 # async def contracts(
 #     brokermod: ModuleType,
 #     symbol: str,
-# ) -> Dict[str, Dict[str, Dict[str, Any]]]:
+# ) -> dict[str, dict[str, dict[str, Any]]]:
 #     """Return option contracts (all expiries) for ``symbol``.
 #     """
 #     async with brokermod.get_client() as client:
@@ -110,15 +120,24 @@ async def bars(
     brokermod: ModuleType,
     symbol: str,
     **kwargs,
-) -> Dict[str, Dict[str, Dict[str, Any]]]:
-    """Return option contracts (all expiries) for ``symbol``.
-    """
+) -> dict[str, dict[str, dict[str, Any]]]:
+    '''
+    Return option contracts (all expiries) for ``symbol``.
+
+    '''
     async with brokermod.get_client() as client:
         return await client.bars(symbol, **kwargs)
 
 
-async def search_w_brokerd(name: str, pattern: str) -> dict:
+async def search_w_brokerd(
+    name: str,
+    pattern: str,
+) -> dict:
 
+    # TODO: WHY NOT WORK!?!
+    # when we `step` through the next block?
+    # import tractor
+    # await tractor.pause()
     async with open_cached_client(name) as client:
 
         # TODO: support multiple asset type concurrent searches.
@@ -130,12 +149,12 @@ async def symbol_search(
     pattern: str,
     **kwargs,
 
-) -> Dict[str, Dict[str, Dict[str, Any]]]:
+) -> dict[str, dict[str, dict[str, Any]]]:
     '''
     Return symbol info from broker.
 
     '''
-    results = []
+    results: list[str] = []
 
     async def search_backend(
         brokermod: ModuleType
@@ -143,6 +162,13 @@ async def symbol_search(
 
         brokername: str = mod.name
 
+        # TODO: figure this the FUCK OUT
+        # -> ok so obvi in the root actor any async task that's
+        # spawned outside the main tractor-root-actor task needs to
+        # call this..
+        # await tractor.devx._debug.maybe_init_greenback()
+        # tractor.pause_from_sync()
+
         async with maybe_spawn_brokerd(
             mod.name,
             infect_asyncio=getattr(
@@ -162,7 +188,6 @@ async def symbol_search(
             ))
 
     async with trio.open_nursery() as n:
-
         for mod in brokermods:
             n.start_soon(search_backend, mod.name)
 
@@ -172,11 +197,13 @@ async def symbol_search(
 async def mkt_info(
     brokermod: ModuleType,
     fqme: str,
+
     **kwargs,
 
 ) -> MktPair:
     '''
-    Return MktPair info from broker including src and dst assets.
+    Return the `piker.accounting.MktPair` info struct from a given
+    backend broker tradable src/dst asset pair.
 
     '''
     async with open_cached_client(brokermod.name) as client:

piker/brokers/ib/_util.py

@@ -38,7 +38,6 @@ from piker.brokers._util import get_logger
 
 if TYPE_CHECKING:
     from .api import Client
-    from ib_insync import IB
     import i3ipc
 
 log = get_logger('piker.brokers.ib')
@@ -62,7 +61,7 @@ no_setup_msg:str = (
 
 
 def try_xdo_manual(
-    vnc_sockaddr: str,
+    client: Client,
 ):
     '''
     Do the "manual" `xdo`-based screen switch + click
@@ -79,6 +78,7 @@ def try_xdo_manual(
         _reset_tech = 'i3ipc_xdotool'
         return True
     except OSError:
+        vnc_sockaddr: str = client.conf.vnc_addrs
         log.exception(
             no_setup_msg.format(vnc_sockaddr=vnc_sockaddr)
         )
@@ -86,7 +86,6 @@ def try_xdo_manual(
 
 
 async def data_reset_hack(
-    # vnc_host: str,
     client: Client,
     reset_type: Literal['data', 'connection'],
 
@@ -118,36 +117,24 @@ async def data_reset_hack(
           that need to be wrangle.
 
     '''
-    ib_client: IB = client.ib
-
     # look up any user defined vnc socket address mapped from
     # a particular API socket port.
-    api_port: str = str(ib_client.client.port)
-    vnc_host: str
-    vnc_port: int
-    vnc_sockaddr: tuple[str] | None = client.conf.get('vnc_addrs')
-
-    if not vnc_sockaddr:
+    vnc_addrs: tuple[str]|None = client.conf.get('vnc_addrs')
+    if not vnc_addrs:
         log.warning(
-            no_setup_msg.format(vnc_sockaddr=vnc_sockaddr)
+            no_setup_msg.format(vnc_sockaddr=client.conf)
             +
             'REQUIRES A `vnc_addrs: array` ENTRY'
         )
 
-    vnc_host, vnc_port = vnc_sockaddr.get(
-        api_port,
-        ('localhost', 3003)
-    )
     global _reset_tech
-
     match _reset_tech:
         case 'vnc':
             try:
                 await tractor.to_asyncio.run_task(
                     partial(
                         vnc_click_hack,
-                        host=vnc_host,
-                        port=vnc_port,
+                        client=client,
                     )
                 )
             except (
@@ -158,29 +145,31 @@ async def data_reset_hack(
                     import i3ipc  # noqa  (since a deps dynamic check)
                 except ModuleNotFoundError:
                     log.warning(
-                        no_setup_msg.format(vnc_sockaddr=vnc_sockaddr)
+                        no_setup_msg.format(vnc_sockaddr=client.conf)
                     )
                     return False
 
-                if vnc_host not in {
-                    'localhost',
-                    '127.0.0.1',
-                }:
-                    focussed, matches = i3ipc_fin_wins_titled()
-                    if not matches:
-                        log.warning(
-                            no_setup_msg.format(vnc_sockaddr=vnc_sockaddr)
-                        )
-                        return False
-                    else:
-                        try_xdo_manual(vnc_sockaddr)
+                # XXX, Xorg only workaround..
+                # TODO? remove now that we have `pyvnc`?
+                # if vnc_host not in {
+                #     'localhost',
+                #     '127.0.0.1',
+                # }:
+                #     focussed, matches = i3ipc_fin_wins_titled()
+                #     if not matches:
+                #         log.warning(
+                #             no_setup_msg.format(vnc_sockaddr=vnc_sockaddr)
+                #         )
+                #         return False
+                #     else:
+                #         try_xdo_manual(vnc_sockaddr)
 
                 # localhost but no vnc-client or it borked..
                 else:
-                    try_xdo_manual(vnc_sockaddr)
+                    try_xdo_manual(client)
 
         case 'i3ipc_xdotool':
-            try_xdo_manual(vnc_sockaddr)
+            try_xdo_manual(client)
             # i3ipc_xdotool_manual_click_hack()
 
         case _ as tech:
@@ -191,15 +180,55 @@ async def data_reset_hack(
 
 
 async def vnc_click_hack(
-    host: str,
-    port: int,
-    reset_type: str = 'data'
+    client: Client,
+    reset_type: str = 'data',
+    pw: str|None = None,
+
 ) -> None:
     '''
     Reset the data or network connection for the VNC attached
     ib-gateway using a (magic) keybinding combo.
 
+    A vnc-server password can be set either by an input `pw` param or
+    set in the client's config with the latter loaded from the user's
+    `brokers.toml` in a vnc-addrs-port-mapping section,
+
+    .. code:: toml
+
+        [ib.vnc_addrs]
+        4002 = {host = 'localhost', port = 5900, pw = 'doggy'}
+
     '''
+    api_port: str = str(client.ib.client.port)
+    conf: dict = client.conf
+    vnc_addrs: dict[int, tuple] = conf.get('vnc_addrs')
+    if not vnc_addrs:
+        return None
+
+    addr_entry: dict|tuple = vnc_addrs.get(
+        api_port,
+        ('localhost', 5900)  # a typical default
+    )
+    if pw is None:
+        match addr_entry:
+            case (
+                host,
+                port,
+            ):
+                pass
+
+            case {
+                'host': host,
+                'port': port,
+                'pw': pw
+            }:
+                pass
+
+            case _:
+                raise ValueError(
+                    f'Invalid `ib.vnc_addrs` entry ?\n'
+                    f'{addr_entry!r}\n'
+                )
     try:
         from pyvnc import (
             AsyncVNCClient,
@@ -226,7 +255,7 @@ async def vnc_click_hack(
             VNCConfig(
                 host=host,
                 port=port,
-                password='doggy',
+                password=pw,
             )
         )
         async with client:

piker/brokers/ib/api.py

@@ -944,6 +944,7 @@ class Client:
                     )
                     if tkr:
                         break
+
                 except TimeoutError as err:
                     timeouterr = err
                     await asyncio.sleep(0.01)
@@ -952,7 +953,9 @@ class Client:
                 else:
                     if not warnset:
                         log.warning(
-                            f'Quote req timed out..maybe venue is closed?\n'
+                            f'Quote req timed out..\n'
+                            f'Maybe the venue is closed?\n'
+                            f'\n'
                             f'{asdict(contract)}'
                         )
                         warnset = True
@@ -964,9 +967,11 @@ class Client:
                 )
                 break
         else:
-            if timeouterr and raise_on_timeout:
-                import pdbp
-                pdbp.set_trace()
+            if (
+                timeouterr
+                and
+                raise_on_timeout
+            ):
                 raise timeouterr
 
             if not warnset:

piker/brokers/ib/broker.py

@@ -117,7 +117,11 @@ def pack_position(
             symbol=fqme,
             currency=con.currency,
             size=float(pos.position),
-            avg_price=float(pos.avgCost) / float(con.multiplier or 1.0),
+            avg_price=(
+                float(pos.avgCost)
+                /
+                float(con.multiplier or 1.0)
+            ),
         ),
     )
 
@@ -563,7 +567,7 @@ async def open_trade_dialog(
         ledgers: dict[str, TransactionLedger] = {}
         tables: dict[str, Account] = {}
         order_msgs: list[Status] = []
-        conf = get_config()
+        conf: dict = get_config()
         accounts_def_inv: bidict[str, str] = bidict(
             conf['accounts']
         ).inverse

piker/brokers/ib/feed.py

@@ -613,7 +613,7 @@ async def get_bars(
                         data_cs.cancel()
 
                     # spawn new data reset task
-                    data_cs, reset_done = await nurse.start(
+                    data_cs, reset_done = await tn.start(
                         partial(
                             wait_on_data_reset,
                             proxy,
@@ -635,12 +635,12 @@ async def get_bars(
     unset_resetter: bool = False
     async with (
         tractor.trionics.collapse_eg(),
-        trio.open_nursery() as nurse
+        trio.open_nursery() as tn
     ):
 
         # start history request that we allow
         # to run indefinitely until a result is acquired
-        nurse.start_soon(query)
+        tn.start_soon(query)
 
         # start history reset loop which waits up to the timeout
         # for a result before triggering a data feed reset.
@@ -660,7 +660,7 @@ async def get_bars(
                 unset_resetter: bool = True
 
             # spawn new data reset task
-            data_cs, reset_done = await nurse.start(
+            data_cs, reset_done = await tn.start(
                 partial(
                     wait_on_data_reset,
                     proxy,
@@ -896,7 +896,10 @@ async def open_aio_quote_stream(
     symbol: str,
     contract: Contract|None = None,
 
-) -> trio.abc.ReceiveStream:
+) -> (
+    trio.abc.Channel|  # iface
+    tractor.to_asyncio.LinkedTaskChannel  # actually
+):
     '''
     Open a real-time `Ticker` quote stream from an `asyncio.Task`
     spawned via `tractor.to_asyncio.open_channel_from()`, deliver the
@@ -919,6 +922,7 @@ async def open_aio_quote_stream(
             yield from_aio
             return
 
+    from_aio: tractor.to_asyncio.LinkedTaskChannel
     async with tractor.to_asyncio.open_channel_from(
         _setup_quote_stream,
         symbol=symbol,
@@ -1079,7 +1083,8 @@ async def stream_quotes(
         con: Contract = details.contract
         first_ticker: Ticker|None = None
 
-        with trio.move_on_after(1.6) as quote_cs:
+        timeout: float = 1.6
+        with trio.move_on_after(timeout) as quote_cs:
             first_ticker: Ticker = await proxy.get_quote(
                 contract=con,
                 raise_on_timeout=False,
@@ -1088,7 +1093,9 @@ async def stream_quotes(
         # XXX should never happen with this ep right?
         # but if so then, more then likely mkt is closed?
         if quote_cs.cancelled_caught:
-            await tractor.pause()
+            log.warning(
+                f'First quote req timed out after {timeout!r}s'
+            )
 
         if first_ticker:
             first_quote: dict = normalize(first_ticker)
@@ -1161,6 +1168,7 @@ async def stream_quotes(
         )
         cs: trio.CancelScope|None = None
         startup: bool = True
+        iter_quotes: trio.abc.Channel
         while (
             startup
             or
@@ -1169,11 +1177,11 @@ async def stream_quotes(
             with trio.CancelScope() as cs:
                 async with (
                     tractor.trionics.collapse_eg(),
-                    trio.open_nursery() as nurse,
+                    trio.open_nursery() as tn,
                     open_aio_quote_stream(
                         symbol=sym,
                         contract=con,
-                    ) as stream,
+                    ) as iter_quotes,
                 ):
                     # ?TODO? can we rm this - particularly for `ib_async`?
                     # ugh, clear ticks since we've consumed them
@@ -1202,9 +1210,9 @@ async def stream_quotes(
                         await rt_ev.wait()
                         cs.cancel()  # cancel called should now be set
 
-                    nurse.start_soon(reset_on_feed)
+                    tn.start_soon(reset_on_feed)
 
-                    async with aclosing(stream):
+                    async with aclosing(iter_quotes):
                         # if syminfo.get('no_vlm', False):
                         if not init_msg.shm_write_opts['has_vlm']:
 
@@ -1219,19 +1227,21 @@ async def stream_quotes(
                             # wait for real volume on feed (trading might be
                             # closed)
                             while True:
-                                ticker = await stream.receive()
+                                ticker = await iter_quotes.receive()
 
                                 # for a real volume contract we rait for
                                 # the first "real" trade to take place
                                 if (
                                     # not calc_price
                                     # and not ticker.rtTime
-                                    not ticker.rtTime
+                                    False
+                                    # not ticker.rtTime
                                 ):
                                     # spin consuming tickers until we
                                     # get a real market datum
                                     log.debug(f"New unsent ticker: {ticker}")
                                     continue
+
                                 else:
                                     log.debug("Received first volume tick")
                                     # ugh, clear ticks since we've
@@ -1247,13 +1257,18 @@ async def stream_quotes(
                             log.debug(f"First ticker received {quote}")
 
                         # tell data-layer spawner-caller that live
-                        # quotes are now streaming.
+                        # quotes are now active desptie not having
+                        # necessarily received a first vlm/clearing
+                        # tick.
+                        ticker = await iter_quotes.receive()
                         feed_is_live.set()
+                        fqme: str = quote['fqme']
+                        await send_chan.send({fqme: quote})
 
                         # last = time.time()
-                        async for ticker in stream:
+                        async for ticker in iter_quotes:
                             quote = normalize(ticker)
-                            fqme = quote['fqme']
+                            fqme: str = quote['fqme']
                             log.debug(
                                 f'Sending quote\n'
                                 f'{quote}'

piker/brokers/kraken/broker.py

@@ -549,7 +549,7 @@ async def open_trade_dialog(
             # to be reloaded.
             balances: dict[str, float] = await client.get_balances()
 
-            verify_balances(
+            await verify_balances(
                 acnt,
                 src_fiat,
                 balances,

piker/brokers/questrade.py

@@ -37,6 +37,12 @@ import tractor
 from async_generator import asynccontextmanager
 import numpy as np
 import wrapt
+
+# TODO, port to `httpx`/`trio-websocket` whenver i get back to
+# writing a proper ws-api streamer for this backend (since the data
+# feeds are free now) as per GH feat-req:
+# https://github.com/pikers/piker/issues/509
+#
 import asks
 
 from ..calc import humanize, percent_change

piker/clearing/_paper_engine.py

@@ -655,6 +655,7 @@ async def open_trade_dialog(
                 # in) use manually constructed table from calling
                 # the `.get_mkt_info()` provider EP above.
                 _mktmap_table=mkt_by_fqme,
+                only_require=list(mkt_by_fqme),
             )
 
             pp_msgs: list[BrokerdPosition] = []

piker/cli/__init__.py

@@ -183,8 +183,8 @@ def pikerd(
                 registry_addrs=regaddrs,
                 loglevel=loglevel,
                 debug_mode=pdb,
-                enable_transports=['uds'],
-                # enable_transports=['tcp'],
+                # enable_transports=['uds'],
+                enable_transports=['tcp'],
             ) as service_mngr,
         ):
             assert service_mngr

piker/config.py

@@ -41,10 +41,13 @@ from .log import get_logger
 log = get_logger('broker-config')
 
 
-# XXX NOTE: taken from ``click`` since apparently they have some
-# super weirdness with sigint and sudo..no clue
-# we're probably going to slowly just modify it to our own version over
-# time..
+# XXX NOTE: taken from `click`
+# |_https://github.com/pallets/click/blob/main/src/click/utils.py#L449
+#
+# (since apparently they have some super weirdness with SIGINT and
+# sudo.. no clue we're probably going to slowly just modify it to our
+# own version over time..)
+#
 def get_app_dir(
     app_name: str,
     roaming: bool = True,
@@ -261,7 +264,7 @@ def load(
         MutableMapping,
     ] = tomllib.loads,
 
-    touch_if_dne: bool = False,
+    touch_if_dne: bool = True,
 
     **tomlkws,
 
@@ -270,7 +273,7 @@ def load(
     Load config file by name.
 
     If desired config is not in the top level piker-user config path then
-    pass the ``path: Path`` explicitly.
+    pass the `path: Path` explicitly.
 
     '''
     # create the $HOME/.config/piker dir if dne
@@ -285,7 +288,8 @@ def load(
 
     if (
         not path.is_file()
-        and touch_if_dne
+        and
+        touch_if_dne
     ):
         # only do a template if no path provided,
         # just touch an empty file with same name.

piker/data/_symcache.py

@@ -31,6 +31,7 @@ from pathlib import Path
 from pprint import pformat
 from typing import (
     Any,
+    Callable,
     Sequence,
     Hashable,
     TYPE_CHECKING,
@@ -56,7 +57,7 @@ from piker.brokers import (
 )
 
 if TYPE_CHECKING:
-    from ..accounting import (
+    from piker.accounting import (
         Asset,
         MktPair,
     )
@@ -161,19 +162,36 @@ class SymbologyCache(Struct):
                     'Implement `Client.get_assets()`!'
                 )
 
-            if get_mkt_pairs := getattr(client, 'get_mkt_pairs', None):
+            get_mkt_pairs: Callable|None = getattr(
+                client,
+                'get_mkt_pairs',
+                None,
+            )
+            if not get_mkt_pairs:
+                log.warning(
+                    'No symbology cache `Pair` support for `{provider}`..\n'
+                    'Implement `Client.get_mkt_pairs()`!'
+                )
+                return self
 
             pairs: dict[str, Struct] = await get_mkt_pairs()
-                for bs_fqme, pair in pairs.items():
+            if not pairs:
+                log.warning(
+                    'No pairs from intial {provider!r} sym-cache request?\n\n'
+                    '`Client.get_mkt_pairs()` -> {pairs!r} ?'
+                )
+                return self
 
-                    # NOTE: every backend defined pair should
-                    # declare it's ns path for roundtrip
-                    # serialization lookup.
+            for bs_fqme, pair in pairs.items():
                 if not getattr(pair, 'ns_path', None):
+                    # XXX: every backend defined pair must declare
+                    # a `.ns_path: tractor.NamespacePath` to enable
+                    # roundtrip serialization lookup from a local
+                    # cache file.
                     raise TypeError(
                         f'Pair-struct for {self.mod.name} MUST define a '
-                            '`.ns_path: str`!\n'
-                            f'{pair}'
+                        '`.ns_path: str`!\n\n'
+                        f'{pair!r}'
                     )
 
                 entry = await self.mod.get_mkt_info(pair.bs_fqme)
@@ -207,12 +225,6 @@ class SymbologyCache(Struct):
                 pair,
             )
 
-            else:
-                log.warning(
-                    'No symbology cache `Pair` support for `{provider}`..\n'
-                    'Implement `Client.get_mkt_pairs()`!'
-                )
-
         return self
 
     @classmethod

piker/data/feed.py

@@ -357,7 +357,9 @@ async def allocate_persistent_feed(
 
     # yield back control to starting nursery once we receive either
     # some history or a real-time quote.
-    log.info(f'loading OHLCV history: {fqme}')
+    log.info(
+        f'loading OHLCV history: {fqme!r}\n'
+    )
     await some_data_ready.wait()
 
     flume = Flume(
@@ -794,7 +796,6 @@ async def install_brokerd_search(
 
 @acm
 async def maybe_open_feed(
-
     fqmes: list[str],
     loglevel: str | None = None,
 
@@ -848,7 +849,6 @@ async def maybe_open_feed(
 
 @acm
 async def open_feed(
-
     fqmes: list[str],
 
     loglevel: str|None = None,

piker/data/flows.py

@@ -36,10 +36,10 @@ from ._sharedmem import (
     ShmArray,
     _Token,
 )
+from piker.accounting import MktPair
 
 if TYPE_CHECKING:
-    from ..accounting import MktPair
-    from .feed import Feed
+    from piker.data.feed import Feed
 
 
 class Flume(Struct):

piker/data/validate.py

@@ -113,9 +113,9 @@ def validate_backend(
             )
             if ep is None:
                 log.warning(
-                    f'Provider backend {mod.name} is missing '
-                    f'{daemon_name} support :(\n'
-                    f'The following endpoint is missing: {name}'
+                    f'Provider backend {mod.name!r} is missing '
+                    f'{daemon_name!r} support?\n'
+                    f'|_module endpoint-func missing: {name!r}\n'
                 )
 
     inits: list[

piker/ui/_app.py

@@ -181,14 +181,7 @@ async def _async_main(
             ):
                 # remove startup status text
                 starting_done()
-                try:
                 await trio.sleep_forever()
-                except KeyboardInterrupt as _kbi:
-                    log.cancel(
-                        f'User canceled Qt app.\n'
-                        f'{_kbi!r}'
-                    )
-                    raise _kbi
 
 
 def _main(

piker/ui/_style.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+# Copyright (C) Tyler Goodlet (in stewardship for piker0)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

piker/ui/_window.py

@@ -217,14 +217,8 @@ class MainWindow(QMainWindow):
         '''Cancel the root actor asap.
 
         '''
-        # raising a KBI seems to get intercepted by Qt, soo just use OS ??
-        # TODO XXX! seems to sometimes cause Qt core dumps if you
-        # cancel during chart bootup..?
-        # -[ ] find a better/more-correct cancellation API in Qt?
-        os.kill(
-            os.getpid(),
-            signal.SIGINT,
-        )
+        # raising KBI seems to get intercepted by by Qt so just use the system.
+        os.kill(os.getpid(), signal.SIGINT)
 
     @property
     def status_bar(self) -> QStatusBar:

pyproject.toml

@@ -63,10 +63,8 @@ dependencies = [
     "trio-util >=0.7.0, <0.8.0",
     "trio-websocket >=0.10.3, <0.11.0",
     "typer >=0.9.0, <1.0.0",
-    "rapidfuzz >=3.5.2, <4.0.0",
-    "pdbp >=1.5.0, <2.0.0",
     "trio >=0.27",
-    "pendulum >=3.0.0, <4.0.0",
+    "pendulum",
     "httpx >=0.27.0, <0.28.0",
     "cryptofeed >=2.4.0, <3.0.0",
     "pyarrow>=18.0.0",
@@ -78,8 +76,14 @@ dependencies = [
     "numba>=0.61.0",
     "pyvnc",
 ]
+# ------ dependencies ------
 
-[project.optional-dependencies]
+
+# TODO: add an `--only daemon` group for running non-ui / pikerd
+# service tree in distributed mode B)
+# https://docs.astral.sh/uv/concepts/projects/dependencies/#optional-dependencies
+
+[dependency-groups]
 uis = [
     # https://docs.astral.sh/uv/concepts/projects/dependencies/#optional-dependencies
     # TODO: make sure the levenshtein shit compiles on nix..
@@ -92,12 +96,11 @@ uis = [
     # for consideration,
     # - 'visidata'
 
-    # TODO: add an `--only daemon` group for running non-ui / pikerd
-    # service tree in distributed mode B)
-    # https://docs.astral.sh/uv/concepts/projects/dependencies/#optional-dependencies
+    "qdarkstyle >=3.0.2, <4.0.0",
+    "pyqt6 >=6.7.0, <7.0.0",
+    "pyqtgraph",
 ]
 
-[dependency-groups]
 # TODO: a toolset that makes debugging a `pikerd` service (tree) easy
 # to hack on directly using more or less the local env:
 # - xonsh + xxh
@@ -106,19 +109,46 @@ uis = [
 #
 # console ehancements and eventually remote debugging extras/helpers.
 # use `uv --dev` to enable
-dev = [
-    "pytest",
-    "elasticsearch >=8.9.0, <9.0.0",
-    "prompt-toolkit ==3.0.40",
-    "cython >=3.0.0, <4.0.0",
+repl = [
+  # debug
+  "pdbp >=1.5.0, <2.0.0",
     "greenback >=1.1.1, <2.0.0",
-    "ruff>=0.9.6",
-    "pyperclip>=1.9.0",
-    "i3ipc>=2.2.1",
-
-    # ?from git, see below.
   "xonsh",
+    "prompt-toolkit ==3.0.40",
+    "pyperclip>=1.9.0",
+
 ]
+testing = [
+  "pytest",
+]
+de = [
+  # DE-specific
+  "i3ipc>=2.2.1",
+]
+dev = [
+  # https://docs.astral.sh/uv/concepts/projects/dependencies/#development-dependencies
+  "cython >=3.0.0, <4.0.0",
+
+  # nested deps-groups
+  # https://docs.astral.sh/uv/concepts/projects/dependencies/#nesting-groups
+  {include-group = 'uis'},
+  {include-group = 'repl'},
+  {include-group = 'testing'},
+  {include-group = 'de'},
+]
+lint = [
+  # XXX, with flake.nix needs to be from nixpkgs
+  "ruff>=0.9.6" 
+  #
+  # ^TODO? these markers don't work; use deps-flags for now?
+  # ; os_name != 'nixos' and platform_system != 'NixOS'",
+  # ; defined('IN_NIX_SHELL')",
+]
+dbs = [
+  "elasticsearch >=8.9.0, <9.0.0",
+]
+# ------ dependency-groups ------
+
 
 [tool.pytest.ini_options]
 # https://docs.pytest.org/en/stable/reference/reference.html#configuration-options
@@ -131,24 +161,29 @@ console_output_style = 'progress'
 # https://docs.pytest.org/en/stable/how-to/plugins.html#disabling-plugins-from-autoloading
 # https://docs.pytest.org/en/stable/how-to/plugins.html#deactivating-unregistering-a-plugin-by-name
 addopts = '-p no:xonsh'
+# ------ tool.pytest ------
 
 
 [project.scripts]
 piker = "piker.cli:cli"
 pikerd = "piker.cli:pikerd"
 ledger = "piker.accounting.cli:ledger"
+# ------ project.scripts ------
 
 [tool.hatch.build.targets.sdist]
 include = ["piker"]
-
 [tool.hatch.build.targets.wheel]
 include = ["piker"]
+# ------ tool.hatch ------
 
 
 # TODO? move to a `uv.toml`?
 [tool.uv]
 python-preference = 'system'
 python-downloads = 'manual'
+# https://docs.astral.sh/uv/concepts/projects/dependencies/#default-groups
+default-groups = ['uis', 'dev']
+# ------ tool.uv ------
 
 
 [tool.uv.sources]