Go Python 3.10+ in anticipation of upcoming feature PRs

Contain mkts

.github/workflows/ci.yml

@@ -1,5 +1,6 @@
 name: CI
 
+
 on:
   # Triggers the workflow on push or pull request events but only for the master branch
   push:
@@ -10,41 +11,21 @@ on:
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
+
 jobs:
 
-  basic_install:
-    name: 'pip install'
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          ref: master
-
-      - name: Setup python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.9'
-
-      - name: Install dependencies
-        run: pip install -e . --upgrade-strategy eager -r requirements.txt
-
-      - name: Run piker cli
-        run: piker 
-
   testing:
-    name: 'test suite'
+    name: 'install + test-suite'
     runs-on: ubuntu-latest
     steps:
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
-          python-version: '3.9'
+          python-version: '3.10'
 
       - name: Install dependencies
         run: pip install -U . -r requirements-test.txt -r requirements.txt --upgrade-strategy eager

.gitignore

@@ -97,6 +97,9 @@ ENV/
 # mkdocs documentation
 /site
 
+# extra scripts dir
+/snippets
+
 # mypy
 .mypy_cache/
 .vscode/settings.json

README.rst

@@ -72,11 +72,73 @@ for a development install::
     pip install -r requirements.txt -e .
 
 
+install for tinas
+*****************
+for windows peeps you can start by installing all the prerequisite software:
+
+- install git with all default settings - https://git-scm.com/download/win
+- install anaconda all default settings - https://www.anaconda.com/products/individual
+- install microsoft build tools (check the box for Desktop development for C++, you might be able to uncheck some optional downloads)  - https://visualstudio.microsoft.com/visual-cpp-build-tools/
+- install visual studio code default settings - https://code.visualstudio.com/download
+
+
+then, `crack a conda shell`_ and run the following commands::
+
+    mkdir code # create code directory
+    cd code # change directory to code
+    git clone https://github.com/pikers/piker.git # downloads piker installation package from github
+    cd piker # change directory to piker
+    
+    conda create -n pikonda # creates conda environment named pikonda
+    conda activate pikonda # activates pikonda
+    
+    conda install -c conda-forge python-levenshtein # in case it is not already installed
+    conda install pip # may already be installed
+    pip # will show if pip is installed
+    
+    pip install -e . -r requirements.txt # install piker in editable mode
+
+test Piker to see if it is working::
+
+    piker -b binance chart btcusdt.binance # formatting for loading a chart
+    piker -b kraken -b binance chart xbtusdt.kraken
+    piker -b kraken -b binance -b ib chart qqq.nasdaq.ib
+    piker -b ib chart tsla.nasdaq.ib
+
+potential error::
+    
+    FileNotFoundError: [Errno 2] No such file or directory: 'C:\\Users\\user\\AppData\\Roaming\\piker\\brokers.toml'
+    
+solution:
+
+- navigate to file directory above (may be different on your machine, location should be listed in the error code)
+- copy and paste file from 'C:\\Users\\user\\code\\data/brokers.toml' or create a blank file using notepad at the location above
+
+Visual Studio Code setup:
+
+- now that piker is installed we can set up vscode as the default terminal for running piker and editing the code
+- open Visual Studio Code
+- file --> Add Folder to Workspace --> C:\Users\user\code\piker (adds piker directory where all piker files are located)
+- file --> Save Workspace As --> save it wherever you want and call it whatever you want, this is going to be your default workspace for running and editing piker code
+- ctrl + shift + p --> start typing Python: Select Interpetter --> when the option comes up select it --> Select at the workspace level --> select the one that shows ('pikonda')
+- change the default terminal to cmd.exe instead of powershell (default)
+- now when you create a new terminal VScode should automatically activate you conda env so that piker can be run as the first command after a new terminal is created
+
+also, try out fancyzones as part of powertoyz for a decent tiling windows manager to manage all the cool new software you are going to be running.
+
+.. _conda installed: https://
+.. _C++ build toolz: https://
+.. _crack a conda shell: https://
+.. _vscode: https://
+
+.. link to the tina guide
+.. _setup a coolio tiled wm console: https://
+
 provider support
 ****************
 for live data feeds the in-progress set of supported brokers is:
 
-- IB_ via ``ib_insync``
+- IB_ via ``ib_insync``, also see our `container docs`_
 - binance_ and kraken_ for crypto over their public websocket API
 - questrade_ (ish) which comes with effectively free L1
 
@@ -88,6 +150,7 @@ coming soon...
 if you want your broker supported and they have an API let us know.
 
 .. _IB: https://interactivebrokers.github.io/tws-api/index.html
+.. _container docs: https://github.com/pikers/piker/tree/master/dockering/ib
 .. _questrade: https://www.questrade.com/api/documentation
 .. _kraken: https://www.kraken.com/features/api#public-market-data
 .. _binance: https://github.com/pikers/piker/pull/182
@@ -98,12 +161,38 @@ if you want your broker supported and they have an API let us know.
 
 check out our charts
 ********************
-bet you weren't expecting this from the foss bby::
+bet you weren't expecting this from the foss::
 
     piker -l info -b kraken -b binance chart btcusdt.binance --pdb
 
 
-this runs the main chart in in debug mode.
+this runs the main chart (currently with 1m sampled OHLC) in in debug
+mode and you can practice paper trading using the following
+micro-manual:
+
+``order_mode`` (
+    edge triggered activation by any of the following keys,
+    ``mouse-click`` on y-level to submit at that price
+    ):
+
+    - ``f``/ ``ctl-f`` to stage buy
+    - ``d``/ ``ctl-d`` to stage sell
+    - ``a`` to stage alert
+
+
+``search_mode`` (
+    ``ctl-l`` or ``ctl-space`` to open,
+    ``ctl-c`` or ``ctl-space`` to close
+    ) :
+
+    - begin typing to have symbol search automatically lookup
+      symbols from all loaded backend (broker) providers
+    - arrow keys and mouse click to navigate selection
+    - vi-like ``ctl-[hjkl]`` for navigation
+
+
+you can also configure your position allocation limits from the
+sidepane.
 
 
 run in distributed mode
@@ -119,10 +208,10 @@ connect your chart::
     piker -l info -b kraken -b binance chart xmrusdt.binance --pdb
 
 
-enjoy persistent real-time data feeds tied to daemon lifetime.
-
-key-bindings and mouse interaction is currently only documented in the
-doce base. help us write some docs dawg.
+enjoy persistent real-time data feeds tied to daemon lifetime. the next
+time you spawn a chart it will load much faster since the data feed has
+been cached and is now always running live in the background until you
+kill ``pikerd``.
 
 
 if anyone asks you what this project is about
@@ -138,3 +227,5 @@ enter the matrix.
 how come there ain't that many docs
 ***********************************
 suck it up, learn the code; no one is trying to sell you on anything.
+also, we need lotsa help so if you want to start somewhere and can't
+necessarily write serious code, this might be the place for you!

config/brokers.toml

@@ -8,18 +8,45 @@ expires_at = 1616095326.355846
 
 [kraken]
 key_descr = "api_0"
-public_key = ""
-private_key = ""
+api_key = ""
+secret = ""
 
-[ib.api]
-ipaddr = "127.0.0.1"
+[ib]
+hosts = [
+    "127.0.0.1",
+]
+# XXX: the order in which ports will be scanned
+# (by the `brokerd` daemon-actor)
+# is determined # by the line order here.
+# TODO: when we eventually spawn gateways in our
+# container, we can just dynamically allocate these
+# using IBC.
+ports = [
+    4002,  # gw
+    7497,  # tws
+]
+
+# XXX: for a paper account the flex web query service
+# is not supported so you have to manually download
+# and XML report and put it in a location that can be
+# accessed by the ``brokerd.ib`` backend code for parsing.
+flex_token = '666666666666666666666666'
+flex_trades_query_id = '666666'  # live account
+
+# when clients are being scanned this determines
+# which clients are preferred to be used for data
+# feeds based on the order of account names, if
+# detected as active on an API client.
+prefer_data_account = [
+    'paper',
+    'margin',
+    'ira',
+]
 
 [ib.accounts]
-margin = ""
-registered = ""
-paper = ""
-
-[ib.api.ports]
-gw = 4002
-tws = 7497
-order = [ "gw", "tws",]
+# the order in which accounts will be selectable
+# in the order mode UI (if found via clients during
+# API-app scanning)when a new symbol is loaded.
+paper = "XX0000000"
+margin = "X0000000"
+ira = "X0000000"

data/brokers.toml

@@ -1,9 +0,0 @@
-[binance]
-
-[kraken]
-
-
-# [ib]
-
-
-# [questrade]

dockering/ib/README.rst

@@ -0,0 +1,30 @@
+running ``ib`` gateway in ``docker``
+------------------------------------
+We have a config based on the (now defunct)
+image from "waytrade":
+
+https://github.com/waytrade/ib-gateway-docker
+
+To startup this image with our custom settings
+simply run the command::
+
+    docker compose up
+
+And you should have the following socket-available services:
+
+- ``x11vnc1@127.0.0.1:3003``
+- ``ib-gw@127.0.0.1:4002``
+
+You can attach to the container via a VNC client
+without password auth.
+
+SECURITY STUFF!?!?!
+-------------------
+Though "``ib``" claims they host filter connections outside
+localhost (aka ``127.0.0.1``) it's probably better if you filter
+the socket at the OS level using a stateless firewall rule::
+
+    ip rule add not unicast iif lo to 0.0.0.0/0 dport 4002
+
+We will soon have this baked into our own custom image but for
+now you'll have to do it urself dawgy.

dockering/ib/docker-compose.yml

@@ -0,0 +1,64 @@
+# rework from the original @
+# https://github.com/waytrade/ib-gateway-docker/blob/master/docker-compose.yml
+version: "3.5"
+
+services:
+  ib-gateway:
+    # other image tags available:
+    # https://github.com/waytrade/ib-gateway-docker#supported-tags
+    image: waytrade/ib-gateway:981.3j
+    restart: always
+    network_mode: 'host'
+
+    volumes:
+      - type: bind
+        source: ./jts.ini
+        target: /root/Jts/jts.ini
+        # don't let IBC clobber this file for
+        # the main reason of not having a stupid
+        # timezone set..
+        read_only: true
+
+      # force our own IBC config
+      - type: bind
+        source: ./ibc.ini
+        target: /root/ibc/config.ini
+
+      # force our noop script - socat isn't needed in host mode.
+      - type: bind
+        source: ./fork_ports_delayed.sh
+        target: /root/scripts/fork_ports_delayed.sh
+
+      # force our noop script - socat isn't needed in host mode.
+      - type: bind
+        source: ./run_x11_vnc.sh
+        target: /root/scripts/run_x11_vnc.sh
+        read_only: true
+
+    # NOTE:to fill these out, define an `.env` file in the same dir as
+    # this compose file which looks something like:
+    # TWS_USERID='myuser'
+    # TWS_PASSWORD='guest'
+    # TRADING_MODE=paper (or live)
+    # VNC_SERVER_PASSWORD='diggity'
+
+    environment:
+      TWS_USERID: ${TWS_USERID}
+      TWS_PASSWORD: ${TWS_PASSWORD}
+      TRADING_MODE: ${TRADING_MODE:-paper}
+      VNC_SERVER_PASSWORD: ${VNC_SERVER_PASSWORD:-}
+
+    # ports:
+    #   - target: 4002
+    #     host_ip: 127.0.0.1
+    #     published: 4002
+    #     protocol: tcp
+
+      # original mappings for use in non-host-mode
+      # which we won't really need going forward since
+      # ideally we just pick the port to have ib-gw listen
+      # on **when** we spawn the container - i.e. everything
+      # will be driven by a ``brokers.toml`` def.
+      # - "127.0.0.1:4001:4001"
+      # - "127.0.0.1:4002:4002"
+      # - "127.0.0.1:5900:5900"

dockering/ib/fork_ports_delayed.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# we now just set this is to a noop script
+# since we can just run the container in
+# `network_mode: 'host'` and get literally
+# the exact same behaviour XD

dockering/ib/ibc.ini

@@ -0,0 +1,711 @@
+# Note that in the comments in this file, TWS refers to both the Trader
+# Workstation and the IB Gateway, unless explicitly stated otherwise.
+#
+# When referred to below, the default value for a setting is the value
+# assumed if either the setting is included but no value is specified, or
+# the setting is not included at all.
+#
+# IBC may also be used to start the FIX CTCI Gateway. All settings
+# relating to this have names prefixed with FIX.
+#
+# The IB API Gateway and the FIX CTCI Gateway share the same code. Which
+# gateway actually runs is governed by an option on the initial gateway
+# login screen. The FIX setting described under IBC Startup
+# Settings below controls this.
+
+
+
+# =============================================================================
+# 1.   IBC Startup Settings
+# =============================================================================
+
+
+# IBC may be used to start the IB Gateway for the FIX CTCI. This
+# setting must be set to 'yes' if you want to run the FIX CTCI gateway. The
+# default is 'no'.
+
+FIX=no
+
+
+
+# =============================================================================
+# 2.   Authentication Settings
+# =============================================================================
+
+# TWS and the IB API gateway require a single username and password.
+# You may specify the username and password using the following settings:
+#
+#	IbLoginId
+#	IbPassword
+#
+# Alternatively, you can specify the username and password in the command
+# files used to start TWS or the Gateway, but this is not recommended for
+# security reasons.
+#
+# If you don't specify them, you will be prompted for them in the usual
+# login dialog when TWS starts (but whatever you have specified will be
+# included in the dialog automatically: for example you may specify the
+# username but not the password, and then you will be prompted for the
+# password via the login dialog). Note that if you specify either
+# the username or the password (or both) in the command file, then
+# IbLoginId and IbPassword settings defined in this file are ignored.
+#
+#
+# The FIX CTCI gateway requires one username and password for FIX order
+# routing, and optionally a separate username and password for market
+# data connections. You may specify the usernames and passwords using
+# the following settings:
+#
+#	FIXLoginId
+#	FIXPassword
+#	IbLoginId	(optional - for market data connections)
+#	IbPassword	(optional - for market data connections)
+#
+# Alternatively you can specify the FIX username and password in the
+# command file used to start the FIX CTCI Gateway, but this is not
+# recommended for security reasons.
+#
+# If you don't specify them, you will be prompted for them in the usual
+# login dialog when FIX CTCI gateway starts (but whatever you have
+# specified will be included in the dialog automatically: for example
+# you may specify the usernames but not the passwords, and then you will
+# be prompted for the passwords via the login dialog). Note that if you
+# specify either the FIX username or the FIX password (or both) on the
+# command line, then FIXLoginId and FIXPassword settings defined in this
+# file are ignored; he same applies to the market data username and
+# password.
+
+# IB API Authentication Settings
+# ------------------------------
+
+# Your TWS username:
+
+IbLoginId=
+
+
+# Your TWS password:
+
+IbPassword=
+
+
+# FIX CTCI Authentication Settings
+# --------------------------------
+
+# Your FIX CTCI username:
+
+FIXLoginId=
+
+
+# Your FIX CTCI password:
+
+FIXPassword=
+
+
+# Second Factor Authentication Settings
+# -------------------------------------
+
+# If you have enabled more than one second factor authentication
+# device, TWS presents a list from which you must select the device
+# you want to use for this login. You can use this setting to
+# instruct IBC to select a particular item in the list on your
+# behalf. Note that you must spell this value exactly as it appears
+# in the list. If no value is set, you must manually select the
+# relevant list entry.
+
+SecondFactorDevice=
+
+
+# If you use the IBKR Mobile app for second factor authentication,
+# and you fail to complete the process before the time limit imposed
+# by IBKR, you can use this setting to tell IBC to exit: arrangements
+# can then be made to automatically restart IBC in order to initiate
+# the login sequence afresh. Otherwise, manual intervention at TWS's
+# Second Factor Authentication dialog is needed to complete the
+# login.
+#
+# Permitted values are 'yes' and 'no'. The default is 'no'.
+#
+# Note that the scripts provided with the IBC zips for Windows and
+# Linux provide options to automatically restart in these
+# circumstances, but only if this setting is also set to 'yes'.
+
+ExitAfterSecondFactorAuthenticationTimeout=no
+
+
+# This setting is only relevant if
+# ExitAfterSecondFactorAuthenticationTimeout is set to 'yes'.
+#
+# It controls how long (in seconds) IBC waits for login to complete
+# after the user acknowledges the second factor authentication
+# alert at the IBKR Mobile app. If login has not completed after
+# this time, IBC terminates.
+# The default value is 40.
+
+SecondFactorAuthenticationExitInterval=
+
+
+# Trading Mode
+# ------------
+#
+# TWS 955 introduced a new Trading Mode combo box on its login
+# dialog. This indicates whether the live account or the paper
+# trading account corresponding to the supplied credentials is
+# to be used. The allowed values are 'live' (the default) and
+# 'paper'. For earlier versions of TWS this setting has no
+# effect.
+
+TradingMode=
+
+
+# Paper-trading Account Warning
+# -----------------------------
+#
+# Logging in to a paper-trading account results in TWS displaying
+# a dialog asking the user to confirm that they are aware that this
+# is not a brokerage account. Until this dialog has been accepted,
+# TWS will not allow API connections to succeed. Setting this
+# to 'yes' (the default) will cause IBC to automatically
+# confirm acceptance. Setting it to 'no' will leave the dialog
+# on display, and the user will have to deal with it manually.
+
+AcceptNonBrokerageAccountWarning=yes
+
+
+# Login Dialog Display Timeout
+#-----------------------------
+#
+# In some circumstances, starting TWS may result in failure to display
+# the login dialog. Restarting TWS may help to resolve this situation,
+# and IBC does this automatically.
+#
+# This setting controls how long (in seconds) IBC waits for the login
+# dialog to appear before restarting TWS.
+#
+# Note that in normal circumstances with a reasonably specified 
+# computer the time to displaying the login dialog is typically less
+# than 20 seconds, and frequently much less. However many factors can
+# influence this, and it is unwise to set this value too low.
+#
+# The default value is 60.
+
+LoginDialogDisplayTimeout = 60
+
+
+
+# =============================================================================
+# 3.   TWS Startup Settings
+# =============================================================================
+
+# Path to settings store
+# ----------------------
+#
+# Path to the directory where TWS should store its settings. This is
+# normally the folder in which TWS is installed. However you may set
+# it to some other location if you wish (for example if you want to
+# run multiple instances of TWS with different settings).
+#
+# It is recommended for clarity that you use an absolute path. The
+# effect of using a relative path is undefined.
+#
+# Linux and macOS users should use the appropriate path syntax.
+#
+# Note that, for Windows users, you MUST use double separator
+# characters to separate the elements of the folder path: for
+# example, IbDir=C:\\IBLiveSettings is valid, but
+# IbDir=C:\IBLiveSettings is NOT valid and will give unexpected
+# results. Linux and macOS users need not use double separators,
+# but they are acceptable.
+#
+# The default is the current working directory when IBC is
+# started.
+
+IbDir=/root/Jts
+
+
+# Store settings on server
+# ------------------------
+#
+# If you wish to store a copy of your TWS settings on IB's
+# servers as well as locally on your computer, set this to
+# 'yes': this enables you to run TWS on different computers
+# with the same configuration, market data lines, etc. If set
+# to 'no', running TWS on different computers will not share the
+# same settings. If no value is specified, TWS will obtain its
+# settings from the same place as the last time this user logged
+# in (whether manually or using IBC).
+
+StoreSettingsOnServer=
+
+
+# Minimize TWS on startup
+# -----------------------
+#
+# Set to 'yes' to minimize TWS when it starts:
+
+MinimizeMainWindow=no
+
+
+# Existing Session Detected Action
+# --------------------------------
+#
+# When a user logs on to an IBKR account for trading purposes by any means, the
+# IBKR account server checks to see whether the account is already logged in
+# elsewhere. If so, a dialog is displayed to both the users that enables them
+# to determine what happens next. The 'ExistingSessionDetectedAction' setting
+# instructs TWS how to proceed when it displays this dialog:
+#
+#   * If the new TWS session is set to 'secondary', the existing session continues
+#     and the new session terminates. Thus a secondary TWS session can never
+#     override any other session.
+#
+#   * If the existing TWS session is set to 'primary', the existing session
+#     continues and the new session terminates (even if the new session is also
+#     set to primary). Thus a primary TWS session can never be overridden by
+#     any new session).
+#
+#   * If both the existing and the new TWS sessions are set to 'primaryoverride',
+#     the existing session terminates and the new session proceeds.
+#
+#   * If the existing TWS session is set to 'manual', the user must handle the
+#     dialog.
+#
+# The difference between 'primary' and 'primaryoverride' is that a
+# 'primaryoverride' session can be overriden over by a new 'primary' session,
+# but a 'primary' session cannot be overriden by any other session.
+#
+# When set to 'primary', if another TWS session is started and manually told to
+# end the 'primary' session, the 'primary' session is automatically reconnected.
+#
+# The default is 'manual'.
+
+ExistingSessionDetectedAction=primary
+
+
+# Override TWS API Port Number
+# ----------------------------
+#
+# If OverrideTwsApiPort is set to an integer, IBC changes the
+# 'Socket port' in TWS's API configuration to that number shortly
+# after startup. Leaving the setting blank will make no change to
+# the current setting. This setting is only intended for use in
+# certain specialized situations where the port number needs to
+# be set dynamically at run-time: most users will never need it,
+# so don't use it unless you know you need it.
+
+OverrideTwsApiPort=4002
+
+
+# Read-only Login
+# ---------------
+#
+# If ReadOnlyLogin is set to 'yes', and the user is enrolled in IB's
+# account security programme, the user will not be asked to perform
+# the second factor authentication action, and login to TWS will
+# occur automatically in read-only mode: in this mode, placing or
+# managing orders is not allowed. If set to 'no', and the user is
+# enrolled in IB's account security programme, the user must perform
+# the relevant second factor authentication action to complete the
+# login. 
+
+# If the user is not enrolled in IB's account security programme,
+# this setting is ignored. The default is 'no'.
+
+ReadOnlyLogin=no
+
+
+# Read-only API
+# -------------
+#
+# If ReadOnlyApi is set to 'yes', API programs cannot submit, modify
+# or cancel orders. If set to 'no', API programs can do these things.
+# If not set, the existing TWS/Gateway configuration is unchanged.
+# NB: this setting is really only supplied for the benefit of new TWS
+# or Gateway instances that are being automatically installed and
+# started without user intervention (eg Docker containers). Where
+# a user is involved, they should use the Global Configuration to
+# set the relevant checkbox (this only needs to be done once) and
+# not provide a value for this setting.
+
+ReadOnlyApi=no
+
+
+# Market data size for US stocks - lots or shares
+# -----------------------------------------------
+#
+# Since IB introduced the option of market data for US stocks showing
+# bid, ask and last sizes in shares rather than lots, TWS and Gateway
+# display a dialog immediately after login notifying the user about
+# this and requiring user input before allowing market data to be
+# accessed. The user can request that the dialog not be shown again.
+#
+# It is recommended that the user should handle this dialog manually
+# rather than using these settings, which are provided for situations 
+# where the user interface is not easily accessible, or where user
+# settings are not preserved between sessions (eg some Docker images).
+#
+# - If this setting is set to 'accept', the dialog will be handled
+#   automatically and the option to not show it again will be
+#   selected.
+#
+#   Note that in this case, the only way to allow the dialog to be
+#   displayed again is to manually enable the 'Bid, Ask and Last
+#   Size Display Update' message in the 'Messages' section of the TWS
+#   configuration dialog. So you should only use 'Accept' if you are
+#   sure you really don't want the dialog to be displayed again, or
+#   you have easy access to the user interface.
+#
+# - If set to 'defer', the dialog will be handled automatically (so
+#   that market data will start), but the option to not show it again
+#   will not be selected, and it will be shown again after the next
+#   login.
+#
+# - If set to 'ignore', the user has to deal with the dialog manually.
+#
+# The default value is 'ignore'.
+#
+# Note if set to 'accept' or 'defer', TWS also automatically sets
+# the API settings checkbox labelled 'Send market data in lots for
+# US stocks for dual-mode API clients'. IBC cannot prevent this.
+# However you can change this immmediately by setting
+# SendMarketDataInLotsForUSstocks (see below) to 'no' .
+
+AcceptBidAskLastSizeDisplayUpdateNotification=accept
+
+
+# This setting determines whether the API settings checkbox labelled
+# 'Send market data in lots for US stocks for dual-mode API clients'
+# is set or cleared. If set to 'yes', the checkbox is set. If set to
+# 'no' the checkbox is cleared. If defaulted, the checkbox is
+# unchanged.
+
+SendMarketDataInLotsForUSstocks=
+
+
+
+# =============================================================================
+# 4.   TWS Auto-Closedown
+# =============================================================================
+#
+# IMPORTANT NOTE: Starting with TWS 974, this setting no longer
+# works properly, because IB have changed the way TWS handles its
+# autologoff mechanism. 
+#
+# You should now configure the TWS autologoff time to something
+# convenient for you, and restart IBC each day.
+#
+# Alternatively, discontinue use of IBC and use the auto-relogin
+# mechanism within TWS 974 and later versions (note that the 
+# auto-relogin mechanism provided by IB is not available if you
+# use IBC).
+
+# Set to yes or no (lower case).
+#
+#   yes   means allow TWS to shut down automatically at its
+# 	  specified shutdown time, which is set via the TWS
+#	  configuration menu.
+#
+#   no    means TWS never shuts down automatically.
+#
+# NB: IB recommends that you do not keep TWS running
+# continuously. If you set this setting to 'no', you may
+# experience incorrect TWS operation.
+#
+# NB: the default for this setting is 'no'. Since this will
+# only work properly with TWS versions earlier than 974, you
+# should explicitly set this to 'yes' for version 974 and later.
+
+IbAutoClosedown=yes
+
+
+
+# =============================================================================
+# 5.   TWS Tidy Closedown Time
+# =============================================================================
+#
+# NB: starting with TWS 974 this is no longer a useful option
+# because both TWS and Gateway now have the same auto-logoff
+# mechanism, and IBC can no longer avoid this.
+#
+# Note that giving this setting a value does not change TWS's
+# auto-logoff in any way: any setting will be additional to the
+# TWS auto-logoff.
+#
+# To tell IBC to tidily close TWS at a specified time every
+# day, set this value to <hh:mm>, for example:
+# ClosedownAt=22:00
+#
+# To tell IBC to tidily close TWS at a specified day and time
+# each week, set this value to <dayOfWeek hh:mm>, for example:
+# ClosedownAt=Friday 22:00
+#
+# Note that the day of the week must be specified using your
+# default locale. Also note that Java will only accept
+# characters encoded to ISO 8859-1 (Latin-1). This means that
+# if the day name in your default locale uses any non-Latin-1
+# characters you need to encode them using Unicode escapes
+# (see http://java.sun.com/docs/books/jls/third_edition/html/lexical.html#3.3
+# for details). For example, to tidily close TWS at 12:00 on
+# Saturday where the default locale is Simplified Chinese,
+# use the following:
+# #ClosedownAt=\u661F\u671F\u516D 12:00
+
+ClosedownAt=
+
+
+
+# =============================================================================
+# 6.   Other TWS Settings
+# =============================================================================
+
+# Accept Incoming Connection
+# --------------------------
+#
+# If set to 'accept', IBC automatically accepts incoming
+# API connection dialogs. If set to 'reject', IBC
+# automatically rejects incoming API connection dialogs. If
+# set to 'manual', the user must decide whether to accept or reject
+# incoming API connection dialogs. The default is 'manual'.
+# NB: it is recommended to set this to 'reject', and to explicitly
+# configure which IP addresses can connect to the API in TWS's API
+# configuration page, as this is much more secure (in this case, no
+# incoming API connection dialogs will occur for those IP addresses).
+
+AcceptIncomingConnectionAction=reject
+
+
+# Allow Blind Trading
+# -------------------
+#
+# If you attempt to place an order for a contract for which
+# you have no market data subscription, TWS displays a dialog
+# to warn you against such blind trading.
+#
+#   yes   means the dialog is dismissed as though the user had
+# 	  clicked the 'Ok' button: this means that you accept
+# 	  the risk and want the order to be submitted.
+#
+#   no    means the dialog remains on display and must be
+#         handled by the user.
+
+AllowBlindTrading=yes
+
+
+# Save Settings on a Schedule
+# ---------------------------
+#
+# You can tell TWS to automatically save its settings on a schedule
+# of your choosing. You can specify one or more specific times,
+# like this:
+#
+# SaveTwsSettingsAt=HH:MM [ HH:MM]...
+#
+# for example:
+# SaveTwsSettingsAt=08:00   12:30 17:30
+#
+# Or you can specify an interval at which settings are to be saved,
+# optionally starting at a specific time and continuing until another
+# time, like this:
+#
+#SaveTwsSettingsAt=Every n [{mins | hours}] [hh:mm] [hh:mm]
+#
+# where the first hh:mm is the start time and the second is the end
+# time. If you don't specify the end time, settings are saved regularly
+# from the start time till midnight. If you don't specify the start time.
+# settings are saved regularly all day, beginning at 00:00. Note that
+# settings will always be saved at the end time, even if that is not
+# exactly one interval later than the previous time. If neither 'mins'
+# nor 'hours' is specified, 'mins' is assumed. Examples:
+#
+# To save every 30 minutes all day starting at 00:00
+#SaveTwsSettingsAt=Every 30
+#SaveTwsSettingsAt=Every 30 mins
+#
+# To save every hour starting at 08:00 and ending at midnight
+#SaveTwsSettingsAt=Every 1 hours 08:00
+#SaveTwsSettingsAt=Every 1 hours 08:00 00:00
+#
+# To save every 90 minutes starting at 08:00 up to and including 17:43
+#SaveTwsSettingsAt=Every 90 08:00 17:43
+
+SaveTwsSettingsAt=
+
+
+
+# =============================================================================
+# 7.   Settings Specific to Indian Versions of TWS
+# =============================================================================
+
+# Indian versions of TWS may display a password expiry
+# notification dialog and a NSE Compliance dialog. These can be
+# dismissed by setting the following to yes. By default the
+# password expiry notice is not dismissed, but the NSE Compliance
+# notice is dismissed.
+
+# Warning: setting DismissPasswordExpiryWarning=yes will mean
+# you will not be notified when your password is about to expire.
+# You must then take other measures to ensure that your password
+# is changed within the expiry period, otherwise IBC will
+# not be able to login successfully.
+
+DismissPasswordExpiryWarning=no
+DismissNSEComplianceNotice=yes
+
+
+
+# =============================================================================
+# 8.   IBC Command Server Settings
+# =============================================================================
+
+# Do NOT CHANGE THE FOLLOWING SETTINGS unless you
+# intend to issue commands to IBC (for example
+# using telnet). Note that these settings have nothing to
+# do with running programs that use the TWS API.
+
+# Command Server Port Number
+# --------------------------
+#
+# The port number that IBC listens on for commands
+# such as "STOP". DO NOT set this to the port number
+# used for TWS API connections. There is no good reason
+# to change this setting unless the port is used by
+# some other application (typically another instance of
+# IBC). The default value is 0, which tells IBC not to
+# start the command server
+
+#CommandServerPort=7462
+
+
+# Permitted Command Sources
+# -------------------------
+#
+# A comma separated list of IP addresses, or host names,
+# which are allowed addresses for sending commands to
+# IBC.  Commands can always be sent from the
+# same host as IBC is running on.
+
+ControlFrom=127.0.0.1
+
+
+# Address for Receiving Commands
+# ------------------------------
+#
+# Specifies the IP address on which the Command Server
+# is so listen. For a multi-homed host, this can be used
+# to specify that connection requests are only to be
+# accepted on the specified address. The default is to
+# accept connection requests on all local addresses.
+
+BindAddress=127.0.0.1
+
+
+# Command Prompt
+# --------------
+#
+# The specified string is output by the server when
+# the connection is first opened and after the completion
+# of each command. This can be useful if sending commands
+# using an interactive program such as telnet. The default
+# is that no prompt is output.
+# For example:
+#
+# CommandPrompt=>
+
+CommandPrompt=
+
+
+# Suppress Command Server Info Messages
+# -------------------------------------
+#
+# Some commands can return intermediate information about
+# their progress. This setting controls whether such
+# information is sent. The default is that such information
+# is not sent.
+
+SuppressInfoMessages=no
+
+
+
+# =============================================================================
+# 9.   Diagnostic Settings
+# =============================================================================
+#
+# IBC can log information about the structure of windows
+# displayed by TWS. This information is useful when adding
+# new features to IBC or when behaviour is not as expected. 
+#
+# The logged information shows the hierarchical organisation
+# of all the components of the window, and includes the
+# current values of text boxes and labels.
+#
+# Note that this structure logging has a small performance
+# impact, and depending on the settings can cause the logfile
+# size to be significantly increased. It is therefore
+# recommended that the LogStructureWhen setting be set to
+# 'never' (the default) unless there is a specific reason
+# that this information is needed.
+
+
+# Scope of Structure Logging
+# --------------------------
+#
+# The LogStructureScope setting indicates which windows are
+# eligible for structure logging:
+#
+#    - if set to 'known', only windows that IBC recognizes
+#      are eligible - these are windows that IBC has some
+#      interest in monitoring, usually to take some action
+#      on the user's behalf;
+#
+#    - if set to 'unknown', only windows that IBC does not
+#      recognize are eligible. Most windows displayed by
+#      TWS fall into this category;
+#
+#    - if set to 'untitled', only windows that IBC does not
+#      recognize and that have no title are eligible. These
+#      are usually message boxes or similar small windows,
+#
+#    - if set to 'all', then every window displayed by TWS
+#      is eligible.
+#
+# The default value is 'known'.
+
+LogStructureScope=all
+
+
+# When to Log Window Structure
+# ----------------------------
+#
+# The LogStructureWhen setting specifies the circumstances
+# when eligible TWS windows have their structure logged:
+#
+#     - if set to 'open' or 'yes' or 'true', IBC logs the
+#       structure of an eligible window the first time it
+#       is encountered;
+#
+#    - if set to 'activate', the structure is logged every
+#      time an eligible window is made active;
+#
+#    - if set to 'never' or 'no' or 'false', structure
+#      information is never logged.
+#
+# The default value is 'never'.
+
+LogStructureWhen=never
+
+
+# DEPRECATED SETTING
+# ------------------
+#
+# LogComponents - THIS SETTING WILL BE REMOVED IN A FUTURE
+# RELEASE
+#
+# If LogComponents is set to any value, this is equivalent
+# to setting LogStructureWhen to that same value and
+# LogStructureScope to 'all': the actual values of those
+# settings are ignored. The default is that the values
+# of LogStructureScope and LogStructureWhen are honoured.
+
+#LogComponents=
+
+
+

dockering/ib/jts.ini

@@ -0,0 +1,33 @@
+[IBGateway]
+ApiOnly=true
+LocalServerPort=4002
+# NOTE: must be set if using IBC's "reject" mode
+TrustedIPs=127.0.0.1
+; RemoteHostOrderRouting=ndc1.ibllc.com
+; WriteDebug=true
+; RemotePortOrderRouting=4001
+; useRemoteSettings=false
+; tradingMode=p
+; Steps=8
+; colorPalletName=dark
+
+# window geo, this may be useful for sending `xdotool` commands?
+; MainWindow.Width=1986
+; screenHeight=3960
+
+
+[Logon]
+Locale=en
+# most markets are oriented around this zone
+# so might as well hard code it.
+TimeZone=America/New_York
+UseSSL=true
+displayedproxymsg=1
+os_titlebar=true
+s3store=true
+useRemoteSettings=false
+
+[Communication]
+ctciAutoEncrypt=true
+Region=usr
+; Peer=cdc1.ibllc.com:4001

dockering/ib/run_x11_vnc.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# start VNC server
+x11vnc \
+    -ncache_cr \
+    -listen localhost \
+    -display :1 \
+    -forever \
+    -shared \
+    -logappend /var/log/x11vnc.log \
+    -bg \
+    -noipv6 \
+    -autoport 3003 \
+    # can't use this because of ``asyncvnc`` issue:
+    # https://github.com/barneygale/asyncvnc/issues/1
+    # -passwd 'ibcansmbz'

notes_to_self.rst

@@ -0,0 +1,28 @@
+Notes to self
+=============
+chicken scratch we shan't forget, consider this staging
+for actual feature issues on wtv git wrapper-provider we're
+using (no we shan't stick with GH long term likely).
+
+
+cool chart features
+-------------------
+- allow right-click to spawn shell with current in view
+  data passed to the new process via ``msgpack-numpy``.
+- expand OHLC datum to lower time frame.
+- auto-highlight current time range on tick feed
+
+
+features from IB charting
+-------------------------
+- vlm diffing from ticks and compare when bar arrives from historical
+  - should help isolate dark vlm / trades
+
+
+chart ux ideas
+--------------
+- hotkey to zoom to order intersection (horizontal line) with previous
+  price levels (+ some margin obvs).
+- L1 "lines" (queue size repr) should normalize to some fixed x width
+  such that when levels with more vlm appear other smaller levels are
+  scaled down giving an immediate indication of the liquidity diff.

piker/__init__.py

@@ -18,10 +18,3 @@
 piker: trading gear for hackers.
 
 """
-import msgpack  # noqa
-
-# TODO: remove this now right?
-import msgpack_numpy
-
-# patch msgpack for numpy arrays
-msgpack_numpy.patch()

piker/_cacheables.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+# Copyright (C) Tyler Goodlet (in stewardship for piker0)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,11 +15,22 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 """
-Async utils no one seems to have built into a core lib (yet).
+Cacheing apis and toolz.
+
 """
-from typing import AsyncContextManager
+
 from collections import OrderedDict
-from contextlib import asynccontextmanager
+from contextlib import (
+    asynccontextmanager,
+)
+
+from tractor.trionics import maybe_open_context
+
+from .brokers import get_brokermod
+from .log import get_logger
+
+
+log = get_logger(__name__)
 
 
 def async_lifo_cache(maxsize=128):
@@ -52,15 +63,17 @@ def async_lifo_cache(maxsize=128):
 
 
 @asynccontextmanager
-async def _just_none():
-    # noop -> skip entering context
-    yield None
+async def open_cached_client(
+    brokername: str,
+) -> 'Client':  # noqa
+    '''
+    Get a cached broker client from the current actor's local vars.
 
+    If one has not been setup do it and cache it.
 
-@asynccontextmanager
-async def maybe_with_if(
-    predicate: bool,
-    context: AsyncContextManager,
-) -> AsyncContextManager:
-    async with context if predicate else _just_none() as output:
-        yield output
+    '''
+    brokermod = get_brokermod(brokername)
+    async with maybe_open_context(
+        acm_func=brokermod.get_client,
+    ) as (cache_hit, client):
+        yield client

piker/_daemon.py

@@ -19,7 +19,7 @@ Structured, daemon tree service management.
 
 """
 from typing import Optional, Union, Callable, Any
-from contextlib import asynccontextmanager
+from contextlib import asynccontextmanager as acm
 from collections import defaultdict
 
 from pydantic import BaseModel
@@ -34,9 +34,11 @@ from .brokers import get_brokermod
 log = get_logger(__name__)
 
 _root_dname = 'pikerd'
+
+_registry_addr = ('127.0.0.1', 6116)
 _tractor_kwargs: dict[str, Any] = {
     # use a different registry addr then tractor's default
-    'arbiter_addr':  ('127.0.0.1', 6116),
+    'arbiter_addr': _registry_addr
 }
 _root_modules = [
     __name__,
@@ -47,7 +49,7 @@ _root_modules = [
 
 class Services(BaseModel):
 
-    actor_n: tractor._trionics.ActorNursery
+    actor_n: tractor._supervise.ActorNursery
     service_n: trio.Nursery
     debug_mode: bool  # tractor sub-actor debug mode flag
     service_tasks: dict[str, tuple[trio.CancelScope, tractor.Portal]] = {}
@@ -78,7 +80,6 @@ class Services(BaseModel):
         ) -> Any:
 
             with trio.CancelScope() as cs:
-
                 async with portal.open_context(
                     target,
                     **kwargs,
@@ -87,19 +88,21 @@ class Services(BaseModel):
 
                     # unblock once the remote context has started
                     task_status.started((cs, first))
-
-                    # wait on any context's return value
-                    ctx_res = await ctx.result()
                     log.info(
-                        f'`pikerd` service {name} started with value {ctx_res}'
+                        f'`pikerd` service {name} started with value {first}'
                     )
-
-                # wait on any error from the sub-actor
-                # NOTE: this will block indefinitely until cancelled
-                # either by error from the target context function or
-                # by being cancelled here by the surroundingn cancel
-                # scope
-                return await (portal.result(), ctx_res)
+                    try:
+                        # wait on any context's return value
+                        ctx_res = await ctx.result()
+                    except tractor.ContextCancelled:
+                        return await self.cancel_service(name)
+                    else:
+                        # wait on any error from the sub-actor
+                        # NOTE: this will block indefinitely until
+                        # cancelled either by error from the target
+                        # context function or by being cancelled here by
+                        # the surrounding cancel scope
+                        return (await portal.result(), ctx_res)
 
         cs, first = await self.service_n.start(open_context_in_task)
 
@@ -109,14 +112,17 @@ class Services(BaseModel):
 
         return cs, first
 
+    # TODO: per service cancellation by scope, we aren't using this
+    # anywhere right?
     async def cancel_service(
         self,
         name: str,
-
     ) -> Any:
-
         log.info(f'Cancelling `pikerd` service {name}')
         cs, portal = self.service_tasks[name]
+        # XXX: not entirely sure why this is required,
+        # and should probably be better fine tuned in
+        # ``tractor``?
         cs.cancel()
         return await portal.cancel_actor()
 
@@ -124,7 +130,7 @@ class Services(BaseModel):
 _services: Optional[Services] = None
 
 
-@asynccontextmanager
+@acm
 async def open_pikerd(
     start_method: str = 'trio',
     loglevel: Optional[str] = None,
@@ -150,7 +156,7 @@ async def open_pikerd(
         tractor.open_root_actor(
 
             # passed through to ``open_root_actor``
-            arbiter_addr=_tractor_kwargs['arbiter_addr'],
+            arbiter_addr=_registry_addr,
             name=_root_dname,
             loglevel=loglevel,
             debug_mode=debug_mode,
@@ -179,7 +185,48 @@ async def open_pikerd(
             yield _services
 
 
-@asynccontextmanager
+@acm
+async def open_piker_runtime(
+    name: str,
+    enable_modules: list[str] = [],
+    start_method: str = 'trio',
+    loglevel: Optional[str] = None,
+
+    # XXX: you should pretty much never want debug mode
+    # for data daemons when running in production.
+    debug_mode: bool = False,
+
+) -> Optional[tractor._portal.Portal]:
+    '''
+    Start a piker actor who's runtime will automatically
+    sync with existing piker actors in local network
+    based on configuration.
+
+    '''
+    global _services
+    assert _services is None
+
+    # XXX: this may open a root actor as well
+    async with (
+        tractor.open_root_actor(
+
+            # passed through to ``open_root_actor``
+            arbiter_addr=_registry_addr,
+            name=name,
+            loglevel=loglevel,
+            debug_mode=debug_mode,
+            start_method=start_method,
+
+            # TODO: eventually we should be able to avoid
+            # having the root have more then permissions to
+            # spawn other specialized daemons I think?
+            enable_modules=_root_modules,
+        ) as _,
+    ):
+        yield tractor.current_actor()
+
+
+@acm
 async def maybe_open_runtime(
     loglevel: Optional[str] = None,
     **kwargs,
@@ -202,7 +249,7 @@ async def maybe_open_runtime(
         yield
 
 
-@asynccontextmanager
+@acm
 async def maybe_open_pikerd(
     loglevel: Optional[str] = None,
     **kwargs,
@@ -253,7 +300,36 @@ class Brokerd:
     locks = defaultdict(trio.Lock)
 
 
-@asynccontextmanager
+@acm
+async def find_service(
+    service_name: str,
+) -> Optional[tractor.Portal]:
+
+    log.info(f'Scanning for service `{service_name}`')
+    # attach to existing daemon by name if possible
+    async with tractor.find_actor(
+        service_name,
+        arbiter_sockaddr=_registry_addr,
+    ) as maybe_portal:
+        yield maybe_portal
+
+
+async def check_for_service(
+    service_name: str,
+
+) -> bool:
+    '''
+    Service daemon "liveness" predicate.
+
+    '''
+    async with tractor.query_actor(
+        service_name,
+        arbiter_sockaddr=_registry_addr,
+    ) as sockaddr:
+        return sockaddr
+
+
+@acm
 async def maybe_spawn_daemon(
 
     service_name: str,
@@ -263,7 +339,7 @@ async def maybe_spawn_daemon(
     **kwargs,
 
 ) -> tractor.Portal:
-    """
+    '''
     If no ``service_name`` daemon-actor can be found,
     spawn one in a local subactor and return a portal to it.
 
@@ -274,7 +350,7 @@ async def maybe_spawn_daemon(
     This can be seen as a service starting api for remote-actor
     clients.
 
-    """
+    '''
     if loglevel:
         get_console_log(loglevel)
 
@@ -283,13 +359,14 @@ async def maybe_spawn_daemon(
     lock = Brokerd.locks[service_name]
     await lock.acquire()
 
-    # attach to existing brokerd if possible
-    async with tractor.find_actor(service_name) as portal:
+    async with find_service(service_name) as portal:
         if portal is not None:
             lock.release()
             yield portal
             return
 
+    log.warning(f"Couldn't find any existing {service_name}")
+
     # ask root ``pikerd`` daemon to spawn the daemon we need if
     # pikerd is not live we now become the root of the
     # process tree
@@ -325,6 +402,7 @@ async def maybe_spawn_daemon(
         async with tractor.wait_for_actor(service_name) as portal:
             lock.release()
             yield portal
+            await portal.cancel_actor()
 
 
 async def spawn_brokerd(
@@ -348,9 +426,19 @@ async def spawn_brokerd(
 
     # ask `pikerd` to spawn a new sub-actor and manage it under its
     # actor nursery
+    modpath = brokermod.__name__
+    broker_enable = [modpath]
+    for submodname in getattr(
+        brokermod,
+        '__enable_modules__',
+        [],
+    ):
+        subpath = f'{modpath}.{submodname}'
+        broker_enable.append(subpath)
+
     portal = await _services.actor_n.start_actor(
         dname,
-        enable_modules=_data_mods + [brokermod.__name__],
+        enable_modules=_data_mods + broker_enable,
         loglevel=loglevel,
         debug_mode=_services.debug_mode,
         **tractor_kwargs
@@ -368,7 +456,7 @@ async def spawn_brokerd(
     return True
 
 
-@asynccontextmanager
+@acm
 async def maybe_spawn_brokerd(
 
     brokername: str,
@@ -376,7 +464,9 @@ async def maybe_spawn_brokerd(
     **kwargs,
 
 ) -> tractor.Portal:
-    '''Helper to spawn a brokerd service.
+    '''
+    Helper to spawn a brokerd service *from* a client
+    who wishes to use the sub-actor-daemon.
 
     '''
     async with maybe_spawn_daemon(
@@ -428,7 +518,7 @@ async def spawn_emsd(
     return True
 
 
-@asynccontextmanager
+@acm
 async def maybe_open_emsd(
 
     brokername: str,
@@ -447,3 +537,25 @@ async def maybe_open_emsd(
 
     ) as portal:
         yield portal
+
+
+# TODO: ideally we can start the tsdb "on demand" but it's
+# probably going to require "rootless" docker, at least if we don't
+# want to expect the user to start ``pikerd`` with root perms all the
+# time.
+# async def maybe_open_marketstored(
+#     loglevel: Optional[str] = None,
+#     **kwargs,
+
+# ) -> tractor._portal.Portal:  # noqa
+
+#     async with maybe_spawn_daemon(
+
+#         'marketstored',
+#         service_task_target=spawn_emsd,
+#         spawn_args={'loglevel': loglevel},
+#         loglevel=loglevel,
+#         **kwargs,
+
+#     ) as portal:
+#         yield portal

piker/_profile.py

@@ -21,7 +21,10 @@ Profiling wrappers for internal libs.
 import time
 from functools import wraps
 
+# NOTE: you can pass a flag to enable this:
+# ``piker chart <args> --profile``.
 _pg_profile: bool = False
+ms_slower_then: float = 0
 
 
 def pg_profile_enabled() -> bool:

piker/brokers/_util.py

@@ -33,13 +33,49 @@ class SymbolNotFound(BrokerError):
 
 
 class NoData(BrokerError):
-    "Symbol data not permitted"
+    '''
+    Symbol data not permitted or no data
+    for time range found.
+
+    '''
+    def __init__(
+        self,
+        *args,
+        frame_size: int = 1000,
+
+    ) -> None:
+        super().__init__(*args)
+
+        # when raised, machinery can check if the backend
+        # set a "frame size" for doing datetime calcs.
+        self.frame_size: int = 1000
+
+
+class DataUnavailable(BrokerError):
+    '''
+    Signal storage requests to terminate.
+
+    '''
+    # TODO: add in a reason that can be displayed in the
+    # UI (for eg. `kraken` is bs and you should complain
+    # to them that you can't pull more OHLC data..)
+
+
+class DataThrottle(BrokerError):
+    '''
+    Broker throttled request rate for data.
+
+    '''
+    # TODO: add in throttle metrics/feedback
+
 
 
 def resproc(
     resp: asks.response_objects.Response,
     log: logging.Logger,
-    return_json: bool = True
+    return_json: bool = True,
+    log_resp: bool = False,
+
 ) -> asks.response_objects.Response:
     """Process response and return its json content.
 
@@ -48,11 +84,12 @@ def resproc(
     if not resp.status_code == 200:
         raise BrokerError(resp.body)
     try:
-        json = resp.json()
+        msg = resp.json()
     except json.decoder.JSONDecodeError:
         log.exception(f"Failed to process {resp}:\n{resp.text}")
         raise BrokerError(resp.text)
-    else:
-        log.debug(f"Received json contents:\n{colorize_json(json)}")
 
-    return json if return_json else resp
+    if log_resp:
+        log.debug(f"Received json contents:\n{colorize_json(msg)}")
+
+    return msg if return_json else resp

piker/brokers/api.py

@@ -1,85 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for piker0)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-"""
-Actor-aware broker agnostic interface.
-
-"""
-from typing import Dict
-from contextlib import asynccontextmanager, AsyncExitStack
-
-import trio
-
-from . import get_brokermod
-from ..log import get_logger
-
-
-log = get_logger(__name__)
-
-
-_cache: Dict[str, 'Client'] = {}  # noqa
-
-
-@asynccontextmanager
-async def open_cached_client(
-    brokername: str,
-    *args,
-    **kwargs,
-) -> 'Client':  # noqa
-    """Get a cached broker client from the current actor's local vars.
-
-    If one has not been setup do it and cache it.
-    """
-    global _cache
-
-    clients = _cache.setdefault('clients', {'_lock': trio.Lock()})
-
-    # global cache task lock
-    lock = clients['_lock']
-
-    client = None
-
-    try:
-        log.info(f"Loading existing `{brokername}` client")
-
-        async with lock:
-            client = clients[brokername]
-            client._consumers += 1
-
-        yield client
-
-    except KeyError:
-        log.info(f"Creating new client for broker {brokername}")
-
-        async with lock:
-            brokermod = get_brokermod(brokername)
-            exit_stack = AsyncExitStack()
-
-            client = await exit_stack.enter_async_context(
-                brokermod.get_client()
-            )
-            client._consumers = 0
-            client._exit_stack = exit_stack
-            clients[brokername] = client
-
-        yield client
-
-    finally:
-        if client is not None:
-            # if no more consumers, teardown the client
-            client._consumers -= 1
-            if client._consumers <= 0:
-                await client._exit_stack.aclose()

piker/brokers/binance.py

@@ -18,13 +18,17 @@
 Binance backend
 
 """
-from contextlib import asynccontextmanager
-from typing import List, Dict, Any, Tuple, Union, Optional
+from contextlib import asynccontextmanager as acm
+from datetime import datetime
+from typing import (
+    Any, Union, Optional,
+    AsyncGenerator, Callable,
+)
 import time
 
 import trio
 from trio_typing import TaskStatus
-import arrow
+import pendulum
 import asks
 from fuzzywuzzy import process as fuzzy
 import numpy as np
@@ -33,11 +37,11 @@ from pydantic.dataclasses import dataclass
 from pydantic import BaseModel
 import wsproto
 
-from .api import open_cached_client
+from .._cacheables import open_cached_client
 from ._util import resproc, SymbolNotFound
 from ..log import get_logger, get_console_log
 from ..data import ShmArray
-from ..data._web_bs import open_autorecon_ws
+from ..data._web_bs import open_autorecon_ws, NoBsWs
 
 log = get_logger(__name__)
 
@@ -88,7 +92,7 @@ class Pair(BaseModel):
     baseCommissionPrecision: int
     quoteCommissionPrecision: int
 
-    orderTypes: List[str]
+    orderTypes: list[str]
 
     icebergAllowed: bool
     ocoAllowed: bool
@@ -96,8 +100,8 @@ class Pair(BaseModel):
     isSpotTradingAllowed: bool
     isMarginTradingAllowed: bool
 
-    filters: List[Dict[str, Union[str, int, float]]]
-    permissions: List[str]
+    filters: list[dict[str, Union[str, int, float]]]
+    permissions: list[str]
 
 
 @dataclass
@@ -129,7 +133,7 @@ class OHLC:
     bar_wap: float = 0.0
 
 
-# convert arrow timestamp to unixtime in miliseconds
+# convert datetime obj timestamp to unixtime in milliseconds
 def binance_timestamp(when):
     return int((when.timestamp() * 1000) + (when.microsecond / 1000))
 
@@ -145,7 +149,7 @@ class Client:
         self,
         method: str,
         params: dict,
-    ) -> Dict[str, Any]:
+    ) -> dict[str, Any]:
         resp = await self._sesh.get(
             path=f'/api/v3/{method}',
             params=params,
@@ -200,7 +204,7 @@ class Client:
         self,
         pattern: str,
         limit: int = None,
-    ) -> Dict[str, Any]:
+    ) -> dict[str, Any]:
         if self._pairs is not None:
             data = self._pairs
         else:
@@ -213,25 +217,27 @@ class Client:
         )
         # repack in dict form
         return {item[0]['symbol']: item[0]
-         for item in matches}
+                for item in matches}
 
     async def bars(
         self,
         symbol: str,
-        start_time: int = None,
-        end_time: int = None,
+        start_dt: Optional[datetime] = None,
+        end_dt: Optional[datetime] = None,
         limit: int = 1000,  # <- max allowed per query
         as_np: bool = True,
 
     ) -> dict:
 
-        if start_time is None:
-            start_time = binance_timestamp(
-                arrow.utcnow().floor('minute').shift(minutes=-limit)
-            )
+        if end_dt is None:
+            end_dt = pendulum.now('UTC')
 
-        if end_time is None:
-            end_time = binance_timestamp(arrow.utcnow())
+        if start_dt is None:
+            start_dt = end_dt.start_of(
+                'minute').subtract(minutes=limit)
+
+        start_time = binance_timestamp(start_dt)
+        end_time = binance_timestamp(end_dt)
 
         # https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-data
         bars = await self._api(
@@ -273,7 +279,7 @@ class Client:
         return array
 
 
-@asynccontextmanager
+@acm
 async def get_client() -> Client:
     client = Client()
     await client.cache_symbols()
@@ -295,7 +301,7 @@ class AggTrade(BaseModel):
     M: bool  # Ignore
 
 
-async def stream_messages(ws):
+async def stream_messages(ws: NoBsWs) -> AsyncGenerator[NoBsWs, dict]:
 
     timeouts = 0
     while True:
@@ -353,7 +359,7 @@ async def stream_messages(ws):
             }
 
 
-def make_sub(pairs: List[str], sub_name: str, uid: int) -> Dict[str, str]:
+def make_sub(pairs: list[str], sub_name: str, uid: int) -> dict[str, str]:
     """Create a request subscription packet dict.
 
     https://binance-docs.github.io/apidocs/spot/en/#live-subscribing-unsubscribing-to-streams
@@ -368,6 +374,37 @@ def make_sub(pairs: List[str], sub_name: str, uid: int) -> Dict[str, str]:
     }
 
 
+@acm
+async def open_history_client(
+    symbol: str,
+
+) -> tuple[Callable, int]:
+
+    # TODO implement history getter for the new storage layer.
+    async with open_cached_client('binance') as client:
+
+        async def get_ohlc(
+            end_dt: Optional[datetime] = None,
+            start_dt: Optional[datetime] = None,
+
+        ) -> tuple[
+            np.ndarray,
+            datetime,  # start
+            datetime,  # end
+        ]:
+
+            array = await client.bars(
+                symbol,
+                start_dt=start_dt,
+                end_dt=end_dt,
+            )
+            start_dt = pendulum.from_timestamp(array[0]['time'])
+            end_dt = pendulum.from_timestamp(array[-1]['time'])
+            return array, start_dt, end_dt
+
+        yield get_ohlc, {'erlangs': 3, 'rate': 3}
+
+
 async def backfill_bars(
     sym: str,
     shm: ShmArray,  # type: ignore # noqa
@@ -385,13 +422,12 @@ async def backfill_bars(
 async def stream_quotes(
 
     send_chan: trio.abc.SendChannel,
-    symbols: List[str],
-    shm: ShmArray,
+    symbols: list[str],
     feed_is_live: trio.Event,
     loglevel: str = None,
 
     # startup sync
-    task_status: TaskStatus[Tuple[Dict, Dict]] = trio.TASK_STATUS_IGNORED,
+    task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
 
 ) -> None:
     # XXX: required to propagate ``tractor`` loglevel to piker logging
@@ -416,8 +452,9 @@ async def stream_quotes(
 
             # XXX: after manually inspecting the response format we
             # just directly pick out the info we need
-            si['price_tick_size'] = syminfo.filters[0]['tickSize']
-            si['lot_tick_size'] = syminfo.filters[2]['stepSize']
+            si['price_tick_size'] = float(syminfo.filters[0]['tickSize'])
+            si['lot_tick_size'] = float(syminfo.filters[2]['stepSize'])
+            si['asset_type'] = 'crypto'
 
         symbol = symbols[0]
 
@@ -427,10 +464,11 @@ async def stream_quotes(
             symbol: {
                 'symbol_info': sym_infos[sym],
                 'shm_write_opts': {'sum_tick_vml': False},
+                'fqsn': sym,
             },
         }
 
-        @asynccontextmanager
+        @acm
         async def subscribe(ws: wsproto.WSConnection):
             # setup subs
 
@@ -480,17 +518,25 @@ async def stream_quotes(
                 # TODO: use ``anext()`` when it lands in 3.10!
                 typ, quote = await msg_gen.__anext__()
 
-            first_quote = {quote['symbol'].lower(): quote}
-            task_status.started((init_msgs,  first_quote))
+            task_status.started((init_msgs,  quote))
 
             # signal to caller feed is ready for consumption
             feed_is_live.set()
 
+            # import time
+            # last = time.time()
+
             # start streaming
             async for typ, msg in msg_gen:
 
+                # period = time.time() - last
+                # hz = 1/period if period else float('inf')
+                # if hz > 60:
+                #     log.info(f'Binance quotez : {hz}')
+
                 topic = msg['symbol'].lower()
                 await send_chan.send({topic: msg})
+                # last = time.time()
 
 
 @tractor.context

piker/brokers/cli.py

@@ -23,7 +23,6 @@ from operator import attrgetter
 from operator import itemgetter
 
 import click
-import pandas as pd
 import trio
 import tractor
 
@@ -47,8 +46,10 @@ _watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
 @click.argument('kwargs', nargs=-1)
 @click.pass_obj
 def api(config, meth, kwargs, keys):
-    """Make a broker-client API method call
-    """
+    '''
+    Make a broker-client API method call
+
+    '''
     # global opts
     broker = config['brokers'][0]
 
@@ -79,13 +80,13 @@ def api(config, meth, kwargs, keys):
 
 
 @cli.command()
-@click.option('--df-output', '-df', flag_value=True,
-              help='Output in `pandas.DataFrame` format')
 @click.argument('tickers', nargs=-1, required=True)
 @click.pass_obj
-def quote(config, tickers, df_output):
-    """Print symbol quotes to the console
-    """
+def quote(config, tickers):
+    '''
+    Print symbol quotes to the console
+
+    '''
     # global opts
     brokermod = config['brokermods'][0]
 
@@ -100,28 +101,19 @@ def quote(config, tickers, df_output):
             if ticker not in syms:
                 brokermod.log.warn(f"Could not find symbol {ticker}?")
 
-    if df_output:
-        cols = next(filter(bool, quotes)).copy()
-        cols.pop('symbol')
-        df = pd.DataFrame(
-            (quote or {} for quote in quotes),
-            columns=cols,
-        )
-        click.echo(df)
-    else:
-        click.echo(colorize_json(quotes))
+    click.echo(colorize_json(quotes))
 
 
 @cli.command()
-@click.option('--df-output', '-df', flag_value=True,
-              help='Output in `pandas.DataFrame` format')
 @click.option('--count', '-c', default=1000,
               help='Number of bars to retrieve')
 @click.argument('symbol', required=True)
 @click.pass_obj
-def bars(config, symbol, count, df_output):
-    """Retreive 1m bars for symbol and print on the console
-    """
+def bars(config, symbol, count):
+    '''
+    Retreive 1m bars for symbol and print on the console
+
+    '''
     # global opts
     brokermod = config['brokermods'][0]
 
@@ -133,7 +125,7 @@ def bars(config, symbol, count, df_output):
             brokermod,
             symbol,
             count=count,
-            as_np=df_output
+            as_np=False,
         )
     )
 
@@ -141,10 +133,7 @@ def bars(config, symbol, count, df_output):
         log.error(f"No quotes could be found for {symbol}?")
         return
 
-    if df_output:
-        click.echo(pd.DataFrame(bars))
-    else:
-        click.echo(colorize_json(bars))
+    click.echo(colorize_json(bars))
 
 
 @cli.command()
@@ -156,8 +145,10 @@ def bars(config, symbol, count, df_output):
 @click.argument('name', nargs=1, required=True)
 @click.pass_obj
 def record(config, rate, name, dhost, filename):
-    """Record client side quotes to a file on disk
-    """
+    '''
+    Record client side quotes to a file on disk
+
+    '''
     # global opts
     brokermod = config['brokermods'][0]
     loglevel = config['loglevel']
@@ -195,8 +186,10 @@ def record(config, rate, name, dhost, filename):
 @click.argument('symbol', required=True)
 @click.pass_context
 def contracts(ctx, loglevel, broker, symbol, ids):
-    """Get list of all option contracts for symbol
-    """
+    '''
+    Get list of all option contracts for symbol
+
+    '''
     brokermod = get_brokermod(broker)
     get_console_log(loglevel)
 
@@ -213,14 +206,14 @@ def contracts(ctx, loglevel, broker, symbol, ids):
 
 
 @cli.command()
-@click.option('--df-output', '-df', flag_value=True,
-              help='Output in `pandas.DataFrame` format')
 @click.option('--date', '-d', help='Contracts expiry date')
 @click.argument('symbol', required=True)
 @click.pass_obj
-def optsquote(config, symbol, df_output, date):
-    """Retreive symbol option quotes on the console
-    """
+def optsquote(config, symbol, date):
+    '''
+    Retreive symbol option quotes on the console
+
+    '''
     # global opts
     brokermod = config['brokermods'][0]
 
@@ -233,22 +226,17 @@ def optsquote(config, symbol, df_output, date):
         log.error(f"No option quotes could be found for {symbol}?")
         return
 
-    if df_output:
-        df = pd.DataFrame(
-            (quote.values() for quote in quotes),
-            columns=quotes[0].keys(),
-        )
-        click.echo(df)
-    else:
-        click.echo(colorize_json(quotes))
+    click.echo(colorize_json(quotes))
 
 
 @cli.command()
 @click.argument('tickers', nargs=-1, required=True)
 @click.pass_obj
 def symbol_info(config, tickers):
-    """Print symbol quotes to the console
-    """
+    '''
+    Print symbol quotes to the console
+
+    '''
     # global opts
     brokermod = config['brokermods'][0]
 
@@ -270,8 +258,10 @@ def symbol_info(config, tickers):
 @click.argument('pattern', required=True)
 @click.pass_obj
 def search(config, pattern):
-    """Search for symbols from broker backend(s).
-    """
+    '''
+    Search for symbols from broker backend(s).
+
+    '''
     # global opts
     brokermods = config['brokermods']
 

piker/brokers/config.py

@@ -1,103 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-"""
-Broker configuration mgmt.
-"""
-import os
-from os.path import dirname
-import shutil
-
-import toml
-import click
-
-from ..log import get_logger
-
-log = get_logger('broker-config')
-
-_config_dir = click.get_app_dir('piker')
-_file_name = 'brokers.toml'
-
-
-def _override_config_dir(
-    path: str
-) -> None:
-    global _config_dir
-    _config_dir = path
-
-
-def get_broker_conf_path():
-    """Return the default config path normally under
-    ``~/.config/piker`` on linux.
-
-    Contains files such as:
-    - brokers.toml
-    - watchlists.toml
-    - signals.toml
-    - strats.toml
-
-    """
-    return os.path.join(_config_dir, _file_name)
-
-
-def repodir():
-    """Return the abspath to the repo directory.
-    """
-    dirpath = os.path.abspath(
-        # we're 3 levels down in **this** module file
-        dirname(dirname(dirname(os.path.realpath(__file__))))
-    )
-    return dirpath
-
-
-def load(
-    path: str = None
-) -> (dict, str):
-    """Load broker config.
-    """
-    path = path or get_broker_conf_path()
-    if not os.path.isfile(path):
-        shutil.copyfile(
-            os.path.join(repodir(), 'data/brokers.toml'),
-            path,
-        )
-
-    config = toml.load(path)
-    log.debug(f"Read config file {path}")
-    return config, path
-
-
-def write(
-    config: dict,  # toml config as dict
-    path: str = None,
-) -> None:
-    """Write broker config to disk.
-
-    Create a ``brokers.ini`` file if one does not exist.
-    """
-    path = path or get_broker_conf_path()
-    dirname = os.path.dirname(path)
-    if not os.path.isdir(dirname):
-        log.debug(f"Creating config dir {_config_dir}")
-        os.makedirs(dirname)
-
-    if not config:
-        raise ValueError(
-            "Watch out you're trying to write a blank config!")
-
-    log.debug(f"Writing config file {path}")
-    with open(path, 'w') as cf:
-        return toml.dump(config, cf)

piker/brokers/core.py

@@ -29,7 +29,7 @@ import trio
 from ..log import get_logger
 from . import get_brokermod
 from .._daemon import maybe_spawn_brokerd
-from .api import open_cached_client
+from .._cacheables import open_cached_client
 
 
 log = get_logger(__name__)
@@ -142,15 +142,23 @@ async def symbol_search(
     brokermods: list[ModuleType],
     pattern: str,
     **kwargs,
+
 ) -> Dict[str, Dict[str, Dict[str, Any]]]:
-    """Return symbol info from broker.
-    """
+    '''
+    Return symbol info from broker.
+
+    '''
     results = []
 
-    async def search_backend(brokername: str) -> None:
+    async def search_backend(
+        brokermod: ModuleType
+    ) -> None:
+
+        brokername: str = mod.name
 
         async with maybe_spawn_brokerd(
-            brokername,
+            mod.name,
+            infect_asyncio=getattr(mod, '_infect_asyncio', False),
         ) as portal:
 
             results.append((

piker/brokers/data.py

@@ -14,9 +14,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-"""
-Real-time data feed machinery
-"""
+'''
+NB: this is the old original implementation that was used way way back
+when the project started with ``kivy``.
+
+This code is left for reference but will likely be merged in
+appropriately and removed.
+
+'''
 import time
 from functools import partial
 from dataclasses import dataclass, field
@@ -33,6 +38,7 @@ import contextlib
 
 import trio
 import tractor
+from tractor.experimental import msgpub
 from async_generator import asynccontextmanager
 
 from ..log import get_logger, get_console_log
@@ -93,7 +99,7 @@ class BrokerFeed:
     )
 
 
-@tractor.msg.pub(tasks=['stock', 'option'])
+@msgpub(tasks=['stock', 'option'])
 async def stream_poll_requests(
     get_topics: Callable,
     get_quotes: Coroutine,
@@ -288,7 +294,7 @@ async def start_quote_stream(
 
         await stream_poll_requests(
 
-            # ``msg.pub`` required kwargs
+            # ``trionics.msgpub`` required kwargs
             task_name=feed_type,
             ctx=ctx,
             topics=symbols,

piker/brokers/ib/__init__.py

@@ -0,0 +1,67 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+Interactive Brokers API backend.
+
+Sub-modules within break into the core functionalities:
+
+- ``broker.py`` part for orders / trading endpoints
+- ``data.py`` for real-time data feed endpoints
+
+- ``client.py`` for the core API machinery which is ``trio``-ized
+  wrapping around ``ib_insync``.
+
+- ``report.py`` for the hackery to build manual pp calcs
+  to avoid ib's absolute bullshit FIFO style position
+  tracking..
+
+"""
+from .api import (
+    get_client,
+)
+from .feed import (
+    open_history_client,
+    open_symbol_search,
+    stream_quotes,
+)
+from .broker import trades_dialogue
+
+__all__ = [
+    'get_client',
+    'trades_dialogue',
+    'open_history_client',
+    'open_symbol_search',
+    'stream_quotes',
+]
+
+
+# tractor RPC enable arg
+__enable_modules__: list[str] = [
+    'api',
+    'feed',
+    'broker',
+]
+
+# passed to ``tractor.ActorNursery.start_actor()``
+_spawn_kwargs = {
+    'infect_asyncio': True,
+}
+
+# annotation to let backend agnostic code
+# know if ``brokerd`` should be spawned with
+# ``tractor``'s aio mode.
+_infect_asyncio: bool = True

piker/brokers/ib/broker.py

@@ -0,0 +1,590 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Order and trades endpoints for use with ``piker``'s EMS.
+
+"""
+from __future__ import annotations
+from dataclasses import asdict
+from functools import partial
+from pprint import pformat
+import time
+from typing import (
+    Any,
+    Optional,
+    AsyncIterator,
+)
+
+import trio
+from trio_typing import TaskStatus
+import tractor
+from ib_insync.contract import (
+    Contract,
+    Option,
+)
+from ib_insync.order import (
+    Trade,
+    OrderStatus,
+)
+from ib_insync.objects import (
+    Fill,
+    Execution,
+)
+from ib_insync.objects import Position
+
+from piker import config
+from piker.log import get_console_log
+from piker.clearing._messages import (
+    BrokerdOrder,
+    BrokerdOrderAck,
+    BrokerdStatus,
+    BrokerdPosition,
+    BrokerdCancel,
+    BrokerdFill,
+    BrokerdError,
+)
+from .api import (
+    _accounts2clients,
+    _adhoc_futes_set,
+    log,
+    get_config,
+    open_client_proxies,
+    Client,
+)
+
+
+def pack_position(
+    pos: Position
+
+) -> dict[str, Any]:
+    con = pos.contract
+
+    if isinstance(con, Option):
+        # TODO: option symbol parsing and sane display:
+        symbol = con.localSymbol.replace(' ', '')
+
+    else:
+        # TODO: lookup fqsn even for derivs.
+        symbol = con.symbol.lower()
+
+    exch = (con.primaryExchange or con.exchange).lower()
+    symkey = '.'.join((symbol, exch))
+    if not exch:
+        # attempt to lookup the symbol from our
+        # hacked set..
+        for sym in _adhoc_futes_set:
+            if symbol in sym:
+                symkey = sym
+                break
+
+    expiry = con.lastTradeDateOrContractMonth
+    if expiry:
+        symkey += f'.{expiry}'
+
+    # TODO: options contracts into a sane format..
+
+    return BrokerdPosition(
+        broker='ib',
+        account=pos.account,
+        symbol=symkey,
+        currency=con.currency,
+        size=float(pos.position),
+        avg_price=float(pos.avgCost) / float(con.multiplier or 1.0),
+    )
+
+
+async def handle_order_requests(
+
+    ems_order_stream: tractor.MsgStream,
+    accounts_def: dict[str, str],
+
+) -> None:
+
+    request_msg: dict
+    async for request_msg in ems_order_stream:
+        log.info(f'Received order request {request_msg}')
+
+        action = request_msg['action']
+        account = request_msg['account']
+
+        acct_number = accounts_def.get(account)
+        if not acct_number:
+            log.error(
+                f'An IB account number for name {account} is not found?\n'
+                'Make sure you have all TWS and GW instances running.'
+            )
+            await ems_order_stream.send(BrokerdError(
+                oid=request_msg['oid'],
+                symbol=request_msg['symbol'],
+                reason=f'No account found: `{account}` ?',
+            ).dict())
+            continue
+
+        client = _accounts2clients.get(account)
+        if not client:
+            log.error(
+                f'An IB client for account name {account} is not found.\n'
+                'Make sure you have all TWS and GW instances running.'
+            )
+            await ems_order_stream.send(BrokerdError(
+                oid=request_msg['oid'],
+                symbol=request_msg['symbol'],
+                reason=f'No api client loaded for account: `{account}` ?',
+            ).dict())
+            continue
+
+        if action in {'buy', 'sell'}:
+            # validate
+            order = BrokerdOrder(**request_msg)
+
+            # call our client api to submit the order
+            reqid = client.submit_limit(
+                oid=order.oid,
+                symbol=order.symbol,
+                price=order.price,
+                action=order.action,
+                size=order.size,
+                account=acct_number,
+
+                # XXX: by default 0 tells ``ib_insync`` methods that
+                # there is no existing order so ask the client to create
+                # a new one (which it seems to do by allocating an int
+                # counter - collision prone..)
+                reqid=order.reqid,
+            )
+            if reqid is None:
+                await ems_order_stream.send(BrokerdError(
+                    oid=request_msg['oid'],
+                    symbol=request_msg['symbol'],
+                    reason='Order already active?',
+                ).dict())
+
+            # deliver ack that order has been submitted to broker routing
+            await ems_order_stream.send(
+                BrokerdOrderAck(
+                    # ems order request id
+                    oid=order.oid,
+                    # broker specific request id
+                    reqid=reqid,
+                    time_ns=time.time_ns(),
+                    account=account,
+                ).dict()
+            )
+
+        elif action == 'cancel':
+            msg = BrokerdCancel(**request_msg)
+            client.submit_cancel(reqid=msg.reqid)
+
+        else:
+            log.error(f'Unknown order command: {request_msg}')
+
+
+async def recv_trade_updates(
+
+    client: Client,
+    to_trio: trio.abc.SendChannel,
+
+) -> None:
+    """Stream a ticker using the std L1 api.
+    """
+    client.inline_errors(to_trio)
+
+    # sync with trio task
+    to_trio.send_nowait(None)
+
+    def push_tradesies(eventkit_obj, obj, fill=None):
+        """Push events to trio task.
+
+        """
+        if fill is not None:
+            # execution details event
+            item = ('fill', (obj, fill))
+
+        elif eventkit_obj.name() == 'positionEvent':
+            item = ('position', obj)
+
+        else:
+            item = ('status', obj)
+
+        log.info(f'eventkit event ->\n{pformat(item)}')
+
+        try:
+            to_trio.send_nowait(item)
+        except trio.BrokenResourceError:
+            log.exception(f'Disconnected from {eventkit_obj} updates')
+            eventkit_obj.disconnect(push_tradesies)
+
+    # hook up to the weird eventkit object - event stream api
+    for ev_name in [
+        'orderStatusEvent',  # all order updates
+        'execDetailsEvent',  # all "fill" updates
+        'positionEvent',  # avg price updates per symbol per account
+
+        # 'commissionReportEvent',
+        # XXX: ugh, it is a separate event from IB and it's
+        # emitted as follows:
+        # self.ib.commissionReportEvent.emit(trade, fill, report)
+
+        # XXX: not sure yet if we need these
+        # 'updatePortfolioEvent',
+
+        # XXX: these all seem to be weird ib_insync intrernal
+        # events that we probably don't care that much about
+        # given the internal design is wonky af..
+        # 'newOrderEvent',
+        # 'orderModifyEvent',
+        # 'cancelOrderEvent',
+        # 'openOrderEvent',
+    ]:
+        eventkit_obj = getattr(client.ib, ev_name)
+        handler = partial(push_tradesies, eventkit_obj)
+        eventkit_obj.connect(handler)
+
+    # let the engine run and stream
+    await client.ib.disconnectedEvent
+
+
+@tractor.context
+async def trades_dialogue(
+
+    ctx: tractor.Context,
+    loglevel: str = None,
+
+) -> AsyncIterator[dict[str, Any]]:
+
+    # XXX: required to propagate ``tractor`` loglevel to piker logging
+    get_console_log(loglevel or tractor.current_actor().loglevel)
+
+    accounts_def = config.load_accounts(['ib'])
+
+    global _client_cache
+
+    # deliver positions to subscriber before anything else
+    all_positions = []
+    accounts = set()
+    clients: list[tuple[Client, trio.MemoryReceiveChannel]] = []
+
+    async with (
+        trio.open_nursery() as nurse,
+        open_client_proxies() as (proxies, aioclients),
+    ):
+        for account, proxy in proxies.items():
+
+            client = aioclients[account]
+
+            async def open_stream(
+                task_status: TaskStatus[
+                    trio.abc.ReceiveChannel
+                ] = trio.TASK_STATUS_IGNORED,
+            ):
+                # each api client has a unique event stream
+                async with tractor.to_asyncio.open_channel_from(
+                    recv_trade_updates,
+                    client=client,
+                ) as (first, trade_event_stream):
+
+                    task_status.started(trade_event_stream)
+                    await trio.sleep_forever()
+
+            trade_event_stream = await nurse.start(open_stream)
+
+            clients.append((client, trade_event_stream))
+
+            assert account in accounts_def
+            accounts.add(account)
+
+        for client in aioclients.values():
+            for pos in client.positions():
+
+                msg = pack_position(pos)
+                msg.account = accounts_def.inverse[msg.account]
+
+                assert msg.account in accounts, (
+                    f'Position for unknown account: {msg.account}')
+
+                all_positions.append(msg.dict())
+
+        trades: list[dict] = []
+        for proxy in proxies.values():
+            trades.append(await proxy.trades())
+
+        log.info(f'Loaded {len(trades)} from this session')
+        # TODO: write trades to local ``trades.toml``
+        # - use above per-session trades data and write to local file
+        # - get the "flex reports" working and pull historical data and
+        # also save locally.
+
+        await ctx.started((
+            all_positions,
+            tuple(name for name in accounts_def if name in accounts),
+        ))
+
+        async with (
+            ctx.open_stream() as ems_stream,
+            trio.open_nursery() as n,
+        ):
+            # start order request handler **before** local trades event loop
+            n.start_soon(handle_order_requests, ems_stream, accounts_def)
+
+            # allocate event relay tasks for each client connection
+            for client, stream in clients:
+                n.start_soon(
+                    deliver_trade_events,
+                    stream,
+                    ems_stream,
+                    accounts_def
+                )
+
+            # block until cancelled
+            await trio.sleep_forever()
+
+
+async def deliver_trade_events(
+
+    trade_event_stream: trio.MemoryReceiveChannel,
+    ems_stream: tractor.MsgStream,
+    accounts_def: dict[str, str],
+
+) -> None:
+    '''Format and relay all trade events for a given client to the EMS.
+
+    '''
+    action_map = {'BOT': 'buy', 'SLD': 'sell'}
+
+    # TODO: for some reason we can receive a ``None`` here when the
+    # ib-gw goes down? Not sure exactly how that's happening looking
+    # at the eventkit code above but we should probably handle it...
+    async for event_name, item in trade_event_stream:
+
+        log.info(f'ib sending {event_name}:\n{pformat(item)}')
+
+        # TODO: templating the ib statuses in comparison with other
+        # brokers is likely the way to go:
+        # https://interactivebrokers.github.io/tws-api/interfaceIBApi_1_1EWrapper.html#a17f2a02d6449710b6394d0266a353313
+        # short list:
+        # - PendingSubmit
+        # - PendingCancel
+        # - PreSubmitted (simulated orders)
+        # - ApiCancelled (cancelled by client before submission
+        #                 to routing)
+        # - Cancelled
+        # - Filled
+        # - Inactive (reject or cancelled but not by trader)
+
+        # XXX: here's some other sucky cases from the api
+        # - short-sale but securities haven't been located, in this
+        #   case we should probably keep the order in some kind of
+        #   weird state or cancel it outright?
+
+        # status='PendingSubmit', message=''),
+        # status='Cancelled', message='Error 404,
+        #   reqId 1550: Order held while securities are located.'),
+        # status='PreSubmitted', message='')],
+
+        if event_name == 'status':
+
+            # XXX: begin normalization of nonsense ib_insync internal
+            # object-state tracking representations...
+
+            # unwrap needed data from ib_insync internal types
+            trade: Trade = item
+            status: OrderStatus = trade.orderStatus
+
+            # skip duplicate filled updates - we get the deats
+            # from the execution details event
+            msg = BrokerdStatus(
+
+                reqid=trade.order.orderId,
+                time_ns=time.time_ns(),  # cuz why not
+                account=accounts_def.inverse[trade.order.account],
+
+                # everyone doin camel case..
+                status=status.status.lower(),  # force lower case
+
+                filled=status.filled,
+                reason=status.whyHeld,
+
+                # this seems to not be necessarily up to date in the
+                # execDetails event.. so we have to send it here I guess?
+                remaining=status.remaining,
+
+                broker_details={'name': 'ib'},
+            )
+
+        elif event_name == 'fill':
+
+            # for wtv reason this is a separate event type
+            # from IB, not sure why it's needed other then for extra
+            # complexity and over-engineering :eyeroll:.
+            # we may just end up dropping these events (or
+            # translating them to ``Status`` msgs) if we can
+            # show the equivalent status events are no more latent.
+
+            # unpack ib_insync types
+            # pep-0526 style:
+            # https://www.python.org/dev/peps/pep-0526/#global-and-local-variable-annotations
+            trade: Trade
+            fill: Fill
+            trade, fill = item
+            execu: Execution = fill.execution
+
+            # TODO: normalize out commissions details?
+            details = {
+                'contract': asdict(fill.contract),
+                'execution': asdict(fill.execution),
+                'commissions': asdict(fill.commissionReport),
+                'broker_time': execu.time,   # supposedly server fill time
+                'name': 'ib',
+            }
+
+            msg = BrokerdFill(
+                # should match the value returned from `.submit_limit()`
+                reqid=execu.orderId,
+                time_ns=time.time_ns(),  # cuz why not
+
+                action=action_map[execu.side],
+                size=execu.shares,
+                price=execu.price,
+
+                broker_details=details,
+                # XXX: required by order mode currently
+                broker_time=details['broker_time'],
+
+            )
+
+        elif event_name == 'error':
+
+            err: dict = item
+
+            # f$#$% gawd dammit insync..
+            con = err['contract']
+            if isinstance(con, Contract):
+                err['contract'] = asdict(con)
+
+            if err['reqid'] == -1:
+                log.error(f'TWS external order error:\n{pformat(err)}')
+
+            # TODO: what schema for this msg if we're going to make it
+            # portable across all backends?
+            # msg = BrokerdError(**err)
+            continue
+
+        elif event_name == 'position':
+            msg = pack_position(item)
+            msg.account = accounts_def.inverse[msg.account]
+
+        elif event_name == 'event':
+
+            # it's either a general system status event or an external
+            # trade event?
+            log.info(f"TWS system status: \n{pformat(item)}")
+
+            # TODO: support this again but needs parsing at the callback
+            # level...
+            # reqid = item.get('reqid', 0)
+            # if getattr(msg, 'reqid', 0) < -1:
+            # log.info(f"TWS triggered trade\n{pformat(msg.dict())}")
+
+            continue
+
+            # msg.reqid = 'tws-' + str(-1 * reqid)
+
+            # mark msg as from "external system"
+            # TODO: probably something better then this.. and start
+            # considering multiplayer/group trades tracking
+            # msg.broker_details['external_src'] = 'tws'
+
+        # XXX: we always serialize to a dict for msgpack
+        # translations, ideally we can move to an msgspec (or other)
+        # encoder # that can be enabled in ``tractor`` ahead of
+        # time so we can pass through the message types directly.
+        await ems_stream.send(msg.dict())
+
+
+def load_flex_trades(
+    path: Optional[str] = None,
+
+) -> dict[str, str]:
+
+    from pprint import pprint
+    from ib_insync import flexreport, util
+
+    conf = get_config()
+
+    if not path:
+        # load ``brokers.toml`` and try to get the flex
+        # token and query id that must be previously defined
+        # by the user.
+        token = conf.get('flex_token')
+        if not token:
+            raise ValueError(
+               'You must specify a ``flex_token`` field in your'
+               '`brokers.toml` in order load your trade log, see our'
+               'intructions for how to set this up here:\n'
+               'PUT LINK HERE!'
+            )
+
+        qid = conf['flex_trades_query_id']
+
+        # TODO: hack this into our logging
+        # system like we do with the API client..
+        util.logToConsole()
+
+        # TODO: rewrite the query part of this with async..httpx?
+        report = flexreport.FlexReport(
+            token=token,
+            queryId=qid,
+        )
+
+    else:
+        # XXX: another project we could potentially look at,
+        # https://pypi.org/project/ibflex/
+        report = flexreport.FlexReport(path=path)
+
+    trade_entries = report.extract('Trade')
+    trades = {
+        # XXX: LOL apparently ``toml`` has a bug
+        # where a section key error will show up in the write
+        # if you leave this as an ``int``?
+        str(t.__dict__['tradeID']): t.__dict__
+        for t in trade_entries
+    }
+
+    ln = len(trades)
+    log.info(f'Loaded {ln} trades from flex query')
+
+    trades_by_account = {}
+    for tid, trade in trades.items():
+        trades_by_account.setdefault(
+            # oddly for some so-called "BookTrade" entries
+            # this field seems to be blank, no cuckin clue.
+            # trade['ibExecID']
+            str(trade['accountId']), {}
+        )[tid] = trade
+
+    section = {'ib': trades_by_account}
+    pprint(section)
+
+    # TODO: load the config first and append in
+    # the new trades loaded here..
+    try:
+        config.write(section, 'trades')
+    except KeyError:
+        import pdbpp; pdbpp.set_trace()  # noqa
+
+
+if __name__ == '__main__':
+    load_flex_trades()

piker/brokers/ib/feed.py

@@ -0,0 +1,938 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Data feed endpoints pre-wrapped and ready for use with ``tractor``/``trio``.
+
+"""
+from __future__ import annotations
+import asyncio
+from contextlib import asynccontextmanager as acm
+from dataclasses import asdict
+from datetime import datetime
+from math import isnan
+import time
+from typing import (
+    Callable,
+    Optional,
+    Awaitable,
+)
+
+from async_generator import aclosing
+from fuzzywuzzy import process as fuzzy
+import numpy as np
+import pendulum
+import tractor
+import trio
+from trio_typing import TaskStatus
+
+from piker.data._sharedmem import ShmArray
+from .._util import SymbolNotFound, NoData
+from .api import (
+    _adhoc_futes_set,
+    log,
+    load_aio_clients,
+    ibis,
+    MethodProxy,
+    open_client_proxies,
+    get_preferred_data_client,
+    Ticker,
+    RequestError,
+    Contract,
+)
+
+
+# https://interactivebrokers.github.io/tws-api/tick_types.html
+tick_types = {
+    77: 'trade',
+
+    # a "utrade" aka an off exchange "unreportable" (dark) vlm:
+    # https://interactivebrokers.github.io/tws-api/tick_types.html#rt_volume
+    48: 'dark_trade',
+
+    # standard L1 ticks
+    0: 'bsize',
+    1: 'bid',
+    2: 'ask',
+    3: 'asize',
+    4: 'last',
+    5: 'size',
+    8: 'volume',
+
+    # ``ib_insync`` already packs these into
+    # quotes under the following fields.
+    # 55: 'trades_per_min',  # `'tradeRate'`
+    # 56: 'vlm_per_min',  # `'volumeRate'`
+    # 89: 'shortable',  # `'shortableShares'`
+}
+
+
+@acm
+async def open_data_client() -> MethodProxy:
+    '''
+    Open the first found preferred "data client" as defined in the
+    user's ``brokers.toml`` in the ``ib.prefer_data_account`` variable
+    and deliver that client wrapped in a ``MethodProxy``.
+
+    '''
+    async with (
+        open_client_proxies() as (proxies, clients),
+    ):
+        account_name, client = get_preferred_data_client(clients)
+        proxy = proxies.get(f'ib.{account_name}')
+        if not proxy:
+            raise ValueError(
+                f'No preferred data client could be found for {account_name}!'
+            )
+
+        yield proxy
+
+
+@acm
+async def open_history_client(
+    symbol: str,
+
+) -> tuple[Callable, int]:
+    '''
+    History retreival endpoint - delivers a historical frame callble
+    that takes in ``pendulum.datetime`` and returns ``numpy`` arrays.
+
+    '''
+    async with open_data_client() as proxy:
+
+        async def get_hist(
+            end_dt: Optional[datetime] = None,
+            start_dt: Optional[datetime] = None,
+
+        ) -> tuple[np.ndarray, str]:
+
+            out, fails = await get_bars(proxy, symbol, end_dt=end_dt)
+
+            # TODO: add logic here to handle tradable hours and only grab
+            # valid bars in the range
+            if out is None:
+                # could be trying to retreive bars over weekend
+                log.error(f"Can't grab bars starting at {end_dt}!?!?")
+                raise NoData(
+                    f'{end_dt}',
+                    frame_size=2000,
+                )
+
+            bars, bars_array, first_dt, last_dt = out
+
+            # volume cleaning since there's -ve entries,
+            # wood luv to know what crookery that is..
+            vlm = bars_array['volume']
+            vlm[vlm < 0] = 0
+
+            return bars_array, first_dt, last_dt
+
+        # TODO: it seems like we can do async queries for ohlc
+        # but getting the order right still isn't working and I'm not
+        # quite sure why.. needs some tinkering and probably
+        # a lookthrough of the ``ib_insync`` machinery, for eg. maybe
+        # we have to do the batch queries on the `asyncio` side?
+        yield get_hist, {'erlangs': 1, 'rate': 6}
+
+
+_pacing: str = (
+    'Historical Market Data Service error '
+    'message:Historical data request pacing violation'
+)
+
+
+async def get_bars(
+
+    proxy: MethodProxy,
+    fqsn: str,
+
+    # blank to start which tells ib to look up the latest datum
+    end_dt: str = '',
+
+) -> (dict, np.ndarray):
+    '''
+    Retrieve historical data from a ``trio``-side task using
+    a ``MethoProxy``.
+
+    '''
+    fails = 0
+    bars: Optional[list] = None
+    first_dt: datetime = None
+    last_dt: datetime = None
+
+    if end_dt:
+        last_dt = pendulum.from_timestamp(end_dt.timestamp())
+
+    for _ in range(10):
+        try:
+            out = await proxy.bars(
+                fqsn=fqsn,
+                end_dt=end_dt,
+            )
+            if out:
+                bars, bars_array = out
+
+            else:
+                await tractor.breakpoint()
+
+            if bars_array is None:
+                raise SymbolNotFound(fqsn)
+
+            first_dt = pendulum.from_timestamp(
+                bars[0].date.timestamp())
+
+            last_dt = pendulum.from_timestamp(
+                bars[-1].date.timestamp())
+
+            time = bars_array['time']
+            assert time[-1] == last_dt.timestamp()
+            assert time[0] == first_dt.timestamp()
+            log.info(
+                f'{len(bars)} bars retreived for {first_dt} -> {last_dt}'
+            )
+
+            return (bars, bars_array, first_dt, last_dt), fails
+
+        except RequestError as err:
+            msg = err.message
+            # why do we always need to rebind this?
+            # _err = err
+
+            if 'No market data permissions for' in msg:
+                # TODO: signalling for no permissions searches
+                raise NoData(
+                    f'Symbol: {fqsn}',
+                )
+
+            elif (
+                err.code == 162
+                and 'HMDS query returned no data' in err.message
+            ):
+                # XXX: this is now done in the storage mgmt layer
+                # and we shouldn't implicitly decrement the frame dt
+                # index since the upper layer may be doing so
+                # concurrently and we don't want to be delivering frames
+                # that weren't asked for.
+                log.warning(
+                    f'NO DATA found ending @ {end_dt}\n'
+                )
+
+                # try to decrement start point and look further back
+                # end_dt = last_dt = last_dt.subtract(seconds=2000)
+
+                raise NoData(
+                    f'Symbol: {fqsn}',
+                    frame_size=2000,
+                )
+
+            elif _pacing in msg:
+
+                log.warning(
+                    'History throttle rate reached!\n'
+                    'Resetting farms with `ctrl-alt-f` hack\n'
+                )
+                # TODO: we might have to put a task lock around this
+                # method..
+                hist_ev = proxy.status_event(
+                    'HMDS data farm connection is OK:ushmds'
+                )
+
+                # XXX: other event messages we might want to try and
+                # wait for but i wasn't able to get any of this
+                # reliable..
+                # reconnect_start = proxy.status_event(
+                #     'Market data farm is connecting:usfuture'
+                # )
+                # live_ev = proxy.status_event(
+                #     'Market data farm connection is OK:usfuture'
+                # )
+
+                # try to wait on the reset event(s) to arrive, a timeout
+                # will trigger a retry up to 6 times (for now).
+                tries: int = 2
+                timeout: float = 10
+
+                # try 3 time with a data reset then fail over to
+                # a connection reset.
+                for i in range(1, tries):
+
+                    log.warning('Sending DATA RESET request')
+                    await data_reset_hack(reset_type='data')
+
+                    with trio.move_on_after(timeout) as cs:
+                        for name, ev in [
+                            # TODO: not sure if waiting on other events
+                            # is all that useful here or not. in theory
+                            # you could wait on one of the ones above
+                            # first to verify the reset request was
+                            # sent?
+                            ('history', hist_ev),
+                        ]:
+                            await ev.wait()
+                            log.info(f"{name} DATA RESET")
+                            break
+
+                    if cs.cancelled_caught:
+                        fails += 1
+                        log.warning(
+                            f'Data reset {name} timeout, retrying {i}.'
+                        )
+
+                        continue
+                else:
+
+                    log.warning('Sending CONNECTION RESET')
+                    await data_reset_hack(reset_type='connection')
+
+                    with trio.move_on_after(timeout) as cs:
+                        for name, ev in [
+                            # TODO: not sure if waiting on other events
+                            # is all that useful here or not. in theory
+                            # you could wait on one of the ones above
+                            # first to verify the reset request was
+                            # sent?
+                            ('history', hist_ev),
+                        ]:
+                            await ev.wait()
+                            log.info(f"{name} DATA RESET")
+
+                    if cs.cancelled_caught:
+                        fails += 1
+                        log.warning('Data CONNECTION RESET timeout!?')
+
+            else:
+                raise
+
+    return None, None
+    # else:  # throttle wasn't fixed so error out immediately
+    #     raise _err
+
+
+async def backfill_bars(
+
+    fqsn: str,
+    shm: ShmArray,  # type: ignore # noqa
+
+    # TODO: we want to avoid overrunning the underlying shm array buffer
+    # and we should probably calc the number of calls to make depending
+    # on that until we have the `marketstore` daemon in place in which
+    # case the shm size will be driven by user config and available sys
+    # memory.
+    count: int = 16,
+
+    task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED,
+
+) -> None:
+    '''
+    Fill historical bars into shared mem / storage afap.
+
+    TODO: avoid pacing constraints:
+    https://github.com/pikers/piker/issues/128
+
+    '''
+    # last_dt1 = None
+    last_dt = None
+
+    with trio.CancelScope() as cs:
+
+        async with open_data_client() as proxy:
+
+            out, fails = await get_bars(proxy, fqsn)
+
+            if out is None:
+                raise RuntimeError("Could not pull currrent history?!")
+
+            (first_bars, bars_array, first_dt, last_dt) = out
+            vlm = bars_array['volume']
+            vlm[vlm < 0] = 0
+            last_dt = first_dt
+
+            # write historical data to buffer
+            shm.push(bars_array)
+
+            task_status.started(cs)
+
+            i = 0
+            while i < count:
+
+                out, fails = await get_bars(proxy, fqsn, end_dt=first_dt)
+
+                if out is None:
+                    # could be trying to retreive bars over weekend
+                    # TODO: add logic here to handle tradable hours and
+                    # only grab valid bars in the range
+                    log.error(f"Can't grab bars starting at {first_dt}!?!?")
+
+                    # XXX: get_bars() should internally decrement dt by
+                    # 2k seconds and try again.
+                    continue
+
+                (first_bars, bars_array, first_dt, last_dt) = out
+                # last_dt1 = last_dt
+                # last_dt = first_dt
+
+                # volume cleaning since there's -ve entries,
+                # wood luv to know what crookery that is..
+                vlm = bars_array['volume']
+                vlm[vlm < 0] = 0
+
+                # TODO we should probably dig into forums to see what peeps
+                # think this data "means" and then use it as an indicator of
+                # sorts? dinkus has mentioned that $vlms for the day dont'
+                # match other platforms nor the summary stat tws shows in
+                # the monitor - it's probably worth investigating.
+
+                shm.push(bars_array, prepend=True)
+                i += 1
+
+
+asset_type_map = {
+    'STK': 'stock',
+    'OPT': 'option',
+    'FUT': 'future',
+    'CONTFUT': 'continuous_future',
+    'CASH': 'forex',
+    'IND': 'index',
+    'CFD': 'cfd',
+    'BOND': 'bond',
+    'CMDTY': 'commodity',
+    'FOP': 'futures_option',
+    'FUND': 'mutual_fund',
+    'WAR': 'warrant',
+    'IOPT': 'warran',
+    'BAG': 'bag',
+    # 'NEWS': 'news',
+}
+
+
+_quote_streams: dict[str, trio.abc.ReceiveStream] = {}
+
+
+async def _setup_quote_stream(
+
+    from_trio: asyncio.Queue,
+    to_trio: trio.abc.SendChannel,
+
+    symbol: str,
+    opts: tuple[int] = (
+        '375',  # RT trade volume (excludes utrades)
+        '233',  # RT trade volume (includes utrades)
+        '236',  # Shortable shares
+
+        # these all appear to only be updated every 25s thus
+        # making them mostly useless and explains why the scanner
+        # is always slow XD
+        # '293',  # Trade count for day
+        '294',  # Trade rate / minute
+        '295',  # Vlm rate / minute
+    ),
+    contract: Optional[Contract] = None,
+
+) -> trio.abc.ReceiveChannel:
+    '''
+    Stream a ticker using the std L1 api.
+
+    This task is ``asyncio``-side and must be called from
+    ``tractor.to_asyncio.open_channel_from()``.
+
+    '''
+    global _quote_streams
+
+    to_trio.send_nowait(None)
+
+    async with load_aio_clients() as accts2clients:
+        caccount_name, client = get_preferred_data_client(accts2clients)
+        contract = contract or (await client.find_contract(symbol))
+        ticker: Ticker = client.ib.reqMktData(contract, ','.join(opts))
+
+        # NOTE: it's batch-wise and slow af but I guess could
+        # be good for backchecking? Seems to be every 5s maybe?
+        # ticker: Ticker = client.ib.reqTickByTickData(
+        #     contract, 'Last',
+        # )
+
+        # # define a simple queue push routine that streams quote packets
+        # # to trio over the ``to_trio`` memory channel.
+        # to_trio, from_aio = trio.open_memory_channel(2**8)  # type: ignore
+        def teardown():
+            ticker.updateEvent.disconnect(push)
+            log.error(f"Disconnected stream for `{symbol}`")
+            client.ib.cancelMktData(contract)
+
+            # decouple broadcast mem chan
+            _quote_streams.pop(symbol, None)
+
+        def push(t: Ticker) -> None:
+            """
+            Push quotes to trio task.
+
+            """
+            # log.debug(t)
+            try:
+                to_trio.send_nowait(t)
+
+            except (
+                trio.BrokenResourceError,
+
+                # XXX: HACK, not sure why this gets left stale (probably
+                # due to our terrible ``tractor.to_asyncio``
+                # implementation for streams.. but if the mem chan
+                # gets left here and starts blocking just kill the feed?
+                # trio.WouldBlock,
+            ):
+                # XXX: eventkit's ``Event.emit()`` for whatever redic
+                # reason will catch and ignore regular exceptions
+                # resulting in tracebacks spammed to console..
+                # Manually do the dereg ourselves.
+                teardown()
+            except trio.WouldBlock:
+                log.warning(
+                    f'channel is blocking symbol feed for {symbol}?'
+                    f'\n{to_trio.statistics}'
+                )
+
+            # except trio.WouldBlock:
+            #     # for slow debugging purposes to avoid clobbering prompt
+            #     # with log msgs
+            #     pass
+
+        ticker.updateEvent.connect(push)
+        try:
+            await asyncio.sleep(float('inf'))
+        finally:
+            teardown()
+
+        # return from_aio
+
+
+@acm
+async def open_aio_quote_stream(
+
+    symbol: str,
+    contract: Optional[Contract] = None,
+
+) -> trio.abc.ReceiveStream:
+
+    from tractor.trionics import broadcast_receiver
+    global _quote_streams
+
+    from_aio = _quote_streams.get(symbol)
+    if from_aio:
+
+        # if we already have a cached feed deliver a rx side clone to consumer
+        async with broadcast_receiver(
+            from_aio,
+            2**6,
+        ) as from_aio:
+            yield from_aio
+            return
+
+    async with tractor.to_asyncio.open_channel_from(
+        _setup_quote_stream,
+        symbol=symbol,
+        contract=contract,
+
+    ) as (first, from_aio):
+
+        # cache feed for later consumers
+        _quote_streams[symbol] = from_aio
+
+        yield from_aio
+
+
+# TODO: cython/mypyc/numba this!
+def normalize(
+    ticker: Ticker,
+    calc_price: bool = False
+
+) -> dict:
+
+    # should be real volume for this contract by default
+    calc_price = False
+
+    # check for special contract types
+    con = ticker.contract
+    if type(con) in (
+        ibis.Commodity,
+        ibis.Forex,
+    ):
+        # commodities and forex don't have an exchange name and
+        # no real volume so we have to calculate the price
+        suffix = con.secType
+        # no real volume on this tract
+        calc_price = True
+
+    else:
+        suffix = con.primaryExchange
+        if not suffix:
+            suffix = con.exchange
+
+        # append a `.<suffix>` to the returned symbol
+        # key for derivatives that normally is the expiry
+        # date key.
+        expiry = con.lastTradeDateOrContractMonth
+        if expiry:
+            suffix += f'.{expiry}'
+
+    # convert named tuples to dicts so we send usable keys
+    new_ticks = []
+    for tick in ticker.ticks:
+        if tick and not isinstance(tick, dict):
+            td = tick._asdict()
+            td['type'] = tick_types.get(
+                td['tickType'],
+                'n/a',
+            )
+
+            new_ticks.append(td)
+
+            tbt = ticker.tickByTicks
+            if tbt:
+                print(f'tickbyticks:\n {ticker.tickByTicks}')
+
+    ticker.ticks = new_ticks
+
+    # some contracts don't have volume so we may want to calculate
+    # a midpoint price based on data we can acquire (such as bid / ask)
+    if calc_price:
+        ticker.ticks.append(
+            {'type': 'trade', 'price': ticker.marketPrice()}
+        )
+
+    # serialize for transport
+    data = asdict(ticker)
+
+    # generate fqsn with possible specialized suffix
+    # for derivatives, note the lowercase.
+    data['symbol'] = data['fqsn'] = '.'.join(
+        (con.symbol, suffix)
+    ).lower()
+
+    # convert named tuples to dicts for transport
+    tbts = data.get('tickByTicks')
+    if tbts:
+        data['tickByTicks'] = [tbt._asdict() for tbt in tbts]
+
+    # add time stamps for downstream latency measurements
+    data['brokerd_ts'] = time.time()
+
+    # stupid stupid shit...don't even care any more..
+    # leave it until we do a proper latency study
+    # if ticker.rtTime is not None:
+    #     data['broker_ts'] = data['rtTime_s'] = float(
+    #         ticker.rtTime.timestamp) / 1000.
+    data.pop('rtTime')
+
+    return data
+
+
+async def stream_quotes(
+
+    send_chan: trio.abc.SendChannel,
+    symbols: list[str],
+    feed_is_live: trio.Event,
+    loglevel: str = None,
+
+    # startup sync
+    task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
+
+) -> None:
+    '''
+    Stream symbol quotes.
+
+    This is a ``trio`` callable routine meant to be invoked
+    once the brokerd is up.
+
+    '''
+    # TODO: support multiple subscriptions
+    sym = symbols[0]
+    log.info(f'request for real-time quotes: {sym}')
+
+    async with open_data_client() as proxy:
+
+        con, first_ticker, details = await proxy.get_sym_details(symbol=sym)
+        first_quote = normalize(first_ticker)
+        # print(f'first quote: {first_quote}')
+
+        def mk_init_msgs() -> dict[str, dict]:
+            '''
+            Collect a bunch of meta-data useful for feed startup and
+            pack in a `dict`-msg.
+
+            '''
+            # pass back some symbol info like min_tick, trading_hours, etc.
+            syminfo = asdict(details)
+            syminfo.update(syminfo['contract'])
+
+            # nested dataclass we probably don't need and that won't IPC
+            # serialize
+            syminfo.pop('secIdList')
+
+            # TODO: more consistent field translation
+            atype = syminfo['asset_type'] = asset_type_map[syminfo['secType']]
+
+            # for stocks it seems TWS reports too small a tick size
+            # such that you can't submit orders with that granularity?
+            min_tick = 0.01 if atype == 'stock' else 0
+
+            syminfo['price_tick_size'] = max(syminfo['minTick'], min_tick)
+
+            # for "traditional" assets, volume is normally discreet, not
+            # a float
+            syminfo['lot_tick_size'] = 0.0
+
+            ibclient = proxy._aio_ns.ib.client
+            host, port = ibclient.host, ibclient.port
+
+            # TODO: for loop through all symbols passed in
+            init_msgs = {
+                # pass back token, and bool, signalling if we're the writer
+                # and that history has been written
+                sym: {
+                    'symbol_info': syminfo,
+                    'fqsn': first_quote['fqsn'],
+                },
+                'status': {
+                    'data_ep': f'{host}:{port}',
+                },
+
+            }
+            return init_msgs
+
+        init_msgs = mk_init_msgs()
+
+        # TODO: we should instead spawn a task that waits on a feed to start
+        # and let it wait indefinitely..instead of this hard coded stuff.
+        with trio.move_on_after(1):
+            contract, first_ticker, details = await proxy.get_quote(symbol=sym)
+
+        # it might be outside regular trading hours so see if we can at
+        # least grab history.
+        if isnan(first_ticker.last):
+            task_status.started((init_msgs, first_quote))
+
+            # it's not really live but this will unblock
+            # the brokerd feed task to tell the ui to update?
+            feed_is_live.set()
+
+            # block and let data history backfill code run.
+            await trio.sleep_forever()
+            return  # we never expect feed to come up?
+
+        async with open_aio_quote_stream(
+            symbol=sym,
+            contract=con,
+        ) as stream:
+
+            # ugh, clear ticks since we've consumed them
+            # (ahem, ib_insync is stateful trash)
+            first_ticker.ticks = []
+
+            task_status.started((init_msgs, first_quote))
+
+            async with aclosing(stream):
+                if type(first_ticker.contract) not in (
+                    ibis.Commodity,
+                    ibis.Forex
+                ):
+                    # wait for real volume on feed (trading might be closed)
+                    while True:
+                        ticker = await stream.receive()
+
+                        # for a real volume contract we rait for the first
+                        # "real" trade to take place
+                        if (
+                            # not calc_price
+                            # and not ticker.rtTime
+                            not ticker.rtTime
+                        ):
+                            # spin consuming tickers until we get a real
+                            # market datum
+                            log.debug(f"New unsent ticker: {ticker}")
+                            continue
+                        else:
+                            log.debug("Received first real volume tick")
+                            # ugh, clear ticks since we've consumed them
+                            # (ahem, ib_insync is truly stateful trash)
+                            ticker.ticks = []
+
+                            # XXX: this works because we don't use
+                            # ``aclosing()`` above?
+                            break
+
+                    quote = normalize(ticker)
+                    log.debug(f"First ticker received {quote}")
+
+                # tell caller quotes are now coming in live
+                feed_is_live.set()
+
+                # last = time.time()
+                async for ticker in stream:
+                    quote = normalize(ticker)
+                    await send_chan.send({quote['fqsn']: quote})
+
+                    # ugh, clear ticks since we've consumed them
+                    ticker.ticks = []
+                    # last = time.time()
+
+
+async def data_reset_hack(
+    reset_type: str = 'data',
+
+) -> None:
+    '''
+    Run key combos for resetting data feeds and yield back to caller
+    when complete.
+
+    This is a linux-only hack around:
+
+    https://interactivebrokers.github.io/tws-api/historical_limitations.html#pacing_violations
+
+    TODOs:
+        - a return type that hopefully determines if the hack was
+          successful.
+        - other OS support?
+        - integration with ``ib-gw`` run in docker + Xorg?
+
+    '''
+
+    async def vnc_click_hack(
+        reset_type: str = 'data'
+    ) -> None:
+        '''
+        Reset the data or netowork connection for the VNC attached
+        ib gateway using magic combos.
+
+        '''
+        key = {'data': 'f', 'connection': 'r'}[reset_type]
+
+        import asyncvnc
+
+        async with asyncvnc.connect(
+            'localhost',
+            port=3003,
+            # password='ibcansmbz',
+        ) as client:
+
+            # move to middle of screen
+            # 640x1800
+            client.mouse.move(
+                x=500,
+                y=500,
+            )
+            client.mouse.click()
+            client.keyboard.press('Ctrl', 'Alt', key)  # keys are stacked
+
+    await tractor.to_asyncio.run_task(vnc_click_hack)
+
+    # we don't really need the ``xdotool`` approach any more B)
+    return True
+
+
+@tractor.context
+async def open_symbol_search(
+    ctx: tractor.Context,
+
+) -> None:
+
+    # TODO: load user defined symbol set locally for fast search?
+    await ctx.started({})
+
+    async with open_data_client() as proxy:
+        async with ctx.open_stream() as stream:
+
+            last = time.time()
+
+            async for pattern in stream:
+                log.debug(f'received {pattern}')
+                now = time.time()
+
+                assert pattern, 'IB can not accept blank search pattern'
+
+                # throttle search requests to no faster then 1Hz
+                diff = now - last
+                if diff < 1.0:
+                    log.debug('throttle sleeping')
+                    await trio.sleep(diff)
+                    try:
+                        pattern = stream.receive_nowait()
+                    except trio.WouldBlock:
+                        pass
+
+                if not pattern or pattern.isspace():
+                    log.warning('empty pattern received, skipping..')
+
+                    # TODO: *BUG* if nothing is returned here the client
+                    # side will cache a null set result and not showing
+                    # anything to the use on re-searches when this query
+                    # timed out. We probably need a special "timeout" msg
+                    # or something...
+
+                    # XXX: this unblocks the far end search task which may
+                    # hold up a multi-search nursery block
+                    await stream.send({})
+
+                    continue
+
+                log.debug(f'searching for {pattern}')
+
+                last = time.time()
+
+                # async batch search using api stocks endpoint and module
+                # defined adhoc symbol set.
+                stock_results = []
+
+                async def stash_results(target: Awaitable[list]):
+                    stock_results.extend(await target)
+
+                async with trio.open_nursery() as sn:
+                    sn.start_soon(
+                        stash_results,
+                        proxy.search_symbols(
+                            pattern=pattern,
+                            upto=5,
+                        ),
+                    )
+
+                    # trigger async request
+                    await trio.sleep(0)
+
+                    # match against our ad-hoc set immediately
+                    adhoc_matches = fuzzy.extractBests(
+                        pattern,
+                        list(_adhoc_futes_set),
+                        score_cutoff=90,
+                    )
+                    log.info(f'fuzzy matched adhocs: {adhoc_matches}')
+                    adhoc_match_results = {}
+                    if adhoc_matches:
+                        # TODO: do we need to pull contract details?
+                        adhoc_match_results = {i[0]: {} for i in adhoc_matches}
+
+                log.debug(f'fuzzy matching stocks {stock_results}')
+                stock_matches = fuzzy.extractBests(
+                    pattern,
+                    stock_results,
+                    score_cutoff=50,
+                )
+
+                matches = adhoc_match_results | {
+                    item[0]: {} for item in stock_matches
+                }
+                # TODO: we used to deliver contract details
+                # {item[2]: item[0] for item in stock_matches}
+
+                log.debug(f"sending matches: {matches.keys()}")
+                await stream.send(matches)

piker/brokers/questrade.py

@@ -19,7 +19,6 @@ Questrade API backend.
 """
 from __future__ import annotations
 import inspect
-import contextlib
 import time
 from datetime import datetime
 from functools import partial
@@ -32,22 +31,19 @@ from typing import (
     Callable,
 )
 
-import arrow
+import pendulum
 import trio
 import tractor
 from async_generator import asynccontextmanager
-import pandas as pd
 import numpy as np
 import wrapt
 import asks
 
 from ..calc import humanize, percent_change
-from . import config
+from .._cacheables import open_cached_client, async_lifo_cache
+from .. import config
 from ._util import resproc, BrokerError, SymbolNotFound
 from ..log import get_logger, colorize_json, get_console_log
-from .._async_utils import async_lifo_cache
-from . import get_brokermod
-from . import api
 
 
 log = get_logger(__name__)
@@ -602,12 +598,16 @@ class Client:
         sid = sids[symbol]
 
         # get last market open end time
-        est_end = now = arrow.utcnow().to('US/Eastern').floor('minute')
+        est_end = now = pendulum.now('UTC').in_timezoe(
+            'America/New_York').start_of('minute')
+
         # on non-paid feeds we can't retreive the first 15 mins
         wd = now.isoweekday()
         if wd > 5:
             quotes = await self.quote([symbol])
-            est_end = arrow.get(quotes[0]['lastTradeTime'])
+            est_end = pendulum.parse(
+                quotes[0]['lastTradeTime']
+            )
             if est_end.hour == 0:
                 # XXX don't bother figuring out extended hours for now
                 est_end = est_end.replace(hour=17)
@@ -668,7 +668,7 @@ def get_OHLCV(
     """
     del bar['end']
     del bar['VWAP']
-    bar['start'] = pd.Timestamp(bar['start']).value/10**9
+    bar['start'] = pendulum.from_timestamp(bar['start']) / 10**9
     return tuple(bar.values())
 
 
@@ -1197,7 +1197,7 @@ async def stream_quotes(
     # XXX: required to propagate ``tractor`` loglevel to piker logging
     get_console_log(loglevel)
 
-    async with api.open_cached_client('questrade') as client:
+    async with open_cached_client('questrade') as client:
         if feed_type == 'stock':
             formatter = format_stock_quote
             get_quotes = await stock_quoter(client, symbols)

piker/calc.py

@@ -20,30 +20,84 @@ Handy financial calculations.
 import math
 import itertools
 
+from bidict import bidict
 
-def humanize(number, digits=1):
-    """Convert large numbers to something with at most 3 digits and
+
+_mag2suffix = bidict({3: 'k', 6: 'M', 9: 'B'})
+
+
+def humanize(
+    number: float,
+    digits: int = 1
+
+) -> str:
+    '''
+    Convert large numbers to something with at most ``digits`` and
     a letter suffix (eg. k: thousand, M: million, B: billion).
-    """
+
+    '''
     try:
         float(number)
     except ValueError:
-        return 0
+        return '0'
+
     if not number or number <= 0:
-        return number
-    mag2suffix = {3: 'k', 6: 'M', 9: 'B'}
-    mag = math.floor(math.log(number, 10))
+        return str(round(number, ndigits=digits))
+
+    mag = round(math.log(number, 10))
     if mag < 3:
-        return number
-    maxmag = max(itertools.takewhile(lambda key: mag >= key, mag2suffix))
-    return "{:.{digits}f}{}".format(
-        number/10**maxmag, mag2suffix[maxmag], digits=digits)
+        return str(round(number, ndigits=digits))
+
+    maxmag = max(
+        itertools.takewhile(
+            lambda key: mag >= key, _mag2suffix
+        )
+    )
+
+    return "{value}{suffix}".format(
+        value=round(number/10**maxmag, ndigits=digits),
+        suffix=_mag2suffix[maxmag],
+    )
 
 
-def percent_change(init, new):
-    """Calcuate the percentage change of some ``new`` value
+def puterize(
+
+    text: str,
+    digits: int = 1,
+
+) -> float:
+    '''Inverse of ``humanize()`` above.
+
+    '''
+    try:
+        suffix = str(text)[-1]
+        mult = _mag2suffix.inverse[suffix]
+        value = text.rstrip(suffix)
+        return round(float(value) * 10**mult, ndigits=digits)
+
+    except KeyError:
+        # no matching suffix try just the value
+        return float(text)
+
+
+def pnl(
+
+    init: float,
+    new: float,
+
+) -> float:
+    '''Calcuate the percentage change of some ``new`` value
     from some initial value, ``init``.
-    """
+
+    '''
     if not (init and new):
         return 0
-    return (new - init) / init * 100.
+
+    return (new - init) / init
+
+
+def percent_change(
+    init: float,
+    new: float,
+) -> float:
+    return pnl(init, new) * 100.

piker/clearing/_allocate.py

@@ -0,0 +1,359 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Position allocation logic and protocols.
+
+'''
+from enum import Enum
+from typing import Optional
+
+from bidict import bidict
+from pydantic import BaseModel, validator
+
+from ..data._source import Symbol
+from ._messages import BrokerdPosition, Status
+
+
+class Position(BaseModel):
+    '''
+    Basic pp (personal position) model with attached fills history.
+
+    This type should be IPC wire ready?
+
+    '''
+    symbol: Symbol
+
+    # last size and avg entry price
+    size: float
+    avg_price: float  # TODO: contextual pricing
+
+    # ordered record of known constituent trade messages
+    fills: list[Status] = []
+
+    def update_from_msg(
+        self,
+        msg: BrokerdPosition,
+
+    ) -> None:
+
+        # XXX: better place to do this?
+        symbol = self.symbol
+
+        lot_size_digits = symbol.lot_size_digits
+        avg_price, size = (
+            round(msg['avg_price'], ndigits=symbol.tick_size_digits),
+            round(msg['size'], ndigits=lot_size_digits),
+        )
+
+        self.avg_price = avg_price
+        self.size = size
+
+    @property
+    def dsize(self) -> float:
+        '''
+        The "dollar" size of the pp, normally in trading (fiat) unit
+        terms.
+
+        '''
+        return self.avg_price * self.size
+
+
+_size_units = bidict({
+    'currency': '$ size',
+    'units': '# units',
+    # TODO: but we'll need a `<brokermod>.get_accounts()` or something
+    # 'percent_of_port': '% of port',
+})
+SizeUnit = Enum(
+    'SizeUnit',
+    _size_units,
+)
+
+
+class Allocator(BaseModel):
+
+    class Config:
+        validate_assignment = True
+        copy_on_model_validation = False
+        arbitrary_types_allowed = True
+
+        # required to get the account validator lookup working?
+        extra = 'allow'
+        underscore_attrs_are_private = False
+
+    symbol: Symbol
+    account: Optional[str] = 'paper'
+    # TODO: for enums this clearly doesn't fucking work, you can't set
+    # a default at startup by passing in a `dict` but yet you can set
+    # that value through assignment..for wtv cucked reason.. honestly, pure
+    # unintuitive garbage.
+    size_unit: str = 'currency'
+    _size_units: dict[str, Optional[str]] = _size_units
+
+    @validator('size_unit', pre=True)
+    def maybe_lookup_key(cls, v):
+        # apply the corresponding enum key for the text "description" value
+        if v not in _size_units:
+            return _size_units.inverse[v]
+
+        assert v in _size_units
+        return v
+
+    # TODO: if we ever want ot support non-uniform entry-slot-proportion
+    # "sizes"
+    # disti_weight: str = 'uniform'
+
+    units_limit: float
+    currency_limit: float
+    slots: int
+
+    def step_sizes(
+        self,
+    ) -> (float, float):
+        '''
+        Return the units size for each unit type as a tuple.
+
+        '''
+        slots = self.slots
+        return (
+            self.units_limit / slots,
+            self.currency_limit / slots,
+        )
+
+    def limit(self) -> float:
+        if self.size_unit == 'currency':
+            return self.currency_limit
+        else:
+            return self.units_limit
+
+    def next_order_info(
+        self,
+
+        # we only need a startup size for exit calcs, we can the
+        # determine how large slots should be if the initial pp size was
+        # larger then the current live one, and the live one is smaller
+        # then the initial config settings.
+        startup_pp: Position,
+        live_pp: Position,
+        price: float,
+        action: str,
+
+    ) -> dict:
+        '''
+        Generate order request info for the "next" submittable order
+        depending on position / order entry config.
+
+        '''
+        sym = self.symbol
+        ld = sym.lot_size_digits
+
+        size_unit = self.size_unit
+        live_size = live_pp.size
+        abs_live_size = abs(live_size)
+        abs_startup_size = abs(startup_pp.size)
+
+        u_per_slot, currency_per_slot = self.step_sizes()
+
+        if size_unit == 'units':
+            slot_size = u_per_slot
+            l_sub_pp = self.units_limit - abs_live_size
+
+        elif size_unit == 'currency':
+            live_cost_basis = abs_live_size * live_pp.avg_price
+            slot_size = currency_per_slot / price
+            l_sub_pp = (self.currency_limit - live_cost_basis) / price
+
+        else:
+            raise ValueError(
+                f"Not valid size unit '{size_unit}'"
+            )
+
+        # an entry (adding-to or starting a pp)
+        if (
+            action == 'buy' and live_size > 0 or
+            action == 'sell' and live_size < 0 or
+            live_size == 0
+        ):
+
+            order_size = min(slot_size, l_sub_pp)
+
+        # an exit (removing-from or going to net-zero pp)
+        else:
+            # when exiting a pp we always try to slot the position
+            # in the instrument's units, since doing so in a derived
+            # size measure (eg. currency value, percent of port) would
+            # result in a mis-mapping of slots sizes in unit terms
+            # (i.e. it would take *more* slots to exit at a profit and
+            # *less* slots to exit at a loss).
+            pp_size = max(abs_startup_size, abs_live_size)
+            slotted_pp = pp_size / self.slots
+
+            if size_unit == 'currency':
+                # compute the "projected" limit's worth of units at the
+                # current pp (weighted) price:
+                slot_size = currency_per_slot / live_pp.avg_price
+
+            else:
+                slot_size = u_per_slot
+
+            # TODO: ensure that the limit can never be set **lower**
+            # then the current pp size? It should be configured
+            # correctly at startup right?
+
+            # if our position is greater then our limit setting
+            # we'll want to use slot sizes which are larger then what
+            # the limit would normally determine.
+            order_size = max(slotted_pp, slot_size)
+
+            if (
+                abs_live_size < slot_size or
+
+                # NOTE: front/back "loading" heurstic:
+                # if the remaining pp is in between 0-1.5x a slot's
+                # worth, dump the whole position in this last exit
+                # therefore conducting so called "back loading" but
+                # **without** going past a net-zero pp. if the pp is
+                # > 1.5x a slot size, then front load: exit a slot's and
+                # expect net-zero to be acquired on the final exit.
+                slot_size < pp_size < round((1.5*slot_size), ndigits=ld) or
+
+                # underlying requires discrete (int) units (eg. stocks)
+                # and thus our slot size (based on our limit) would
+                # exit a fractional unit's worth so, presuming we aren't
+                # supporting a fractional-units-style broker, we need
+                # exit the final unit.
+                ld == 0 and abs_live_size == 1
+            ):
+                order_size = abs_live_size
+
+        slots_used = 1.0  # the default uniform policy
+        if order_size < slot_size:
+            # compute a fractional slots size to display
+            slots_used = self.slots_used(
+                Position(symbol=sym, size=order_size, avg_price=price)
+            )
+
+        return {
+            'size': abs(round(order_size, ndigits=ld)),
+            'size_digits': ld,
+
+            # TODO: incorporate multipliers for relevant derivatives
+            'fiat_size': round(order_size * price, ndigits=2),
+            'slots_used': slots_used,
+
+            # update line LHS label with account name
+            'account': self.account,
+        }
+
+    def slots_used(
+        self,
+        pp: Position,
+
+    ) -> float:
+        '''
+        Calc and return the number of slots used by this ``Position``.
+
+        '''
+        abs_pp_size = abs(pp.size)
+
+        if self.size_unit == 'currency':
+            # live_currency_size = size or (abs_pp_size * pp.avg_price)
+            live_currency_size = abs_pp_size * pp.avg_price
+            prop = live_currency_size / self.currency_limit
+
+        else:
+            # return (size or abs_pp_size) / alloc.units_limit
+            prop = abs_pp_size / self.units_limit
+
+        # TODO: REALLY need a way to show partial slots..
+        # for now we round at the midway point between slots
+        return round(prop * self.slots)
+
+
+_derivs = (
+    'future',
+    'continuous_future',
+    'option',
+    'futures_option',
+)
+
+
+def mk_allocator(
+
+    symbol: Symbol,
+    startup_pp: Position,
+
+    # default allocation settings
+    defaults: dict[str, float] = {
+        'account': None,  # select paper by default
+        'size_unit': 'currency',
+        'units_limit': 400,
+        'currency_limit': 5e3,
+        'slots': 4,
+    },
+    **kwargs,
+
+) -> Allocator:
+
+    if kwargs:
+        defaults.update(kwargs)
+
+    # load and retreive user settings for default allocations
+    # ``config.toml``
+    user_def = {
+        'currency_limit': 6e3,
+        'slots': 6,
+    }
+
+    defaults.update(user_def)
+
+    alloc = Allocator(
+        symbol=symbol,
+        **defaults,
+    )
+
+    asset_type = symbol.type_key
+
+    # specific configs by asset class / type
+
+    if asset_type in _derivs:
+        # since it's harder to know how currency "applies" in this case
+        # given leverage properties
+        alloc.size_unit = '# units'
+
+        # set units limit to slots size thus making make the next
+        # entry step 1.0
+        alloc.units_limit = alloc.slots
+
+    # if the current position is already greater then the limit
+    # settings, increase the limit to the current position
+    if alloc.size_unit == 'currency':
+        startup_size = startup_pp.size * startup_pp.avg_price
+
+        if startup_size > alloc.currency_limit:
+            alloc.currency_limit = round(startup_size, ndigits=2)
+
+    else:
+        startup_size = abs(startup_pp.size)
+
+        if startup_size > alloc.units_limit:
+            alloc.units_limit = startup_size
+
+            if asset_type in _derivs:
+                alloc.slots = alloc.units_limit
+
+    return alloc

piker/clearing/_client.py

@@ -18,15 +18,15 @@
 Orders and execution client API.
 
 """
-from contextlib import asynccontextmanager
+from contextlib import asynccontextmanager as acm
 from typing import Dict
 from pprint import pformat
 from dataclasses import dataclass, field
 
 import trio
 import tractor
+from tractor.trionics import broadcast_receiver
 
-from ..data._source import Symbol
 from ..log import get_logger
 from ._ems import _emsd_main
 from .._daemon import maybe_open_emsd
@@ -38,7 +38,7 @@ log = get_logger(__name__)
 
 @dataclass
 class OrderBook:
-    """Buy-side (client-side ?) order book ctl and tracking.
+    '''EMS-client-side order book ctl and tracking.
 
     A style similar to "model-view" is used here where this api is
     provided as a supervised control for an EMS actor which does all the
@@ -48,7 +48,7 @@ class OrderBook:
     Currently, this is mostly for keeping local state to match the EMS
     and use received events to trigger graphics updates.
 
-    """
+    '''
     # mem channels used to relay order requests to the EMS daemon
     _to_ems: trio.abc.SendChannel
     _from_order_book: trio.abc.ReceiveChannel
@@ -57,35 +57,20 @@ class OrderBook:
     _ready_to_receive: trio.Event = trio.Event()
 
     def send(
-
         self,
-        uuid: str,
-        symbol: str,
-        brokers: list[str],
-        price: float,
-        size: float,
-        action: str,
-        exec_mode: str,
+        msg: Order,
 
     ) -> dict:
-        msg = Order(
-            action=action,
-            price=price,
-            size=size,
-            symbol=symbol,
-            brokers=brokers,
-            oid=uuid,
-            exec_mode=exec_mode,  # dark or live
-        )
-
-        self._sent_orders[uuid] = msg
+        self._sent_orders[msg.oid] = msg
         self._to_ems.send_nowait(msg.dict())
         return msg
 
     def update(
         self,
+
         uuid: str,
         **data: dict,
+
     ) -> dict:
         cmd = self._sent_orders[uuid]
         msg = cmd.dict()
@@ -123,10 +108,15 @@ def get_orders(
     global _orders
 
     if _orders is None:
+        size = 100
+        tx, rx = trio.open_memory_channel(size)
+        brx = broadcast_receiver(rx, size)
+
         # setup local ui event streaming channels for request/resp
         # streamging with EMS daemon
         _orders = OrderBook(
-            *trio.open_memory_channel(100),
+            _to_ems=tx,
+            _from_order_book=brx,
         )
 
     return _orders
@@ -157,35 +147,27 @@ async def relay_order_cmds_from_sync_code(
 
     """
     book = get_orders()
-    orders_stream = book._from_order_book
-
-    async for cmd in orders_stream:
-
-        print(cmd)
-        if cmd['symbol'] == symbol_key:
-
-            # send msg over IPC / wire
-            log.info(f'Send order cmd:\n{pformat(cmd)}')
-            await to_ems_stream.send(cmd)
-
-        else:
-            # XXX BRUTAL HACKZORZES !!!
-            # re-insert for another consumer
-            # we need broadcast channelz...asap
-            # https://github.com/goodboy/tractor/issues/204
-            book._to_ems.send_nowait(cmd)
+    async with book._from_order_book.subscribe() as orders_stream:
+        async for cmd in orders_stream:
+            if cmd['symbol'] == symbol_key:
+                log.info(f'Send order cmd:\n{pformat(cmd)}')
+                # send msg over IPC / wire
+                await to_ems_stream.send(cmd)
 
 
-@asynccontextmanager
+@acm
 async def open_ems(
-    broker: str,
-    symbol: Symbol,
+    fqsn: str,
 
-) -> (OrderBook, tractor.MsgStream, dict):
-    """Spawn an EMS daemon and begin sending orders and receiving
+) -> (
+    OrderBook,
+    tractor.MsgStream,
+    dict,
+):
+    '''
+    Spawn an EMS daemon and begin sending orders and receiving
     alerts.
 
-
     This EMS tries to reduce most broker's terrible order entry apis to
     a very simple protocol built on a few easy to grok and/or
     "rantsy" premises:
@@ -214,23 +196,24 @@ async def open_ems(
     - 'dark_executed', 'broker_executed'
     - 'broker_filled'
 
-    """
+    '''
     # wait for service to connect back to us signalling
     # ready for order commands
     book = get_orders()
 
+    from ..data._source import unpack_fqsn
+    broker, symbol, suffix = unpack_fqsn(fqsn)
+
     async with maybe_open_emsd(broker) as portal:
 
         async with (
-
             # connect to emsd
             portal.open_context(
 
                 _emsd_main,
-                broker=broker,
-                symbol=symbol.key,
+                fqsn=fqsn,
 
-            ) as (ctx, positions),
+            ) as (ctx, (positions, accounts)),
 
             # open 2-way trade command stream
             ctx.open_stream() as trades_stream,
@@ -238,8 +221,8 @@ async def open_ems(
             async with trio.open_nursery() as n:
                 n.start_soon(
                     relay_order_cmds_from_sync_code,
-                    symbol.key,
+                    fqsn,
                     trades_stream
                 )
 
-                yield book, trades_stream, positions
+                yield book, trades_stream, positions, accounts

piker/clearing/_ems.py

@@ -22,7 +22,7 @@ from contextlib import asynccontextmanager
 from dataclasses import dataclass, field
 from pprint import pformat
 import time
-from typing import AsyncIterator, Callable, Any
+from typing import AsyncIterator, Callable
 
 from bidict import bidict
 from pydantic import BaseModel
@@ -30,10 +30,9 @@ import trio
 from trio_typing import TaskStatus
 import tractor
 
-from .. import data
 from ..log import get_logger
 from ..data._normalize import iterticks
-from ..data.feed import Feed
+from ..data.feed import Feed, maybe_open_feed
 from .._daemon import maybe_spawn_brokerd
 from . import _paper_engine as paper
 from ._messages import (
@@ -48,11 +47,14 @@ log = get_logger(__name__)
 
 # TODO: numba all of this
 def mk_check(
+
     trigger_price: float,
     known_last: float,
     action: str,
+
 ) -> Callable[[float, float], bool]:
-    """Create a predicate for given ``exec_price`` based on last known
+    '''
+    Create a predicate for given ``exec_price`` based on last known
     price, ``known_last``.
 
     This is an automatic alert level thunk generator based on where the
@@ -60,7 +62,7 @@ def mk_check(
     interest is; pick an appropriate comparison operator based on
     avoiding the case where the a predicate returns true immediately.
 
-    """
+    '''
     # str compares:
     # https://stackoverflow.com/questions/46708708/compare-strings-in-numba-compiled-function
 
@@ -78,8 +80,9 @@ def mk_check(
 
         return check_lt
 
-    else:
-        return None
+    raise ValueError(
+        f'trigger: {trigger_price}, last: {known_last}'
+    )
 
 
 @dataclass
@@ -111,8 +114,8 @@ class _DarkBook:
 
     # tracks most recent values per symbol each from data feed
     lasts: dict[
-        tuple[str, str],
-        float
+        str,
+        float,
     ] = field(default_factory=dict)
 
     # mapping of piker ems order ids to current brokerd order flow message
@@ -123,7 +126,7 @@ class _DarkBook:
 # XXX: this is in place to prevent accidental positions that are too
 # big. Now obviously this won't make sense for crypto like BTC, but
 # for most traditional brokers it should be fine unless you start
-# slinging NQ futes or something.
+# slinging NQ futes or something; check ur margin.
 _DEFAULT_SIZE: float = 1.0
 
 
@@ -132,42 +135,43 @@ async def clear_dark_triggers(
     brokerd_orders_stream: tractor.MsgStream,
     ems_client_order_stream: tractor.MsgStream,
     quote_stream: tractor.ReceiveMsgStream,  # noqa
-
     broker: str,
-    symbol: str,
+    fqsn: str,
 
     book: _DarkBook,
 
 ) -> None:
-    """Core dark order trigger loop.
+    '''
+    Core dark order trigger loop.
 
     Scan the (price) data feed and submit triggered orders
     to broker.
 
-    """
-    # this stream may eventually contain multiple symbols
+    '''
     # XXX: optimize this for speed!
+    # TODO:
+    # - numba all this!
+    # - this stream may eventually contain multiple symbols
     async for quotes in quote_stream:
-
-        # TODO: numba all this!
-
         # start = time.time()
         for sym, quote in quotes.items():
-
-            execs = book.orders.get(sym, None)
-            if execs is None:
-                continue
-
+            execs = book.orders.get(sym, {})
             for tick in iterticks(
                 quote,
                 # dark order price filter(s)
-                types=('ask', 'bid', 'trade', 'last')
+                types=(
+                    'ask',
+                    'bid',
+                    'trade',
+                    'last',
+                    # 'dark_trade',  # TODO: should allow via config?
+                )
             ):
                 price = tick.get('price')
                 ttype = tick['type']
 
                 # update to keep new cmds informed
-                book.lasts[(broker, symbol)] = price
+                book.lasts[sym] = price
 
                 for oid, (
                     pred,
@@ -178,13 +182,21 @@ async def clear_dark_triggers(
                 ) in (
                     tuple(execs.items())
                 ):
-
-                    if not pred or (ttype not in tf) or (not pred(price)):
+                    if (
+                        not pred or
+                        ttype not in tf or
+                        not pred(price)
+                    ):
+                        log.debug(
+                            f'skipping quote for {sym} '
+                            f'{pred}, {ttype} not in {tf}?, {pred(price)}'
+                        )
                         # majority of iterations will be non-matches
                         continue
 
                     action: str = cmd['action']
                     symbol: str = cmd['symbol']
+                    bfqsn: str = symbol.replace(f'.{broker}', '')
 
                     if action == 'alert':
                         # nothing to do but relay a status
@@ -203,6 +215,7 @@ async def clear_dark_triggers(
                         msg = BrokerdOrder(
                             action=cmd['action'],
                             oid=oid,
+                            account=cmd['account'],
                             time_ns=time.time_ns(),
 
                             # this **creates** new order request for the
@@ -213,7 +226,7 @@ async def clear_dark_triggers(
                             # order-request and instead create a new one.
                             reqid=None,
 
-                            symbol=sym,
+                            symbol=bfqsn,
                             price=submit_price,
                             size=cmd['size'],
                         )
@@ -235,39 +248,59 @@ async def clear_dark_triggers(
                         oid=oid,  # ems order id
                         resp=resp,
                         time_ns=time.time_ns(),
-
-                        symbol=symbol,
+                        symbol=fqsn,
                         trigger_price=price,
-
                         broker_details={'name': broker},
-
                         cmd=cmd,  # original request message
 
                     ).dict()
 
                     # remove exec-condition from set
                     log.info(f'removing pred for {oid}')
-                    execs.pop(oid)
+                    pred = execs.pop(oid, None)
+                    if not pred:
+                        log.warning(
+                            f'pred for {oid} was already removed!?'
+                        )
 
-                    await ems_client_order_stream.send(msg)
+                    try:
+                        await ems_client_order_stream.send(msg)
+                    except (
+                        trio.ClosedResourceError,
+                    ):
+                        log.warning(
+                            f'client {ems_client_order_stream} stream is broke'
+                        )
+                        break
 
                 else:  # condition scan loop complete
                     log.debug(f'execs are {execs}')
                     if execs:
-                        book.orders[symbol] = execs
+                        book.orders[fqsn] = execs
 
         # print(f'execs scan took: {time.time() - start}')
 
 
 @dataclass
 class TradesRelay:
+
+    # for now we keep only a single connection open with
+    # each ``brokerd`` for simplicity.
     brokerd_dialogue: tractor.MsgStream
-    positions: dict[str, float]
+
+    # map of symbols to dicts of accounts to pp msgs
+    positions: dict[str, dict[str, BrokerdPosition]]
+
+    # allowed account names
+    accounts: tuple[str]
+
+    # count of connected ems clients for this ``brokerd``
     consumers: int = 0
 
 
-class _Router(BaseModel):
-    '''Order router which manages and tracks per-broker dark book,
+class Router(BaseModel):
+    '''
+    Order router which manages and tracks per-broker dark book,
     alerts, clearing and related data feed management.
 
     A singleton per ``emsd`` actor.
@@ -276,8 +309,6 @@ class _Router(BaseModel):
     # setup at actor spawn time
     nursery: trio.Nursery
 
-    feeds: dict[tuple[str, str], Any] = {}
-
     # broker to book map
     books: dict[str, _DarkBook] = {}
 
@@ -343,12 +374,12 @@ class _Router(BaseModel):
             relay.consumers -= 1
 
 
-_router: _Router = None
+_router: Router = None
 
 
 async def open_brokerd_trades_dialogue(
 
-    router: _Router,
+    router: Router,
     feed: Feed,
     symbol: str,
     _exec_mode: str,
@@ -357,7 +388,8 @@ async def open_brokerd_trades_dialogue(
     task_status: TaskStatus[TradesRelay] = trio.TASK_STATUS_IGNORED,
 
 ) -> tuple[dict, tractor.MsgStream]:
-    '''Open and yield ``brokerd`` trades dialogue context-stream if none
+    '''
+    Open and yield ``brokerd`` trades dialogue context-stream if none
     already exists.
 
     '''
@@ -366,7 +398,9 @@ async def open_brokerd_trades_dialogue(
     broker = feed.mod.name
 
     # TODO: make a `tractor` bug/test for this!
-    # portal = feed._brokerd_portal
+    # if only i could member what the problem was..
+    # probably some GC of the portal thing?
+    # portal = feed.portal
 
     # XXX: we must have our own portal + channel otherwise
     # when the data feed closes it may result in a half-closed
@@ -392,8 +426,7 @@ async def open_brokerd_trades_dialogue(
             # actor to simulate the real IPC load it'll have when also
             # pulling data from feeds
             open_trades_endpoint = paper.open_paperboi(
-                broker=broker,
-                symbol=symbol,
+                fqsn='.'.join([symbol, broker]),
                 loglevel=loglevel,
             )
 
@@ -405,9 +438,11 @@ async def open_brokerd_trades_dialogue(
             )
 
         try:
-            async with (
+            positions: list[BrokerdPosition]
+            accounts: tuple[str]
 
-                open_trades_endpoint as (brokerd_ctx, positions),
+            async with (
+                open_trades_endpoint as (brokerd_ctx, (positions, accounts,)),
                 brokerd_ctx.open_stream() as brokerd_trades_stream,
 
             ):
@@ -426,10 +461,24 @@ async def open_brokerd_trades_dialogue(
                 # normalizing them to EMS messages and relaying back to
                 # the piker order client set.
 
+                # locally cache and track positions per account.
+                pps = {}
+                for msg in positions:
+                    log.info(f'loading pp: {msg}')
+
+                    account = msg['account']
+                    assert account in accounts
+
+                    pps.setdefault(
+                        f'{msg["symbol"]}.{broker}',
+                        {}
+                    )[account] = msg
+
                 relay = TradesRelay(
                     brokerd_dialogue=brokerd_trades_stream,
-                    positions=positions,
-                    consumers=1
+                    positions=pps,
+                    accounts=accounts,
+                    consumers=1,
                 )
 
                 _router.relays[broker] = relay
@@ -451,7 +500,9 @@ async def open_brokerd_trades_dialogue(
         finally:
             # parent context must have been closed
             # remove from cache so next client will respawn if needed
-            _router.relays.pop(broker)
+            relay = _router.relays.pop(broker, None)
+            if not relay:
+                log.warning(f'Relay for {broker} was already removed!?')
 
 
 @tractor.context
@@ -466,7 +517,7 @@ async def _setup_persistent_emsd(
     # open a root "service nursery" for the ``emsd`` actor
     async with trio.open_nursery() as service_nursery:
 
-        _router = _Router(nursery=service_nursery)
+        _router = Router(nursery=service_nursery)
 
         # TODO: send back the full set of persistent
         # orders/execs?
@@ -480,10 +531,11 @@ async def translate_and_relay_brokerd_events(
 
     broker: str,
     brokerd_trades_stream: tractor.MsgStream,
-    router: _Router,
+    router: Router,
 
 ) -> AsyncIterator[dict]:
-    '''Trades update loop - receive updates from ``brokerd`` trades
+    '''
+    Trades update loop - receive updates from ``brokerd`` trades
     endpoint, convert to EMS response msgs, transmit **only** to
     ordering client(s).
 
@@ -511,19 +563,39 @@ async def translate_and_relay_brokerd_events(
 
         name = brokerd_msg['name']
 
-        log.info(f'Received broker trade event:\n{pformat(brokerd_msg)}')
+        log.info(
+            f'Received broker trade event:\n'
+            f'{pformat(brokerd_msg)}'
+        )
 
         if name == 'position':
 
             pos_msg = BrokerdPosition(**brokerd_msg).dict()
 
-            # keep up to date locally in ``emsd``
-            relay.positions.setdefault(pos_msg['symbol'], {}).update(pos_msg)
+            # XXX: this will be useful for automatic strats yah?
+            # keep pps per account up to date locally in ``emsd`` mem
+            sym, broker = pos_msg['symbol'], pos_msg['broker']
 
-            # relay through position msgs immediately by
+            relay.positions.setdefault(
+                # NOTE: translate to a FQSN!
+                f'{sym}.{broker}',
+                {}
+            ).setdefault(
+                pos_msg['account'], {}
+            ).update(pos_msg)
+
+            # fan-out-relay position msgs immediately by
             # broadcasting updates on all client streams
-            for client_stream in router.clients:
-                await client_stream.send(pos_msg)
+            for client_stream in router.clients.copy():
+                try:
+                    await client_stream.send(pos_msg)
+                except(
+                    trio.ClosedResourceError,
+                    trio.BrokenResourceError,
+                ):
+                    router.clients.remove(client_stream)
+                    log.warning(
+                        f'client for {client_stream} was already closed?')
 
             continue
 
@@ -546,19 +618,28 @@ async def translate_and_relay_brokerd_events(
             # packed at submission since we already know it ahead of
             # time
             paper = brokerd_msg['broker_details'].get('paper_info')
+            ext = brokerd_msg['broker_details'].get('external')
             if paper:
                 # paperboi keeps the ems id up front
                 oid = paper['oid']
 
-            else:
+            elif ext:
                 # may be an order msg specified as "external" to the
                 # piker ems flow (i.e. generated by some other
                 # external broker backend client (like tws for ib)
-                ext = brokerd_msg['broker_details'].get('external')
-                if ext:
-                    log.error(f"External trade event {ext}")
+                log.error(f"External trade event {ext}")
 
                 continue
+
+            else:
+                # something is out of order, we don't have an oid for
+                # this broker-side message.
+                log.error(
+                    'Unknown oid:{oid} for msg:\n'
+                    f'{pformat(brokerd_msg)}'
+                    'Unable to relay message to client side!?'
+                )
+
         else:
             # check for existing live flow entry
             entry = book._ems_entries.get(oid)
@@ -580,7 +661,8 @@ async def translate_and_relay_brokerd_events(
                 # cancelled by the ems controlling client before we
                 # received this ack, in which case we relay that cancel
                 # signal **asap** to the backend broker
-                if entry.action == 'cancel':
+                action = getattr(entry, 'action', None)
+                if action and action == 'cancel':
                     # assign newly providerd broker backend request id
                     entry.reqid = reqid
 
@@ -624,8 +706,11 @@ async def translate_and_relay_brokerd_events(
             # another stupid ib error to handle
             # if 10147 in message: cancel
 
+            resp = 'broker_errored'
+            broker_details = msg.dict()
+
             # don't relay message to order requester client
-            continue
+            # continue
 
         elif name in (
             'status',
@@ -704,7 +789,7 @@ async def process_client_order_cmds(
     symbol: str,
     feed: Feed,  # noqa
     dark_book: _DarkBook,
-    router: _Router,
+    router: Router,
 
 ) -> None:
 
@@ -744,6 +829,7 @@ async def process_client_order_cmds(
                     oid=oid,
                     reqid=reqid,
                     time_ns=time.time_ns(),
+                    account=live_entry.account,
                 )
 
                 # NOTE: cancel response will be relayed back in messages
@@ -751,7 +837,9 @@ async def process_client_order_cmds(
                 if reqid:
 
                     # send cancel to brokerd immediately!
-                    log.info("Submitting cancel for live order {reqid}")
+                    log.info(
+                        f'Submitting cancel for live order {reqid}'
+                    )
 
                     await brokerd_order_stream.send(msg.dict())
 
@@ -788,11 +876,15 @@ async def process_client_order_cmds(
 
             msg = Order(**cmd)
 
-            sym = msg.symbol
+            fqsn = msg.symbol
             trigger_price = msg.price
             size = msg.size
             exec_mode = msg.exec_mode
             broker = msg.brokers[0]
+            # remove the broker part before creating a message
+            # to send to the specific broker since they probably
+            # aren't expectig their own name, but should they?
+            sym = fqsn.replace(f'.{broker}', '')
 
             if exec_mode == 'live' and action in ('buy', 'sell',):
 
@@ -800,11 +892,10 @@ async def process_client_order_cmds(
 
                     # sanity check on emsd id
                     assert live_entry.oid == oid
-
+                    reqid = live_entry.reqid
                     # if we already had a broker order id then
                     # this is likely an order update commmand.
-                    log.info(
-                        f"Modifying live {broker} order: {live_entry.reqid}")
+                    log.info(f"Modifying live {broker} order: {reqid}")
 
                 msg = BrokerdOrder(
                     oid=oid,  # no ib support for oids...
@@ -818,6 +909,7 @@ async def process_client_order_cmds(
                     action=action,
                     price=trigger_price,
                     size=size,
+                    account=msg.account,
                 )
 
                 # send request to backend
@@ -850,7 +942,7 @@ async def process_client_order_cmds(
                 # price received from the feed, instead of being
                 # like every other shitty tina platform that makes
                 # the user choose the predicate operator.
-                last = dark_book.lasts[(broker, sym)]
+                last = dark_book.lasts[fqsn]
                 pred = mk_check(trigger_price, last, action)
 
                 spread_slap: float = 5
@@ -881,7 +973,7 @@ async def process_client_order_cmds(
                 # dark book entry if the order id already exists
 
                 dark_book.orders.setdefault(
-                    sym, {}
+                    fqsn, {}
                 )[oid] = (
                     pred,
                     tickfilter,
@@ -908,19 +1000,19 @@ async def process_client_order_cmds(
 async def _emsd_main(
 
     ctx: tractor.Context,
-    broker: str,
-    symbol: str,
+    fqsn: str,
+
     _exec_mode: str = 'dark',  # ('paper', 'dark', 'live')
     loglevel: str = 'info',
 
 ) -> None:
     '''EMS (sub)actor entrypoint providing the
     execution management (micro)service which conducts broker
-    order control on behalf of clients.
+    order clearing control on behalf of clients.
 
     This is the daemon (child) side routine which starts an EMS runtime
-    (one per broker-feed) and and begins streaming back alerts from
-    broker executions/fills.
+    task (one per broker-feed) and and begins streaming back alerts from
+    each broker's executions/fills.
 
     ``send_order_cmds()`` is called here to execute in a task back in
     the actor which started this service (spawned this actor), presuming
@@ -944,13 +1036,15 @@ async def _emsd_main(
           reponse" proxy-broker.
        |
         - ``process_client_order_cmds()``:
-          accepts order cmds from requesting piker clients, registers
-          execs with exec loop
+          accepts order cmds from requesting clients, registers dark orders and
+          alerts with clearing loop.
 
     '''
     global _router
     assert _router
 
+    from ..data._source import unpack_fqsn
+    broker, symbol, suffix = unpack_fqsn(fqsn)
     dark_book = _router.get_dark_book(broker)
 
     # TODO: would be nice if in tractor we can require either a ctx arg,
@@ -958,32 +1052,24 @@ async def _emsd_main(
     # tractor.Context instead of strictly requiring a ctx arg.
     ems_ctx = ctx
 
-    cached_feed = _router.feeds.get((broker, symbol))
-    if cached_feed:
-        # TODO: use cached feeds per calling-actor
-        log.warning(f'Opening duplicate feed for {(broker, symbol)}')
+    feed: Feed
 
     # spawn one task per broker feed
     async with (
-        # TODO: eventually support N-brokers
-        data.open_feed(
-            broker,
-            [symbol],
+        maybe_open_feed(
+            [fqsn],
             loglevel=loglevel,
-        ) as feed,
+        ) as (feed, quote_stream),
     ):
-        if not cached_feed:
-            _router.feeds[(broker, symbol)] = feed
 
         # XXX: this should be initial price quote from target provider
-        first_quote = feed.first_quote
+        first_quote = feed.first_quotes[fqsn]
+
+        book = _router.get_dark_book(broker)
+        book.lasts[fqsn] = first_quote['last']
 
         # open a stream with the brokerd backend for order
         # flow dialogue
-
-        book = _router.get_dark_book(broker)
-        book.lasts[(broker, symbol)] = first_quote[symbol]['last']
-
         async with (
 
             # only open if one isn't already up: we try to keep
@@ -1002,10 +1088,15 @@ async def _emsd_main(
 
             brokerd_stream = relay.brokerd_dialogue  # .clone()
 
-            # signal to client that we're started
-            # TODO: we could eventually send back **all** brokerd
-            # positions here?
-            await ems_ctx.started(relay.positions)
+            # flatten out collected pps from brokerd for delivery
+            pp_msgs = {
+                fqsn: list(pps.values())
+                for fqsn, pps in relay.positions.items()
+            }
+
+            # signal to client that we're started and deliver
+            # all known pps and accounts for this ``brokerd``.
+            await ems_ctx.started((pp_msgs, list(relay.accounts)))
 
             # establish 2-way stream with requesting order-client and
             # begin handling inbound order requests and updates
@@ -1015,13 +1106,11 @@ async def _emsd_main(
                 n.start_soon(
                     clear_dark_triggers,
 
-                    # relay.brokerd_dialogue,
                     brokerd_stream,
                     ems_client_order_stream,
-                    feed.stream,
-
+                    quote_stream,
                     broker,
-                    symbol,
+                    fqsn,  # form: <name>.<venue>.<suffix>.<broker>
                     book
                 )
 
@@ -1029,6 +1118,7 @@ async def _emsd_main(
                 try:
                     _router.clients.add(ems_client_order_stream)
 
+                    # main entrypoint, run here until cancelled.
                     await process_client_order_cmds(
 
                         ems_client_order_stream,
@@ -1036,7 +1126,7 @@ async def _emsd_main(
                         # relay.brokerd_dialogue,
                         brokerd_stream,
 
-                        symbol,
+                        fqsn,
                         feed,
                         dark_book,
                         _router,
@@ -1048,7 +1138,7 @@ async def _emsd_main(
 
                     dialogues = _router.dialogues
 
-                    for oid, client_stream in dialogues.items():
+                    for oid, client_stream in dialogues.copy().items():
 
                         if client_stream == ems_client_order_stream:
 

piker/clearing/_messages.py

@@ -24,6 +24,8 @@ from typing import Optional, Union
 # import msgspec
 from pydantic import BaseModel
 
+from ..data._source import Symbol
+
 # Client -> emsd
 
 
@@ -42,7 +44,8 @@ class Order(BaseModel):
     action: str  # {'buy', 'sell', 'alert'}
     # internal ``emdsd`` unique "order id"
     oid: str  # uuid4
-    symbol: str
+    symbol: Union[str, Symbol]
+    account: str  # should we set a default as '' ?
 
     price: float
     size: float
@@ -56,6 +59,13 @@ class Order(BaseModel):
     # the backend broker
     exec_mode: str  # {'dark', 'live', 'paper'}
 
+    class Config:
+        # just for pre-loading a ``Symbol`` when used
+        # in the order mode staging process
+        arbitrary_types_allowed = True
+        # don't copy this model instance when used in
+        # a recursive model
+        copy_on_model_validation = False
 
 # Client <- emsd
 # update msgs from ems which relay state change info
@@ -77,12 +87,11 @@ class Status(BaseModel):
     #   'broker_cancelled',
     #   'broker_executed',
     #   'broker_filled',
+    #   'broker_errored',
 
     #   'alert_submitted',
     #   'alert_triggered',
 
-    #   'position',
-
     # }
     resp: str  # "response", see above
 
@@ -111,6 +120,7 @@ class BrokerdCancel(BaseModel):
     oid: str  # piker emsd order id
     time_ns: int
 
+    account: str
     # "broker request id": broker specific/internal order id if this is
     # None, creates a new order otherwise if the id is valid the backend
     # api must modify the existing matching order. If the broker allows
@@ -124,6 +134,7 @@ class BrokerdOrder(BaseModel):
 
     action: str  # {buy, sell}
     oid: str
+    account: str
     time_ns: int
 
     # "broker request id": broker specific/internal order id if this is
@@ -144,8 +155,11 @@ class BrokerdOrder(BaseModel):
 
 
 class BrokerdOrderAck(BaseModel):
-    '''Immediate reponse to a brokerd order request providing
-    the broker specifci unique order id.
+    '''
+    Immediate reponse to a brokerd order request providing the broker
+    specific unique order id so that the EMS can associate this
+    (presumably differently formatted broker side ID) with our own
+    ``.oid`` (which is a uuid4).
 
     '''
     name: str = 'ack'
@@ -155,6 +169,7 @@ class BrokerdOrderAck(BaseModel):
 
     # emsd id originally sent in matching request msg
     oid: str
+    account: str = ''
 
 
 class BrokerdStatus(BaseModel):
@@ -163,10 +178,13 @@ class BrokerdStatus(BaseModel):
     reqid: Union[int, str]
     time_ns: int
 
+    # XXX: should be best effort set for every update
+    account: str = ''
+
     # {
     #   'submitted',
     #   'cancelled',
-    #   'executed',
+    #   'filled',
     # }
     status: str
 
@@ -188,7 +206,8 @@ class BrokerdStatus(BaseModel):
 
 
 class BrokerdFill(BaseModel):
-    '''A single message indicating a "fill-details" event from the broker
+    '''
+    A single message indicating a "fill-details" event from the broker
     if avaiable.
 
     '''
@@ -212,12 +231,18 @@ class BrokerdFill(BaseModel):
 
 
 class BrokerdError(BaseModel):
-    '''Optional error type that can be relayed to emsd for error handling.
+    '''
+    Optional error type that can be relayed to emsd for error handling.
 
     This is still a TODO thing since we're not sure how to employ it yet.
+
     '''
     name: str = 'error'
-    reqid: Union[int, str]
+    oid: str
+
+    # if no brokerd order request was actually submitted (eg. we errored
+    # at the ``pikerd`` layer) then there will be ``reqid`` allocated.
+    reqid: Optional[Union[int, str]] = None
 
     symbol: str
     reason: str

piker/clearing/_paper_engine.py

@@ -32,10 +32,11 @@ from dataclasses import dataclass
 
 from .. import data
 from ..data._normalize import iterticks
+from ..data._source import unpack_fqsn
 from ..log import get_logger
 from ._messages import (
     BrokerdCancel, BrokerdOrder, BrokerdOrderAck, BrokerdStatus,
-    BrokerdFill,
+    BrokerdFill, BrokerdPosition, BrokerdError
 )
 
 
@@ -60,6 +61,7 @@ class PaperBoi:
     _buys: bidict
     _sells: bidict
     _reqids: bidict
+    _positions: dict[str, BrokerdPosition]
 
     # init edge case L1 spread
     last_ask: Tuple[float, float] = (float('inf'), 0)  # price, size
@@ -101,6 +103,9 @@ class PaperBoi:
         # in the broker trades event processing loop
         await trio.sleep(0.05)
 
+        if action == 'sell':
+            size = -size
+
         msg = BrokerdStatus(
             status='submitted',
             reqid=reqid,
@@ -118,7 +123,7 @@ class PaperBoi:
             ) or (
             action == 'sell' and (clear_price := self.last_bid[0]) >= price
         ):
-            await self.fake_fill(clear_price, size, action, reqid, oid)
+            await self.fake_fill(symbol, clear_price, size, action, reqid, oid)
 
         else:
             # register this submissions as a paper live order
@@ -170,6 +175,8 @@ class PaperBoi:
 
     async def fake_fill(
         self,
+
+        symbol: str,
         price: float,
         size: float,
         action: str,  # one of {'buy', 'sell'}
@@ -181,6 +188,7 @@ class PaperBoi:
         # remaining lots to fill
         order_complete: bool = True,
         remaining: float = 0,
+
     ) -> None:
         """Pretend to fill a broker order @ price and size.
 
@@ -232,6 +240,49 @@ class PaperBoi:
             )
             await self.ems_trades_stream.send(msg.dict())
 
+        # lookup any existing position
+        token = f'{symbol}.{self.broker}'
+        pp_msg = self._positions.setdefault(
+            token,
+            BrokerdPosition(
+                broker=self.broker,
+                account='paper',
+                symbol=symbol,
+                # TODO: we need to look up the asset currency from
+                # broker info. i guess for crypto this can be
+                # inferred from the pair?
+                currency='',
+                size=0.0,
+                avg_price=0,
+            )
+        )
+
+        # "avg position price" calcs
+        # TODO: eventually it'd be nice to have a small set of routines
+        # to do this stuff from a sequence of cleared orders to enable
+        # so called "contextual positions".
+        new_size = size + pp_msg.size
+
+        # old size minus the new size gives us size differential with
+        # +ve -> increase in pp size
+        # -ve -> decrease in pp size
+        size_diff = abs(new_size) - abs(pp_msg.size)
+
+        if new_size == 0:
+            pp_msg.avg_price = 0
+
+        elif size_diff > 0:
+            # only update the "average position price" when the position
+            # size increases not when it decreases (i.e. the position is
+            # being made smaller)
+            pp_msg.avg_price = (
+                abs(size) * price + pp_msg.avg_price * abs(pp_msg.size)
+            ) / abs(new_size)
+
+        pp_msg.size = new_size
+
+        await self.ems_trades_stream.send(pp_msg.dict())
+
 
 async def simulate_fills(
     quote_stream: 'tractor.ReceiveStream',  # noqa
@@ -255,6 +306,7 @@ async def simulate_fills(
 
     # this stream may eventually contain multiple symbols
     async for quotes in quote_stream:
+
         for sym, quote in quotes.items():
 
             for tick in iterticks(
@@ -274,6 +326,7 @@ async def simulate_fills(
                     )
 
                     orders = client._buys.get(sym, {})
+
                     book_sequence = reversed(
                         sorted(orders.keys(), key=itemgetter(1)))
 
@@ -307,6 +360,7 @@ async def simulate_fills(
 
                         # clearing price would have filled entirely
                         await client.fake_fill(
+                            symbol=sym,
                             # todo slippage to determine fill price
                             price=tick_price,
                             size=size,
@@ -332,6 +386,19 @@ async def handle_order_requests(
         action = request_msg['action']
 
         if action in {'buy', 'sell'}:
+
+            account = request_msg['account']
+            if account != 'paper':
+                log.error(
+                    'This is a paper account, only a `paper` selection is valid'
+                )
+                await ems_order_stream.send(BrokerdError(
+                    oid=request_msg['oid'],
+                    symbol=request_msg['symbol'],
+                    reason=f'Paper only. No account found: `{account}` ?',
+                ).dict())
+                continue
+
             # validate
             order = BrokerdOrder(**request_msg)
 
@@ -380,16 +447,16 @@ async def trades_dialogue(
 
     ctx: tractor.Context,
     broker: str,
-    symbol: str,
+    fqsn: str,
     loglevel: str = None,
 
 ) -> None:
+    tractor.log.get_console_log(loglevel)
 
     async with (
 
         data.open_feed(
-            broker,
-            [symbol],
+            [fqsn],
             loglevel=loglevel,
         ) as feed,
 
@@ -397,7 +464,7 @@ async def trades_dialogue(
         # TODO: load paper positions per broker from .toml config file
         # and pass as symbol to position data mapping: ``dict[str, dict]``
         # await ctx.started(all_positions)
-        await ctx.started({})
+        await ctx.started(({}, {'paper',}))
 
         async with (
             ctx.open_stream() as ems_stream,
@@ -411,6 +478,9 @@ async def trades_dialogue(
                 _sells={},
 
                 _reqids={},
+
+                # TODO: load paper positions from ``positions.toml``
+                _positions={},
             )
 
             n.start_soon(handle_order_requests, client, ems_stream)
@@ -421,15 +491,16 @@ async def trades_dialogue(
 
 @asynccontextmanager
 async def open_paperboi(
-    broker: str,
-    symbol: str,
+    fqsn: str,
     loglevel: str,
 
 ) -> Callable:
-    '''Spawn a paper engine actor and yield through access to
+    '''
+    Spawn a paper engine actor and yield through access to
     its context.
 
     '''
+    broker, symbol, expiry = unpack_fqsn(fqsn)
     service_name = f'paperboi.{broker}'
 
     async with (
@@ -448,14 +519,9 @@ async def open_paperboi(
         async with portal.open_context(
                 trades_dialogue,
                 broker=broker,
-                symbol=symbol,
+                fqsn=fqsn,
                 loglevel=loglevel,
 
         ) as (ctx, first):
-            try:
-                yield ctx, first
 
-            finally:
-                # be sure to tear down the paper service on exit
-                with trio.CancelScope(shield=True):
-                    await portal.cancel_actor()
+            yield ctx, first

piker/cli/__init__.py

@@ -1,43 +1,55 @@
-"""
+# piker: trading gear for hackers
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
 CLI commons.
-"""
+
+'''
 import os
+from pprint import pformat
 
 import click
 import trio
 import tractor
 
 from ..log import get_console_log, get_logger, colorize_json
-from ..brokers import get_brokermod, config
+from ..brokers import get_brokermod
 from .._daemon import _tractor_kwargs
+from .. import config
 
 
 log = get_logger('cli')
 DEFAULT_BROKER = 'questrade'
 
-_config_dir = click.get_app_dir('piker')
-_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
-_context_defaults = dict(
-    default_map={
-        # Questrade specific quote poll rates
-        'monitor': {
-            'rate': 3,
-        },
-        'optschain': {
-            'rate': 1,
-        },
-    }
-)
-
 
 @click.command()
 @click.option('--loglevel', '-l', default='warning', help='Logging level')
 @click.option('--tl', is_flag=True, help='Enable tractor logging')
 @click.option('--pdb', is_flag=True, help='Enable tractor debug mode')
 @click.option('--host', '-h', default='127.0.0.1', help='Host address to bind')
-def pikerd(loglevel, host, tl, pdb):
-    """Spawn the piker broker-daemon.
-    """
+@click.option(
+    '--tsdb',
+    is_flag=True,
+    help='Enable local ``marketstore`` instance'
+)
+def pikerd(loglevel, host, tl, pdb, tsdb):
+    '''
+    Spawn the piker broker-daemon.
+
+    '''
     from .._daemon import open_pikerd
     log = get_console_log(loglevel)
 
@@ -51,13 +63,38 @@ def pikerd(loglevel, host, tl, pdb):
         ))
 
     async def main():
-        async with open_pikerd(loglevel=loglevel, debug_mode=pdb):
+
+        async with (
+            open_pikerd(
+                loglevel=loglevel,
+                debug_mode=pdb,
+            ),  # normally delivers a ``Services`` handle
+            trio.open_nursery() as n,
+        ):
+            if tsdb:
+                from piker.data._ahab import start_ahab
+                from piker.data.marketstore import start_marketstore
+
+                log.info('Spawning `marketstore` supervisor')
+                ctn_ready, config, (cid, pid) = await n.start(
+                    start_ahab,
+                    'marketstored',
+                    start_marketstore,
+
+                )
+                log.info(
+                    f'`marketstore` up!\n'
+                    f'`marketstored` pid: {pid}\n'
+                    f'docker container id: {cid}\n'
+                    f'config: {pformat(config)}'
+                )
+
             await trio.sleep_forever()
 
     trio.run(main)
 
 
-@click.group(context_settings=_context_defaults)
+@click.group(context_settings=config._context_defaults)
 @click.option(
     '--brokers', '-b',
     default=[DEFAULT_BROKER],
@@ -86,8 +123,8 @@ def cli(ctx, brokers, loglevel, tl, configdir):
         'loglevel': loglevel,
         'tractorloglevel': None,
         'log': get_console_log(loglevel),
-        'confdir': _config_dir,
-        'wl_path': _watchlists_data_path,
+        'confdir': config._config_dir,
+        'wl_path': config._watchlists_data_path,
     })
 
     # allow enabling same loglevel in ``tractor`` machinery
@@ -106,15 +143,13 @@ def services(config, tl, names):
         async with tractor.get_arbiter(
             *_tractor_kwargs['arbiter_addr']
         ) as portal:
-            registry = await portal.run('self', 'get_registry')
+            registry = await portal.run_from_ns('self', 'get_registry')
             json_d = {}
-            for uid, socket in registry.items():
-                name, uuid = uid
+            for key, socket in registry.items():
+                # name, uuid = uid
                 host, port = socket
-                json_d[f'{name}.{uuid}'] = f'{host}:{port}'
-            click.echo(
-                f"Available `piker` services:\n{colorize_json(json_d)}"
-            )
+                json_d[key] = f'{host}:{port}'
+            click.echo(f"{colorize_json(json_d)}")
 
     tractor.run(
         list_services,

piker/config.py

@@ -0,0 +1,267 @@
+# piker: trading gear for hackers
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+Broker configuration mgmt.
+
+"""
+import platform
+import sys
+import os
+from os.path import dirname
+import shutil
+from typing import Optional
+
+from bidict import bidict
+import toml
+
+from .log import get_logger
+
+log = get_logger('broker-config')
+
+
+# taken from ``click`` since apparently they have some
+# super weirdness with sigint and sudo..no clue
+def get_app_dir(app_name, roaming=True, force_posix=False):
+    r"""Returns the config folder for the application.  The default behavior
+    is to return whatever is most appropriate for the operating system.
+
+    To give you an idea, for an app called ``"Foo Bar"``, something like
+    the following folders could be returned:
+
+    Mac OS X:
+      ``~/Library/Application Support/Foo Bar``
+    Mac OS X (POSIX):
+      ``~/.foo-bar``
+    Unix:
+      ``~/.config/foo-bar``
+    Unix (POSIX):
+      ``~/.foo-bar``
+    Win XP (roaming):
+      ``C:\Documents and Settings\<user>\Local Settings\Application Data\Foo``
+    Win XP (not roaming):
+      ``C:\Documents and Settings\<user>\Application Data\Foo Bar``
+    Win 7 (roaming):
+      ``C:\Users\<user>\AppData\Roaming\Foo Bar``
+    Win 7 (not roaming):
+      ``C:\Users\<user>\AppData\Local\Foo Bar``
+
+    .. versionadded:: 2.0
+
+    :param app_name: the application name.  This should be properly capitalized
+                     and can contain whitespace.
+    :param roaming: controls if the folder should be roaming or not on Windows.
+                    Has no affect otherwise.
+    :param force_posix: if this is set to `True` then on any POSIX system the
+                        folder will be stored in the home folder with a leading
+                        dot instead of the XDG config home or darwin's
+                        application support folder.
+    """
+
+    def _posixify(name):
+        return "-".join(name.split()).lower()
+
+    # if WIN:
+    if platform.system() == 'Windows':
+        key = "APPDATA" if roaming else "LOCALAPPDATA"
+        folder = os.environ.get(key)
+        if folder is None:
+            folder = os.path.expanduser("~")
+        return os.path.join(folder, app_name)
+    if force_posix:
+        return os.path.join(
+            os.path.expanduser("~/.{}".format(_posixify(app_name))))
+    if sys.platform == "darwin":
+        return os.path.join(
+            os.path.expanduser("~/Library/Application Support"), app_name
+        )
+    return os.path.join(
+        os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")),
+        _posixify(app_name),
+    )
+
+
+_config_dir = _click_config_dir = get_app_dir('piker')
+_parent_user = os.environ.get('SUDO_USER')
+
+if _parent_user:
+    non_root_user_dir = os.path.expanduser(
+        f'~{_parent_user}'
+    )
+    root = 'root'
+    _config_dir = (
+        non_root_user_dir +
+        _click_config_dir[
+            _click_config_dir.rfind(root) + len(root):
+        ]
+    )
+
+_conf_names: set[str] = {
+    'brokers',
+    'trades',
+    'watchlists',
+}
+
+_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
+_context_defaults = dict(
+    default_map={
+        # Questrade specific quote poll rates
+        'monitor': {
+            'rate': 3,
+        },
+        'optschain': {
+            'rate': 1,
+        },
+    }
+)
+
+
+def _override_config_dir(
+    path: str
+) -> None:
+    global _config_dir
+    _config_dir = path
+
+
+def _conf_fn_w_ext(
+    name: str,
+) -> str:
+    # change this if we ever change the config file format.
+    return f'{name}.toml'
+
+
+def get_conf_path(
+    conf_name: str = 'brokers',
+
+) -> str:
+    """Return the default config path normally under
+    ``~/.config/piker`` on linux.
+
+    Contains files such as:
+    - brokers.toml
+    - watchlists.toml
+    - trades.toml
+
+    # maybe coming soon ;)
+    - signals.toml
+    - strats.toml
+
+    """
+    assert conf_name in _conf_names
+    fn = _conf_fn_w_ext(conf_name)
+    return os.path.join(
+        _config_dir,
+        fn,
+    )
+
+
+def repodir():
+    '''
+    Return the abspath to the repo directory.
+
+    '''
+    dirpath = os.path.abspath(
+        # we're 3 levels down in **this** module file
+        dirname(dirname(os.path.realpath(__file__)))
+    )
+    return dirpath
+
+
+def load(
+    conf_name: str = 'brokers',
+    path: str = None
+
+) -> (dict, str):
+    '''
+    Load config file by name.
+
+    '''
+    path = path or get_conf_path(conf_name)
+    if not os.path.isfile(path):
+        fn = _conf_fn_w_ext(conf_name)
+
+        template = os.path.join(
+            repodir(),
+            'config',
+            fn
+        )
+        # try to copy in a template config to the user's directory
+        # if one exists.
+        if os.path.isfile(template):
+            shutil.copyfile(template, path)
+
+    config = toml.load(path)
+    log.debug(f"Read config file {path}")
+    return config, path
+
+
+def write(
+    config: dict,  # toml config as dict
+    name: str = 'brokers',
+    path: str = None,
+
+) -> None:
+    ''''
+    Write broker config to disk.
+
+    Create a ``brokers.ini`` file if one does not exist.
+
+    '''
+    path = path or get_conf_path(name)
+    dirname = os.path.dirname(path)
+    if not os.path.isdir(dirname):
+        log.debug(f"Creating config dir {_config_dir}")
+        os.makedirs(dirname)
+
+    if not config:
+        raise ValueError(
+            "Watch out you're trying to write a blank config!")
+
+    log.debug(
+        f"Writing config `{name}` file to:\n"
+        f"{path}"
+    )
+    with open(path, 'w') as cf:
+        return toml.dump(config, cf)
+
+
+def load_accounts(
+
+    providers: Optional[list[str]] = None
+
+) -> bidict[str, Optional[str]]:
+
+    conf, path = load()
+    accounts = bidict()
+    for provider_name, section in conf.items():
+        accounts_section = section.get('accounts')
+        if (
+            providers is None or
+            providers and provider_name in providers
+        ):
+            if accounts_section is None:
+                log.warning(f'No accounts named for {provider_name}?')
+                continue
+            else:
+                for label, value in accounts_section.items():
+                    accounts[
+                        f'{provider_name}.{label}'
+                    ] = value
+
+    # our default paper engine entry
+    accounts['paper'] = None
+
+    return accounts

piker/data/_ahab.py

@@ -0,0 +1,385 @@
+# piker: trading gear for hackers
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Supervisor for docker with included specific-image service helpers.
+
+'''
+import os
+import time
+from typing import (
+    Optional,
+    Callable,
+    Any,
+)
+from contextlib import asynccontextmanager as acm
+
+import trio
+from trio_typing import TaskStatus
+import tractor
+from tractor.msg import NamespacePath
+import docker
+import json
+from docker.models.containers import Container as DockerContainer
+from docker.errors import (
+    DockerException,
+    APIError,
+)
+from requests.exceptions import ConnectionError, ReadTimeout
+
+from ..log import get_logger, get_console_log
+from .. import config
+
+log = get_logger(__name__)
+
+
+class DockerNotStarted(Exception):
+    'Prolly you dint start da daemon bruh'
+
+
+class ContainerError(RuntimeError):
+    'Error reported via app-container logging level'
+
+
+@acm
+async def open_docker(
+    url: Optional[str] = None,
+    **kwargs,
+
+) -> docker.DockerClient:
+
+    client: Optional[docker.DockerClient] = None
+    try:
+        client = docker.DockerClient(
+            base_url=url,
+            **kwargs
+        ) if url else docker.from_env(**kwargs)
+
+        yield client
+
+    except (
+        DockerException,
+        APIError,
+    ) as err:
+
+        def unpack_msg(err: Exception) -> str:
+            args = getattr(err, 'args', None)
+            if args:
+                return args
+            else:
+                return str(err)
+
+        # could be more specific so let's check if it's just perms.
+        if err.args:
+            errs = err.args
+            for err in errs:
+                msg = unpack_msg(err)
+                if 'PermissionError' in msg:
+                    raise DockerException('You dint run as root yo!')
+
+                elif 'FileNotFoundError' in msg:
+                    raise DockerNotStarted('Did you start da service sister?')
+
+        # not perms?
+        raise
+
+    finally:
+        if client:
+            client.close()
+
+
+class Container:
+    '''
+    Wrapper around a ``docker.models.containers.Container`` to include
+    log capture and relay through our native logging system and helper
+    method(s) for cancellation/teardown.
+
+    '''
+    def __init__(
+        self,
+        cntr: DockerContainer,
+    ) -> None:
+
+        self.cntr = cntr
+        # log msg de-duplication
+        self.seen_so_far = set()
+
+    async def process_logs_until(
+        self,
+        patt: str,
+        bp_on_msg: bool = False,
+    ) -> bool:
+        '''
+        Attempt to capture container log messages and relay through our
+        native logging system.
+
+        '''
+        seen_so_far = self.seen_so_far
+
+        while True:
+            logs = self.cntr.logs()
+            entries = logs.decode().split('\n')
+            for entry in entries:
+
+                # ignore null lines
+                if not entry:
+                    continue
+
+                try:
+                    record = json.loads(entry.strip())
+                except json.JSONDecodeError:
+                    if 'Error' in entry:
+                        raise RuntimeError(entry)
+                    raise
+
+                msg = record['msg']
+                level = record['level']
+                if msg and entry not in seen_so_far:
+                    seen_so_far.add(entry)
+                    if bp_on_msg:
+                        await tractor.breakpoint()
+
+                    getattr(log, level, log.error)(f'{msg}')
+
+                    # print(f'level: {level}')
+                    if level in ('error', 'fatal'):
+                        raise ContainerError(msg)
+
+                if patt in msg:
+                    return True
+
+                # do a checkpoint so we don't block if cancelled B)
+                await trio.sleep(0.01)
+
+        return False
+
+    def try_signal(
+        self,
+        signal: str = 'SIGINT',
+
+    ) -> bool:
+        try:
+            # XXX: market store doesn't seem to shutdown nicely all the
+            # time with this (maybe because there are still open grpc
+            # connections?) noticably after client connections have been
+            # made or are in use/teardown. It works just fine if you
+            # just start and stop the container tho?..
+            log.cancel(f'SENDING {signal} to {self.cntr.id}')
+            self.cntr.kill(signal)
+            return True
+
+        except docker.errors.APIError as err:
+            if 'is not running' in err.explanation:
+                return False
+
+    async def cancel(
+        self,
+        stop_msg: str,
+    ) -> None:
+
+        cid = self.cntr.id
+        # first try a graceful cancel
+        log.cancel(
+            f'SIGINT cancelling container: {cid}\n'
+            f'waiting on stop msg: "{stop_msg}"'
+        )
+        self.try_signal('SIGINT')
+
+        start = time.time()
+        for _ in range(30):
+
+            with trio.move_on_after(0.5) as cs:
+                cs.shield = True
+                await self.process_logs_until(stop_msg)
+
+                # if we aren't cancelled on above checkpoint then we
+                # assume we read the expected stop msg and terminated.
+                break
+
+            try:
+                log.info(f'Polling for container shutdown:\n{cid}')
+
+                if self.cntr.status not in {'exited', 'not-running'}:
+                    self.cntr.wait(
+                        timeout=0.1,
+                        condition='not-running',
+                    )
+
+                break
+
+            except (
+                ReadTimeout,
+            ):
+                log.info(f'Still waiting on container:\n{cid}')
+                continue
+
+            except (
+                docker.errors.APIError,
+                ConnectionError,
+            ):
+                log.exception('Docker connection failure')
+                break
+        else:
+            delay = time.time() - start
+            log.error(
+                f'Failed to kill container {cid} after {delay}s\n'
+                'sending SIGKILL..'
+            )
+            # get out the big guns, bc apparently marketstore
+            # doesn't actually know how to terminate gracefully
+            # :eyeroll:...
+            self.try_signal('SIGKILL')
+            self.cntr.wait(
+                timeout=3,
+                condition='not-running',
+            )
+
+        log.cancel(f'Container stopped: {cid}')
+
+
+@tractor.context
+async def open_ahabd(
+    ctx: tractor.Context,
+    endpoint: str,  # ns-pointer str-msg-type
+
+    **kwargs,
+
+) -> None:
+    get_console_log('info', name=__name__)
+
+    async with open_docker() as client:
+
+        # TODO: eventually offer a config-oriented API to do the mounts,
+        # params, etc. passing to ``Containter.run()``?
+        # call into endpoint for container config/init
+        ep_func = NamespacePath(endpoint).load_ref()
+        (
+            dcntr,
+            cntr_config,
+            start_msg,
+            stop_msg,
+        ) = ep_func(client)
+        cntr = Container(dcntr)
+
+        with trio.move_on_after(1):
+            found = await cntr.process_logs_until(start_msg)
+
+            if not found and cntr not in client.containers.list():
+                raise RuntimeError(
+                    'Failed to start `marketstore` check logs deats'
+                )
+
+        await ctx.started((
+            cntr.cntr.id,
+            os.getpid(),
+            cntr_config,
+        ))
+
+        try:
+
+            # TODO: we might eventually want a proxy-style msg-prot here
+            # to allow remote control of containers without needing
+            # callers to have root perms?
+            await trio.sleep_forever()
+
+        finally:
+            with trio.CancelScope(shield=True):
+                await cntr.cancel(stop_msg)
+
+
+async def start_ahab(
+    service_name: str,
+    endpoint: Callable[docker.DockerClient, DockerContainer],
+    task_status: TaskStatus[
+        tuple[
+            trio.Event,
+            dict[str, Any],
+        ],
+    ] = trio.TASK_STATUS_IGNORED,
+
+) -> None:
+    '''
+    Start a ``docker`` container supervisor with given service name.
+
+    Currently the actor calling this task should normally be started
+    with root permissions (until we decide to use something that doesn't
+    require this, like docker's rootless mode or some wrapper project) but
+    te root perms are de-escalated after the docker supervisor sub-actor
+    is started.
+
+    '''
+    cn_ready = trio.Event()
+    try:
+        async with tractor.open_nursery(
+            loglevel='runtime',
+        ) as tn:
+
+            portal = await tn.start_actor(
+                service_name,
+                enable_modules=[__name__]
+            )
+
+            # TODO: we have issues with this on teardown
+            # where ``tractor`` tries to issue ``os.kill()``
+            # and hits perms errors since the root process
+            # doesn't any longer have root perms..
+
+            # de-escalate root perms to the original user
+            # after the docker supervisor actor is spawned.
+            if config._parent_user:
+                import pwd
+                os.setuid(
+                    pwd.getpwnam(
+                        config._parent_user
+                    )[2]  # named user's uid
+                )
+
+            async with portal.open_context(
+                open_ahabd,
+                endpoint=str(NamespacePath.from_ref(endpoint)),
+            ) as (ctx, first):
+
+                cid, pid, cntr_config = first
+
+                task_status.started((
+                    cn_ready,
+                    cntr_config,
+                    (cid, pid),
+                ))
+
+                await trio.sleep_forever()
+
+    # since we demoted root perms in this parent
+    # we'll get a perms error on proc cleanup in
+    # ``tractor`` nursery exit. just make sure
+    # the child is terminated and don't raise the
+    # error if so.
+
+    # TODO: we could also consider adding
+    # a ``tractor.ZombieDetected`` or something that we could raise
+    # if we find the child didn't terminate.
+    except PermissionError:
+        log.warning('Failed to cancel root permsed container')
+
+    except (
+        trio.MultiError,
+    ) as err:
+        for subexc in err.exceptions:
+            if isinstance(subexc, PermissionError):
+                log.warning('Failed to cancel root perms-ed container')
+                return
+        else:
+            raise

piker/data/_normalize.py

@@ -14,25 +14,69 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-"""
+'''
 Stream format enforcement.
-"""
 
-from typing import AsyncIterator, Optional, Tuple
-
-import numpy as np
+'''
+from itertools import chain
+from typing import AsyncIterator
 
 
 def iterticks(
     quote: dict,
-    types: Tuple[str] = ('trade', 'utrade'),
+    types: tuple[str] = (
+        'trade',
+        'dark_trade',
+    ),
+    deduplicate_darks: bool = False,
+
 ) -> AsyncIterator:
-    """Iterate through ticks delivered per quote cycle.
-    """
+    '''
+    Iterate through ticks delivered per quote cycle.
+
+    '''
+    if deduplicate_darks:
+        assert 'dark_trade' in types
+
     # print(f"{quote}\n\n")
     ticks = quote.get('ticks', ())
+    trades = {}
+    darks = {}
+
     if ticks:
+
+        # do a first pass and attempt to remove duplicate dark
+        # trades with the same tick signature.
+        if deduplicate_darks:
+            for tick in ticks:
+                ttype = tick.get('type')
+
+                time = tick.get('time', None)
+                if time:
+                    sig = (
+                        time,
+                        tick['price'],
+                        tick['size']
+                    )
+
+                    if ttype == 'dark_trade':
+                        darks[sig] = tick
+
+                    elif ttype == 'trade':
+                        trades[sig] = tick
+
+            # filter duplicates
+            for sig, tick in trades.items():
+                tick = darks.pop(sig, None)
+                if tick:
+                    ticks.remove(tick)
+                    # print(f'DUPLICATE {tick}')
+
+            # re-insert ticks
+            ticks.extend(list(chain(trades.values(), darks.values())))
+
         for tick in ticks:
             # print(f"{quote['symbol']}: {tick}")
-            if tick.get('type') in types:
+            ttype = tick.get('type')
+            if ttype in types:
                 yield tick

piker/data/_sampling.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of pikers)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,40 +15,57 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 """
-Data buffers for fast shared humpy.
+Sampling and broadcast machinery for (soft) real-time delivery of
+financial data flows.
+
 """
+from __future__ import annotations
+from collections import Counter
 import time
-from typing import Dict, List
+from typing import TYPE_CHECKING, Optional, Union
 
 import tractor
 import trio
 from trio_typing import TaskStatus
 
-from ._sharedmem import ShmArray
 from ..log import get_logger
 
+if TYPE_CHECKING:
+    from ._sharedmem import ShmArray
+    from .feed import _FeedsBus
 
 log = get_logger(__name__)
 
 
-# TODO: we could stick these in a composed type to avoid
-# angering the "i hate module scoped variables crowd" (yawn).
-_shms: Dict[int, List[ShmArray]] = {}
-_start_increment: Dict[str, trio.Event] = {}
-_incrementers: Dict[int, trio.CancelScope] = {}
-_subscribers: Dict[str, tractor.Context] = {}
+class sampler:
+    '''
+    Global sampling engine registry.
 
+    Manages state for sampling events, shm incrementing and
+    sample period logic.
 
-def shm_incrementing(shm_token_name: str) -> trio.Event:
-    global _start_increment
-    return _start_increment.setdefault(shm_token_name, trio.Event())
+    '''
+    # TODO: we could stick these in a composed type to avoid
+    # angering the "i hate module scoped variables crowd" (yawn).
+    ohlcv_shms: dict[int, list[ShmArray]] = {}
+
+    # holds one-task-per-sample-period tasks which are spawned as-needed by
+    # data feed requests with a given detected time step usually from
+    # history loading.
+    incrementers: dict[int, trio.CancelScope] = {}
+
+    # holds all the ``tractor.Context`` remote subscriptions for
+    # a particular sample period increment event: all subscribers are
+    # notified on a step.
+    subscribers: dict[int, tractor.Context] = {}
 
 
 async def increment_ohlc_buffer(
     delay_s: int,
     task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED,
 ):
-    """Task which inserts new bars into the provide shared memory array
+    '''
+    Task which inserts new bars into the provide shared memory array
     every ``delay_s`` seconds.
 
     This task fulfills 2 purposes:
@@ -59,8 +76,8 @@ async def increment_ohlc_buffer(
 
     Note that if **no** actor has initiated this task then **none** of
     the underlying buffers will actually be incremented.
-    """
 
+    '''
     # # wait for brokerd to signal we should start sampling
     # await shm_incrementing(shm_token['shm_name']).wait()
 
@@ -69,19 +86,18 @@ async def increment_ohlc_buffer(
     # to solve this is to make this task aware of the instrument's
     # tradable hours?
 
-    global _incrementers
-
     # adjust delay to compensate for trio processing time
-    ad = min(_shms.keys()) - 0.001
+    ad = min(sampler.ohlcv_shms.keys()) - 0.001
 
     total_s = 0  # total seconds counted
-    lowest = min(_shms.keys())
+    lowest = min(sampler.ohlcv_shms.keys())
+    lowest_shm = sampler.ohlcv_shms[lowest][0]
     ad = lowest - 0.001
 
     with trio.CancelScope() as cs:
 
         # register this time period step as active
-        _incrementers[delay_s] = cs
+        sampler.incrementers[delay_s] = cs
         task_status.started(cs)
 
         while True:
@@ -91,8 +107,10 @@ async def increment_ohlc_buffer(
             total_s += lowest
 
             # increment all subscribed shm arrays
-            # TODO: this in ``numba``
-            for delay_s, shms in _shms.items():
+            # TODO:
+            # - this in ``numba``
+            # - just lookup shms for this step instead of iterating?
+            for delay_s, shms in sampler.ohlcv_shms.items():
                 if total_s % delay_s != 0:
                     continue
 
@@ -117,69 +135,121 @@ async def increment_ohlc_buffer(
                     # write to the buffer
                     shm.push(last)
 
-                # broadcast the buffer index step
-                # yield {'index': shm._last.value}
-                for ctx in _subscribers.get(delay_s, ()):
-                    try:
-                        await ctx.send_yield({'index': shm._last.value})
-                    except (
-                        trio.BrokenResourceError,
-                        trio.ClosedResourceError
-                    ):
-                        log.error(f'{ctx.chan.uid} dropped connection')
+            await broadcast(delay_s, shm=lowest_shm)
 
 
-@tractor.stream
+async def broadcast(
+    delay_s: int,
+    shm: Optional[ShmArray] = None,
+
+) -> None:
+    '''
+    Broadcast the given ``shm: ShmArray``'s buffer index step to any
+    subscribers for a given sample period.
+
+    The sent msg will include the first and last index which slice into
+    the buffer's non-empty data.
+
+    '''
+    subs = sampler.subscribers.get(delay_s, ())
+
+    first = last = -1
+
+    if shm is None:
+        periods = sampler.ohlcv_shms.keys()
+        # if this is an update triggered by a history update there
+        # might not actually be any sampling bus setup since there's
+        # no "live feed" active yet.
+        if periods:
+            lowest = min(periods)
+            shm = sampler.ohlcv_shms[lowest][0]
+            first = shm._first.value
+            last = shm._last.value
+
+    for stream in subs:
+        try:
+            await stream.send({
+                'first': first,
+                'last': last,
+                'index': last,
+            })
+        except (
+            trio.BrokenResourceError,
+            trio.ClosedResourceError
+        ):
+            log.error(
+                f'{stream._ctx.chan.uid} dropped connection'
+            )
+            try:
+                subs.remove(stream)
+            except ValueError:
+                log.warning(
+                    f'{stream._ctx.chan.uid} sub already removed!?'
+                )
+
+
+@tractor.context
 async def iter_ohlc_periods(
     ctx: tractor.Context,
     delay_s: int,
+
 ) -> None:
-    """
+    '''
     Subscribe to OHLC sampling "step" events: when the time
     aggregation period increments, this event stream emits an index
     event.
 
-    """
+    '''
     # add our subscription
-    global _subscribers
-    subs = _subscribers.setdefault(delay_s, [])
-    subs.append(ctx)
+    subs = sampler.subscribers.setdefault(delay_s, [])
+    await ctx.started()
+    async with ctx.open_stream() as stream:
+        subs.append(stream)
 
-    try:
-        # stream and block until cancelled
-        await trio.sleep_forever()
-    finally:
-        subs.remove(ctx)
+        try:
+            # stream and block until cancelled
+            await trio.sleep_forever()
+        finally:
+            try:
+                subs.remove(stream)
+            except ValueError:
+                log.error(
+                    f'iOHLC step stream was already dropped {ctx.chan.uid}?'
+                )
 
 
 async def sample_and_broadcast(
 
-    bus: '_FeedBus',  # noqa
+    bus: _FeedsBus,  # noqa
     shm: ShmArray,
     quote_stream: trio.abc.ReceiveChannel,
+    brokername: str,
     sum_tick_vlm: bool = True,
 
 ) -> None:
 
     log.info("Started shared mem bar writer")
 
+    overruns = Counter()
+
     # iterate stream delivered by broker
     async for quotes in quote_stream:
-
         # TODO: ``numba`` this!
-        for sym, quote in quotes.items():
-
-            # TODO: in theory you can send the IPC msg *before*
-            # writing to the sharedmem array to decrease latency,
-            # however, that will require `tractor.msg.pub` support
-            # here or at least some way to prevent task switching
-            # at the yield such that the array write isn't delayed
-            # while another consumer is serviced..
+        for broker_symbol, quote in quotes.items():
+            # TODO: in theory you can send the IPC msg *before* writing
+            # to the sharedmem array to decrease latency, however, that
+            # will require at least some way to prevent task switching
+            # at the yield such that the array write isn't delayed while
+            # another consumer is serviced..
 
             # start writing the shm buffer with appropriate
             # trade data
-            for tick in quote['ticks']:
 
+            # TODO: we should probably not write every single
+            # value to an OHLC sample stream XD
+            # for a tick stream sure.. but this is excessive..
+            ticks = quote['ticks']
+            for tick in ticks:
                 ticktype = tick['type']
 
                 # write trade events to shm last OHLC sample
@@ -229,86 +299,221 @@ async def sample_and_broadcast(
             # end up triggering backpressure which which will
             # eventually block this producer end of the feed and
             # thus other consumers still attached.
-            subs = bus._subscribers[sym.lower()]
+            subs: list[
+                tuple[
+                    Union[tractor.MsgStream, trio.MemorySendChannel],
+                    tractor.Context,
+                    Optional[float],  # tick throttle in Hz
+                ]
+            ] = bus._subscribers[broker_symbol.lower()]
 
-            for (stream, tick_throttle) in subs:
+            # NOTE: by default the broker backend doesn't append
+            # it's own "name" into the fqsn schema (but maybe it
+            # should?) so we have to manually generate the correct
+            # key here.
+            bsym = f'{broker_symbol}.{brokername}'
+            lags: int = 0
+
+            for (stream, ctx, tick_throttle) in subs:
 
                 try:
-                    if tick_throttle:
-                        await stream.send(quote)
+                    with trio.move_on_after(0.2) as cs:
+                        if tick_throttle:
+                            # this is a send mem chan that likely
+                            # pushes to the ``uniform_rate_send()`` below.
+                            try:
+                                stream.send_nowait(
+                                    (bsym, quote)
+                                )
+                            except trio.WouldBlock:
+                                chan = ctx.chan
+                                if ctx:
+                                    log.warning(
+                                        f'Feed overrun {bus.brokername} ->'
+                                        f'{chan.uid} !!!'
+                                    )
+                                else:
+                                    key = id(stream)
+                                    overruns[key] += 1
+                                    log.warning(
+                                        f'Feed overrun {broker_symbol}'
+                                        '@{bus.brokername} -> '
+                                        f'feed @ {tick_throttle} Hz'
+                                    )
+                                    if overruns[key] > 6:
+                                        # TODO: should we check for the
+                                        # context being cancelled? this
+                                        # could happen but the
+                                        # channel-ipc-pipe is still up.
+                                        if not chan.connected():
+                                            log.warning(
+                                                'Dropping broken consumer:\n'
+                                                f'{broker_symbol}:'
+                                                f'{ctx.cid}@{chan.uid}'
+                                            )
+                                            await stream.aclose()
+                                            raise trio.BrokenResourceError
+                                        else:
+                                            log.warning(
+                                                'Feed getting overrun bro!\n'
+                                                f'{broker_symbol}:'
+                                                f'{ctx.cid}@{chan.uid}'
+                                            )
+                                            continue
 
-                    else:
-                        await stream.send({sym: quote})
+                        else:
+                            await stream.send(
+                                {bsym: quote}
+                            )
+
+                    if cs.cancelled_caught:
+                        lags += 1
+                        if lags > 10:
+                            await tractor.breakpoint()
 
                 except (
                     trio.BrokenResourceError,
-                    trio.ClosedResourceError
+                    trio.ClosedResourceError,
+                    trio.EndOfChannel,
                 ):
+                    chan = ctx.chan
+                    if ctx:
+                        log.warning(
+                            'Dropped `brokerd`-quotes-feed connection:\n'
+                            f'{broker_symbol}:'
+                            f'{ctx.cid}@{chan.uid}'
+                        )
+                    if tick_throttle:
+                        assert stream._closed
+
                     # XXX: do we need to deregister here
                     # if it's done in the fee bus code?
                     # so far seems like no since this should all
-                    # be single-threaded.
-                    log.warning(
-                        f'{stream._ctx.chan.uid} dropped  '
-                        '`brokerd`-quotes-feed connection'
-                    )
-                    subs.remove((stream, tick_throttle))
+                    # be single-threaded. Doing it anyway though
+                    # since there seems to be some kinda race..
+                    try:
+                        subs.remove((stream, tick_throttle))
+                    except ValueError:
+                        log.error(
+                            f'Stream was already removed from subs!?\n'
+                            f'{broker_symbol}:'
+                            f'{ctx.cid}@{chan.uid}'
+                        )
 
 
+# TODO: a less naive throttler, here's some snippets:
+# token bucket by njs:
+# https://gist.github.com/njsmith/7ea44ec07e901cb78ebe1dd8dd846cb9
+
 async def uniform_rate_send(
 
     rate: float,
     quote_stream: trio.abc.ReceiveChannel,
     stream: tractor.MsgStream,
 
+    task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
+
 ) -> None:
 
-    sleep_period = 1/rate - 0.000616
+    # TODO: compute the approx overhead latency per cycle
+    left_to_sleep = throttle_period = 1/rate - 0.000616
+
+    # send cycle state
+    first_quote = last_quote = None
     last_send = time.time()
+    diff = 0
+
+    task_status.started()
 
     while True:
 
-        first_quote = await quote_stream.receive()
-        start = time.time()
+        # compute the remaining time to sleep for this throttled cycle
+        left_to_sleep = throttle_period - diff
 
-        # append quotes since last iteration into the last quote's
-        # tick array/buffer.
-
-        # TODO: once we decide to get fancy really we should have
-        # a shared mem tick buffer that is just continually filled and
-        # the UI just ready from it at it's display rate.
-        # we'll likely head toward this once we get this issue going:
-        #
-        while True:
-            try:
-                next_quote = quote_stream.receive_nowait()
-                ticks = next_quote.get('ticks')
-
-                if ticks:
-                    first_quote['ticks'].extend(ticks)
-
-            except trio.WouldBlock:
-                now = time.time()
-                rate = 1 / (now - last_send)
-                last_send = now
-
-                # print(f'{rate} Hz sending quotes\n{first_quote}')
-
-                # TODO: now if only we could sync this to the display
-                # rate timing exactly lul
+        if left_to_sleep > 0:
+            with trio.move_on_after(left_to_sleep) as cs:
                 try:
-                    await stream.send({first_quote['symbol']: first_quote})
+                    sym, last_quote = await quote_stream.receive()
+                except trio.EndOfChannel:
+                    log.exception(f"feed for {stream} ended?")
                     break
-                except trio.ClosedResourceError:
-                    # if the feed consumer goes down then drop
-                    # out of this rate limiter
-                    log.warning(f'{stream} closed')
-                    return
 
-        end = time.time()
-        diff = end - start
+                diff = time.time() - last_send
 
-        # throttle to provided transmit rate
-        period = max(sleep_period - diff, 0)
-        if period > 0:
-            await trio.sleep(period)
+                if not first_quote:
+                    first_quote = last_quote
+
+                if (throttle_period - diff) > 0:
+                    # received a quote but the send cycle period hasn't yet
+                    # expired we aren't supposed to send yet so append
+                    # to the tick frame.
+
+                    # append quotes since last iteration into the last quote's
+                    # tick array/buffer.
+                    ticks = last_quote.get('ticks')
+
+                    # XXX: idea for frame type data structure we could
+                    # use on the wire instead of a simple list?
+                    # frames = {
+                    #     'index': ['type_a', 'type_c', 'type_n', 'type_n'],
+
+                    #     'type_a': [tick0, tick1, tick2, .., tickn],
+                    #     'type_b': [tick0, tick1, tick2, .., tickn],
+                    #     'type_c': [tick0, tick1, tick2, .., tickn],
+                    #     ...
+                    #     'type_n': [tick0, tick1, tick2, .., tickn],
+                    # }
+
+                    # TODO: once we decide to get fancy really we should
+                    # have a shared mem tick buffer that is just
+                    # continually filled and the UI just ready from it
+                    # at it's display rate.
+                    if ticks:
+                        first_quote['ticks'].extend(ticks)
+
+                    # send cycle isn't due yet so continue waiting
+                    continue
+
+        if cs.cancelled_caught:
+            # 2 cases:
+            # no quote has arrived yet this cycle so wait for
+            # the next one.
+            if not first_quote:
+                # if no last quote was received since the last send
+                # cycle **AND** if we timed out waiting for a most
+                # recent quote **but** the throttle cycle is now due to
+                # be sent -> we want to immediately send the next
+                # received quote ASAP.
+                sym, first_quote = await quote_stream.receive()
+
+            # we have a quote already so send it now.
+
+        # measured_rate = 1 / (time.time() - last_send)
+        # log.info(
+        #     f'`{sym}` throttled send hz: {round(measured_rate, ndigits=1)}'
+        # )
+
+        # TODO: now if only we could sync this to the display
+        # rate timing exactly lul
+        try:
+            await stream.send({sym: first_quote})
+        except (
+            # NOTE: any of these can be raised by ``tractor``'s IPC
+            # transport-layer and we want to be highly resilient
+            # to consumers which crash or lose network connection.
+            # I.e. we **DO NOT** want to crash and propagate up to
+            # ``pikerd`` these kinds of errors!
+            trio.ClosedResourceError,
+            trio.BrokenResourceError,
+            ConnectionResetError,
+        ):
+            # if the feed consumer goes down then drop
+            # out of this rate limiter
+            log.warning(f'{stream} closed')
+            await stream.aclose()
+            return
+
+        # reset send cycle state
+        first_quote = last_quote = None
+        diff = 0
+        last_send = time.time()

piker/data/_sharedmem.py

@@ -15,48 +15,65 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 """
-NumPy compatible shared memory buffers for real-time FSP.
+NumPy compatible shared memory buffers for real-time IPC streaming.
 
 """
-from dataclasses import dataclass, asdict
+from __future__ import annotations
 from sys import byteorder
-from typing import List, Tuple, Optional
+import time
+from typing import Optional
 from multiprocessing.shared_memory import SharedMemory, _USE_POSIX
-from multiprocessing import resource_tracker as mantracker
 
 if _USE_POSIX:
     from _posixshmem import shm_unlink
 
 import tractor
 import numpy as np
+from pydantic import BaseModel
+from numpy.lib import recfunctions as rfn
 
 from ..log import get_logger
-from ._source import base_ohlc_dtype, base_iohlc_dtype
+from ._source import base_iohlc_dtype
 
 
 log = get_logger(__name__)
 
 
-# Tell the "resource tracker" thing to fuck off.
-class ManTracker(mantracker.ResourceTracker):
-    def register(self, name, rtype):
-        pass
-
-    def unregister(self, name, rtype):
-        pass
-
-    def ensure_running(self):
-        pass
+# how  much is probably dependent on lifestyle
+_secs_in_day = int(60 * 60 * 24)
+# we try for a buncha times, but only on a run-every-other-day kinda week.
+_days_worth = 16
+_default_size = _days_worth * _secs_in_day
+# where to start the new data append index
+_rt_buffer_start = int((_days_worth - 1) * _secs_in_day)
 
 
-# "know your land and know your prey"
-# https://www.dailymotion.com/video/x6ozzco
-mantracker._resource_tracker = ManTracker()
-mantracker.register = mantracker._resource_tracker.register
-mantracker.ensure_running = mantracker._resource_tracker.ensure_running
-ensure_running = mantracker._resource_tracker.ensure_running
-mantracker.unregister = mantracker._resource_tracker.unregister
-mantracker.getfd = mantracker._resource_tracker.getfd
+def cuckoff_mantracker():
+
+    from multiprocessing import resource_tracker as mantracker
+
+    # Tell the "resource tracker" thing to fuck off.
+    class ManTracker(mantracker.ResourceTracker):
+        def register(self, name, rtype):
+            pass
+
+        def unregister(self, name, rtype):
+            pass
+
+        def ensure_running(self):
+            pass
+
+    # "know your land and know your prey"
+    # https://www.dailymotion.com/video/x6ozzco
+    mantracker._resource_tracker = ManTracker()
+    mantracker.register = mantracker._resource_tracker.register
+    mantracker.ensure_running = mantracker._resource_tracker.ensure_running
+    # ensure_running = mantracker._resource_tracker.ensure_running
+    mantracker.unregister = mantracker._resource_tracker.unregister
+    mantracker.getfd = mantracker._resource_tracker.getfd
+
+
+cuckoff_mantracker()
 
 
 class SharedInt:
@@ -82,29 +99,42 @@ class SharedInt:
         if _USE_POSIX:
             # We manually unlink to bypass all the "resource tracker"
             # nonsense meant for non-SC systems.
-            shm_unlink(self._shm.name)
+            name = self._shm.name
+            try:
+                shm_unlink(name)
+            except FileNotFoundError:
+                # might be a teardown race here?
+                log.warning(f'Shm for {name} already unlinked?')
 
 
-@dataclass
-class _Token:
-    """Internal represenation of a shared memory "token"
+class _Token(BaseModel):
+    '''
+    Internal represenation of a shared memory "token"
     which can be used to key a system wide post shm entry.
-    """
+
+    '''
+    class Config:
+        frozen = True
+
     shm_name: str  # this servers as a "key" value
     shm_first_index_name: str
     shm_last_index_name: str
-    dtype_descr: List[Tuple[str]]
+    dtype_descr: tuple
 
-    def __post_init__(self):
-        # np.array requires a list for dtype
-        self.dtype_descr = np.dtype(list(map(tuple, self.dtype_descr))).descr
+    @property
+    def dtype(self) -> np.dtype:
+        return np.dtype(list(map(tuple, self.dtype_descr))).descr
 
     def as_msg(self):
-        return asdict(self)
+        return self.dict()
 
     @classmethod
-    def from_msg(self, msg: dict) -> '_Token':
-        return msg if isinstance(msg, _Token) else _Token(**msg)
+    def from_msg(cls, msg: dict) -> _Token:
+        if isinstance(msg, _Token):
+            return msg
+
+        msg['dtype_descr'] = tuple(map(tuple, msg['dtype_descr']))
+        return _Token(**msg)
 
 
 # TODO: this api?
@@ -127,20 +157,23 @@ def _make_token(
     key: str,
     dtype: Optional[np.dtype] = None,
 ) -> _Token:
-    """Create a serializable token that can be used
+    '''
+    Create a serializable token that can be used
     to access a shared array.
-    """
+
+    '''
     dtype = base_iohlc_dtype if dtype is None else dtype
     return _Token(
-        key,
-        key + "_first",
-        key + "_last",
-        np.dtype(dtype).descr
+        shm_name=key,
+        shm_first_index_name=key + "_first",
+        shm_last_index_name=key + "_last",
+        dtype_descr=np.dtype(dtype).descr
     )
 
 
 class ShmArray:
-    """A shared memory ``numpy`` (compatible) array API.
+    '''
+    A shared memory ``numpy`` (compatible) array API.
 
     An underlying shared memory buffer is allocated based on
     a user specified ``numpy.ndarray``. This fixed size array
@@ -150,7 +183,7 @@ class ShmArray:
     ``SharedInt`` interfaces) values such that multiple processes can
     interact with the same array using a synchronized-index.
 
-    """
+    '''
     def __init__(
         self,
         shmarr: np.ndarray,
@@ -168,19 +201,24 @@ class ShmArray:
 
         self._len = len(shmarr)
         self._shm = shm
+        self._post_init: bool = False
 
         # pushing data does not write the index (aka primary key)
-        self._write_fields = list(shmarr.dtype.fields.keys())[1:]
+        dtype = shmarr.dtype
+        if dtype.fields:
+            self._write_fields = list(shmarr.dtype.fields.keys())[1:]
+        else:
+            self._write_fields = None
 
     # TODO: ringbuf api?
 
     @property
     def _token(self) -> _Token:
         return _Token(
-            self._shm.name,
-            self._first._shm.name,
-            self._last._shm.name,
-            self._array.dtype.descr,
+            shm_name=self._shm.name,
+            shm_first_index_name=self._first._shm.name,
+            shm_last_index_name=self._last._shm.name,
+            dtype_descr=tuple(self._array.dtype.descr),
         )
 
     @property
@@ -196,44 +234,151 @@ class ShmArray:
 
     @property
     def array(self) -> np.ndarray:
-        return self._array[self._first.value:self._last.value]
+        '''
+        Return an up-to-date ``np.ndarray`` view of the
+        so-far-written data to the underlying shm buffer.
+
+        '''
+        a = self._array[self._first.value:self._last.value]
+
+        # first, last = self._first.value, self._last.value
+        # a = self._array[first:last]
+
+        # TODO: eventually comment this once we've not seen it in the
+        # wild in a long time..
+        # XXX: race where first/last indexes cause a reader
+        # to load an empty array..
+        if len(a) == 0 and self._post_init:
+            raise RuntimeError('Empty array race condition hit!?')
+            # breakpoint()
+
+        return a
+
+    def ustruct(
+        self,
+        fields: Optional[list[str]] = None,
+
+        # type that all field values will be cast to
+        # in the returned view.
+        common_dtype: np.dtype = np.float,
+
+    ) -> np.ndarray:
+
+        array = self._array
+
+        if fields:
+            selection = array[fields]
+            # fcount = len(fields)
+        else:
+            selection = array
+            # fcount = len(array.dtype.fields)
+
+        # XXX: manual ``.view()`` attempt that also doesn't work.
+        # uview = selection.view(
+        #     dtype='<f16',
+        # ).reshape(-1, 4, order='A')
+
+        # assert len(selection) == len(uview)
+
+        u = rfn.structured_to_unstructured(
+            selection,
+            # dtype=float,
+            copy=True,
+        )
+
+        # unstruct = np.ndarray(u.shape, dtype=a.dtype, buffer=shm.buf)
+        # array[:] = a[:]
+        return u
+        # return ShmArray(
+        #     shmarr=u,
+        #     first=self._first,
+        #     last=self._last,
+        #     shm=self._shm
+        # )
 
     def last(
         self,
         length: int = 1,
+
     ) -> np.ndarray:
+        '''
+        Return the last ``length``'s worth of ("row") entries from the
+        array.
+
+        '''
         return self.array[-length:]
 
     def push(
         self,
         data: np.ndarray,
+
+        field_map: Optional[dict[str, str]] = None,
         prepend: bool = False,
+        update_first: bool = True,
+        start: Optional[int] = None,
+
     ) -> int:
-        """Ring buffer like "push" to append data
+        '''
+        Ring buffer like "push" to append data
         into the buffer and return updated "last" index.
-        """
+
+        NB: no actual ring logic yet to give a "loop around" on overflow
+        condition, lel.
+
+        '''
         length = len(data)
 
         if prepend:
-            index = self._first.value - length
+            index = (start or self._first.value) - length
+
+            if index < 0:
+                raise ValueError(
+                    f'Array size of {self._len} was overrun during prepend.\n'
+                    f'You have passed {abs(index)} too many datums.'
+                )
+
         else:
-            index = self._last.value
+            index = start if start is not None else self._last.value
 
         end = index + length
 
-        fields = self._write_fields
+        if field_map:
+            src_names, dst_names = zip(*field_map.items())
+        else:
+            dst_names = src_names = self._write_fields
 
         try:
-            self._array[fields][index:end] = data[fields][:]
-            if prepend:
-                self._first.value = index
-            else:
-                self._last.value = end
-            return end
-        except ValueError as err:
-            # shoudl raise if diff detected
-            self.diff_err_fields(data)
+            self._array[
+                list(dst_names)
+            ][index:end] = data[list(src_names)][:]
 
+            # NOTE: there was a race here between updating
+            # the first and last indices and when the next reader
+            # tries to access ``.array`` (which due to the index
+            # overlap will be empty). Pretty sure we've fixed it now
+            # but leaving this here as a reminder.
+            if prepend and update_first and length:
+                assert index < self._first.value
+
+            if (
+                index < self._first.value
+                and update_first
+            ):
+                assert prepend, 'prepend=True not passed but index decreased?'
+                self._first.value = index
+
+            elif not prepend:
+                self._last.value = end
+
+            self._post_init = True
+            return end
+
+        except ValueError as err:
+            if field_map:
+                raise
+
+            # should raise if diff detected
+            self.diff_err_fields(data)
             raise err
 
     def diff_err_fields(
@@ -258,6 +403,7 @@ class ShmArray:
                 f"Input array has unknown field(s): {only_in_theirs}"
             )
 
+    # TODO: support "silent" prepends that don't update ._first.value?
     def prepend(
         self,
         data: np.ndarray,
@@ -284,21 +430,20 @@ class ShmArray:
         ...
 
 
-# how  much is probably dependent on lifestyle
-_secs_in_day = int(60 * 60 * 12)
-_default_size = 2 * _secs_in_day
-
 def open_shm_array(
+
     key: Optional[str] = None,
     size: int = _default_size,
     dtype: Optional[np.dtype] = None,
     readonly: bool = False,
+
 ) -> ShmArray:
-    """Open a memory shared ``numpy`` using the standard library.
+    '''Open a memory shared ``numpy`` using the standard library.
 
     This call unlinks (aka permanently destroys) the buffer on teardown
     and thus should be used from the parent-most accessor (process).
-    """
+
+    '''
     # create new shared mem segment for which we
     # have write permission
     a = np.zeros(size, dtype=dtype)
@@ -309,7 +454,11 @@ def open_shm_array(
         create=True,
         size=a.nbytes
     )
-    array = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf)
+    array = np.ndarray(
+        a.shape,
+        dtype=a.dtype,
+        buffer=shm.buf
+    )
     array[:] = a[:]
     array.setflags(write=int(not readonly))
 
@@ -335,7 +484,24 @@ def open_shm_array(
         )
     )
 
-    last.value = first.value = int(_secs_in_day)
+    # start the "real-time" updated section after 3-days worth of 1s
+    # sampled OHLC. this allows appending up to a days worth from
+    # tick/quote feeds before having to flush to a (tsdb) storage
+    # backend, and looks something like,
+    # -------------------------
+    # |              |        i
+    # _________________________
+    # <-------------> <------->
+    #  history         real-time
+    #
+    # Once fully "prepended", the history section will leave the
+    # ``ShmArray._start.value: int = 0`` and the yet-to-be written
+    # real-time section will start at ``ShmArray.index: int``.
+
+    # this sets the index to 3/4 of the length of the buffer
+    # leaving a "days worth of second samples" for the real-time
+    # section.
+    last.value = first.value = _rt_buffer_start
 
     shmarr = ShmArray(
         array,
@@ -349,6 +515,7 @@ def open_shm_array(
 
     # "unlink" created shm on process teardown by
     # pushing teardown calls onto actor context stack
+
     tractor._actor._lifetime_stack.callback(shmarr.close)
     tractor._actor._lifetime_stack.callback(shmarr.destroy)
 
@@ -356,27 +523,48 @@ def open_shm_array(
 
 
 def attach_shm_array(
-    token: Tuple[str, str, Tuple[str, str]],
+    token: tuple[str, str, tuple[str, str]],
     size: int = _default_size,
     readonly: bool = True,
+
 ) -> ShmArray:
-    """Attach to an existing shared memory array previously
+    '''
+    Attach to an existing shared memory array previously
     created by another process using ``open_shared_array``.
 
     No new shared mem is allocated but wrapper types for read/write
     access are constructed.
-    """
+
+    '''
     token = _Token.from_msg(token)
     key = token.shm_name
 
     if key in _known_tokens:
         assert _Token.from_msg(_known_tokens[key]) == token, "WTF"
 
+    # XXX: ugh, looks like due to the ``shm_open()`` C api we can't
+    # actually place files in a subdir, see discussion here:
+    # https://stackoverflow.com/a/11103289
+
     # attach to array buffer and view as per dtype
-    shm = SharedMemory(name=key)
+    _err: Optional[Exception] = None
+    for _ in range(3):
+        try:
+            shm = SharedMemory(
+                name=key,
+                create=False,
+            )
+            break
+        except OSError as oserr:
+            _err = oserr
+            time.sleep(0.1)
+    else:
+        if _err:
+            raise _err
+
     shmarr = np.ndarray(
         (size,),
-        dtype=token.dtype_descr,
+        dtype=token.dtype,
         buffer=shm.buf
     )
     shmarr.setflags(write=int(not readonly))
@@ -424,8 +612,10 @@ def maybe_open_shm_array(
     key: str,
     dtype: Optional[np.dtype] = None,
     **kwargs,
-) -> Tuple[ShmArray, bool]:
-    """Attempt to attach to a shared memory block using a "key" lookup
+
+) -> tuple[ShmArray, bool]:
+    '''
+    Attempt to attach to a shared memory block using a "key" lookup
     to registered blocks in the users overall "system" registry
     (presumes you don't have the block's explicit token).
 
@@ -439,7 +629,8 @@ def maybe_open_shm_array(
 
     If you know the explicit ``_Token`` for your memory segment instead
     use ``attach_shm_array``.
-    """
+
+    '''
     try:
         # see if we already know this key
         token = _known_tokens[key]
@@ -459,3 +650,35 @@ def maybe_open_shm_array(
         # to fail if a block has been allocated
         # on the OS by someone else.
         return open_shm_array(key=key, dtype=dtype, **kwargs), True
+
+
+def try_read(
+    array: np.ndarray
+
+) -> Optional[np.ndarray]:
+    '''
+    Try to read the last row from a shared mem array or ``None``
+    if the array read returns a zero-length array result.
+
+    Can be used to check for backfilling race conditions where an array
+    is currently being (re-)written by a writer actor but the reader is
+    unaware and reads during the window where the first and last indexes
+    are being updated.
+
+    '''
+    try:
+        return array[-1]
+    except IndexError:
+        # XXX: race condition with backfilling shm.
+        #
+        # the underlying issue is that a backfill (aka prepend) and subsequent
+        # shm array first/last index update could result in an empty array
+        # read here since the indices may be updated in such a way that
+        # a read delivers an empty array (though it seems like we
+        # *should* be able to prevent that?). also, as and alt and
+        # something we need anyway, maybe there should be some kind of
+        # signal that a prepend is taking place and this consumer can
+        # respond (eg. redrawing graphics) accordingly.
+
+        # the array read was emtpy
+        return None

piker/data/_source.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship for piker0)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,11 +17,12 @@
 """
 numpy data source coversion helpers.
 """
-from typing import Dict, Any, List
+from __future__ import annotations
+from typing import Any
 import decimal
 
+from bidict import bidict
 import numpy as np
-import pandas as pd
 from pydantic import BaseModel
 # from numba import from_dtype
 
@@ -32,7 +33,7 @@ ohlc_fields = [
     ('high', float),
     ('low', float),
     ('close', float),
-    ('volume', int),
+    ('volume', float),
     ('bar_wap', float),
 ]
 
@@ -47,21 +48,37 @@ base_ohlc_dtype = np.dtype(ohlc_fields)
 # https://github.com/numba/numba/issues/4511
 # numba_ohlc_dtype = from_dtype(base_ohlc_dtype)
 
-# map time frame "keys" to minutes values
-tf_in_1m = {
-    '1m': 1,
-    '5m':  5,
-    '15m': 15,
-    '30m':  30,
-    '1h': 60,
-    '4h': 240,
-    '1d': 1440,
-}
+# map time frame "keys" to seconds values
+tf_in_1s = bidict({
+    1: '1s',
+    60: '1m',
+    60*5: '5m',
+    60*15: '15m',
+    60*30: '30m',
+    60*60: '1h',
+    60*60*24: '1d',
+})
+
+
+def mk_fqsn(
+    provider: str,
+    symbol: str,
+
+) -> str:
+    '''
+    Generate a "fully qualified symbol name" which is
+    a reverse-hierarchical cross broker/provider symbol
+
+    '''
+    return '.'.join([symbol, provider]).lower()
 
 
 def float_digits(
     value: float,
 ) -> int:
+    if value == 0:
+        return 0
+
     return int(-decimal.Decimal(str(value)).as_tuple().exponent)
 
 
@@ -75,94 +92,166 @@ def ohlc_zeros(length: int) -> np.ndarray:
     return np.zeros(length, dtype=base_ohlc_dtype)
 
 
+def unpack_fqsn(fqsn: str) -> tuple[str, str, str]:
+    '''
+    Unpack a fully-qualified-symbol-name to ``tuple``.
+
+    '''
+    venue = ''
+    suffix = ''
+
+    # TODO: probably reverse the order of all this XD
+    tokens = fqsn.split('.')
+    if len(tokens) < 3:
+        # probably crypto
+        symbol, broker = tokens
+        return (
+            broker,
+            symbol,
+            '',
+        )
+
+    elif len(tokens) > 3:
+        symbol, venue, suffix, broker = tokens
+    else:
+        symbol, venue, broker = tokens
+        suffix = ''
+
+    # head, _, broker = fqsn.rpartition('.')
+    # symbol, _, suffix = head.rpartition('.')
+    return (
+        broker,
+        '.'.join([symbol, venue]),
+        suffix,
+    )
+
+
 class Symbol(BaseModel):
-    """I guess this is some kinda container thing for dealing with
+    '''
+    I guess this is some kinda container thing for dealing with
     all the different meta-data formats from brokers?
 
-    Yah, i guess dats what it izz.
-    """
+    '''
     key: str
     tick_size: float = 0.01
-    lot_tick_size: float = 0.01  # "volume" precision as min step value
-    broker_info: Dict[str, Dict[str, Any]] = {}
+    lot_tick_size: float = 0.0  # "volume" precision as min step value
+    tick_size_digits: int = 2
+    lot_size_digits: int = 0
+    suffix: str = ''
+    broker_info: dict[str, dict[str, Any]] = {}
 
     # specifies a "class" of financial instrument
     # ex. stock, futer, option, bond etc.
-    type_key: str
+
+    # @validate_arguments
+    @classmethod
+    def from_broker_info(
+        cls,
+        broker: str,
+        symbol: str,
+        info: dict[str, Any],
+        suffix: str = '',
+
+    # XXX: like wtf..
+    # ) -> 'Symbol':
+    ) -> None:
+
+        tick_size = info.get('price_tick_size', 0.01)
+        lot_tick_size = info.get('lot_tick_size', 0.0)
+
+        return Symbol(
+            key=symbol,
+            tick_size=tick_size,
+            lot_tick_size=lot_tick_size,
+            tick_size_digits=float_digits(tick_size),
+            lot_size_digits=float_digits(lot_tick_size),
+            suffix=suffix,
+            broker_info={broker: info},
+        )
+
+    @classmethod
+    def from_fqsn(
+        cls,
+        fqsn: str,
+        info: dict[str, Any],
+
+    # XXX: like wtf..
+    # ) -> 'Symbol':
+    ) -> None:
+        broker, key, suffix = unpack_fqsn(fqsn)
+        return cls.from_broker_info(
+            broker,
+            key,
+            info=info,
+            suffix=suffix,
+        )
 
     @property
-    def brokers(self) -> List[str]:
+    def type_key(self) -> str:
+        return list(self.broker_info.values())[0]['asset_type']
+
+    @property
+    def brokers(self) -> list[str]:
         return list(self.broker_info.keys())
 
-    def digits(self) -> int:
-        """Return the trailing number of digits specified by the min
-        tick size for the instrument.
-
-        """
-        return float_digits(self.tick_size)
-
-    def lot_digits(self) -> int:
-        return float_digits(self.lot_tick_size)
-
     def nearest_tick(self, value: float) -> float:
-        """Return the nearest tick value based on mininum increment.
+        '''
+        Return the nearest tick value based on mininum increment.
 
-        """
+        '''
         mult = 1 / self.tick_size
         return round(value * mult) / mult
 
+    def front_feed(self) -> tuple[str, str]:
+        '''
+        Return the "current" feed key for this symbol.
 
-def from_df(
-    df: pd.DataFrame,
-    source=None,
-    default_tf=None
-) -> np.recarray:
-    """Convert OHLC formatted ``pandas.DataFrame`` to ``numpy.recarray``.
+        (i.e. the broker + symbol key in a tuple).
 
-    """
-    df.reset_index(inplace=True)
+        '''
+        return (
+            list(self.broker_info.keys())[0],
+            self.key,
+        )
 
-    # hackery to convert field names
-    date = 'Date'
-    if 'date' in df.columns:
-        date = 'date'
+    def tokens(self) -> tuple[str]:
+        broker, key = self.front_feed()
+        if self.suffix:
+            return (key, self.suffix, broker)
+        else:
+            return (key, broker)
 
-    # convert to POSIX time
-    df[date] = [d.timestamp() for d in df[date]]
+    def front_fqsn(self) -> str:
+        '''
+        fqsn = "fully qualified symbol name"
 
-    # try to rename from some camel case
-    columns = {
-        'Date': 'time',
-        'date': 'time',
-        'Open': 'open',
-        'High': 'high',
-        'Low': 'low',
-        'Close': 'close',
-        'Volume': 'volume',
+        Basically the idea here is for all client-ish code (aka programs/actors
+        that ask the provider agnostic layers in the stack for data) should be
+        able to tell which backend / venue / derivative each data feed/flow is
+        from by an explicit string key of the current form:
 
-        # most feeds are providing this over sesssion anchored
-        'vwap': 'bar_wap',
+        <instrumentname>.<venue>.<suffixwithmetadata>.<brokerbackendname>
 
-        # XXX: ib_insync calls this the "wap of the bar"
-        # but no clue what is actually is...
-        # https://github.com/pikers/piker/issues/119#issuecomment-729120988
-        'average': 'bar_wap',
-    }
+        TODO: I have thoughts that we should actually change this to be
+        more like an "attr lookup" (like how the web should have done
+        urls, but marketting peeps ruined it etc. etc.):
 
-    df = df.rename(columns=columns)
+        <broker>.<venue>.<instrumentname>.<suffixwithmetadata>
 
-    for name in df.columns:
-        # if name not in base_ohlc_dtype.names[1:]:
-        if name not in base_ohlc_dtype.names:
-            del df[name]
+        '''
+        tokens = self.tokens()
+        fqsn = '.'.join(tokens)
+        return fqsn
 
-    # TODO: it turns out column access on recarrays is actually slower:
-    # https://jakevdp.github.io/PythonDataScienceHandbook/02.09-structured-data-numpy.html#RecordArrays:-Structured-Arrays-with-a-Twist
-    # it might make sense to make these structured arrays?
-    array = df.to_records(index=False)
-    _nan_to_closest_num(array)
+    def iterfqsns(self) -> list[str]:
+        keys = []
+        for broker in self.broker_info.keys():
+            fqsn = mk_fqsn(self.key, broker)
+            if self.suffix:
+                fqsn += f'.{self.suffix}'
+            keys.append(fqsn)
 
-    return array
+        return keys
 
 
 def _nan_to_closest_num(array: np.ndarray):

piker/data/_web_bs.py

@@ -20,7 +20,7 @@ ToOlS fOr CoPInG wITh "tHE wEB" protocols.
 """
 from contextlib import asynccontextmanager, AsyncExitStack
 from types import ModuleType
-from typing import Any, Callable
+from typing import Any, Callable, AsyncGenerator
 import json
 
 import trio
@@ -53,11 +53,13 @@ class NoBsWs:
     def __init__(
         self,
         url: str,
+        token: str,
         stack: AsyncExitStack,
         fixture: Callable,
         serializer: ModuleType = json,
     ):
         self.url = url
+        self.token = token
         self.fixture = fixture
         self._stack = stack
         self._ws: 'WebSocketConnection' = None  # noqa
@@ -81,9 +83,15 @@ class NoBsWs:
                     trio_websocket.open_websocket_url(self.url)
                 )
                 # rerun user code fixture
-                ret = await self._stack.enter_async_context(
-                    self.fixture(self)
-                )
+                if self.token == '':
+                    ret = await self._stack.enter_async_context(
+                        self.fixture(self)
+                    )
+                else:
+                    ret = await self._stack.enter_async_context(
+                        self.fixture(self, self.token)
+                    )
+
                 assert ret is None
 
                 log.info(f'Connection success: {self.url}')
@@ -127,12 +135,14 @@ async def open_autorecon_ws(
 
     # TODO: proper type annot smh
     fixture: Callable,
-):
+    # used for authenticated websockets
+    token: str = '',
+) -> AsyncGenerator[tuple[...],  NoBsWs]:
     """Apparently we can QoS for all sorts of reasons..so catch em.
 
     """
     async with AsyncExitStack() as stack:
-        ws = NoBsWs(url, stack, fixture=fixture)
+        ws = NoBsWs(url, token, stack, fixture=fixture)
         await ws._connect()
 
         try:

piker/data/cli.py

@@ -16,26 +16,34 @@
 
 """
 marketstore cli.
+
 """
-from typing import List
 from functools import partial
 from pprint import pformat
 
+from anyio_marketstore import open_marketstore_client
 import trio
 import tractor
 import click
+import numpy as np
 
 from .marketstore import (
     get_client,
-    stream_quotes,
+    # stream_quotes,
     ingest_quote_stream,
-    _url,
+    # _url,
     _tick_tbk_ids,
     mk_tbk,
 )
 from ..cli import cli
 from .. import watchlists as wl
 from ..log import get_logger
+from ._sharedmem import (
+    maybe_open_shm_array,
+)
+from ._source import (
+    base_iohlc_dtype,
+)
 
 
 log = get_logger(__name__)
@@ -49,51 +57,58 @@ log = get_logger(__name__)
 )
 @click.argument('names', nargs=-1)
 @click.pass_obj
-def ms_stream(config: dict, names: List[str], url: str):
-    """Connect to a marketstore time bucket stream for (a set of) symbols(s)
+def ms_stream(
+    config: dict,
+    names: list[str],
+    url: str,
+) -> None:
+    '''
+    Connect to a marketstore time bucket stream for (a set of) symbols(s)
     and print to console.
-    """
+
+    '''
     async def main():
-        async for quote in stream_quotes(symbols=names):
-            log.info(f"Received quote:\n{quote}")
+        # async for quote in stream_quotes(symbols=names):
+        #    log.info(f"Received quote:\n{quote}")
+        ...
 
     trio.run(main)
 
 
-@cli.command()
-@click.option(
-    '--url',
-    default=_url,
-    help='HTTP URL of marketstore instance'
-)
-@click.argument('names', nargs=-1)
-@click.pass_obj
-def ms_destroy(config: dict, names: List[str], url: str) -> None:
-    """Destroy symbol entries in the local marketstore instance.
-    """
-    async def main():
-        nonlocal names
-        async with get_client(url) as client:
-
-            if not names:
-                names = await client.list_symbols()
-
-            # default is to wipe db entirely.
-            answer = input(
-                "This will entirely wipe you local marketstore db @ "
-                f"{url} of the following symbols:\n {pformat(names)}"
-                "\n\nDelete [N/y]?\n")
-
-            if answer == 'y':
-                for sym in names:
-                    # tbk = _tick_tbk.format(sym)
-                    tbk = tuple(sym, *_tick_tbk_ids)
-                    print(f"Destroying {tbk}..")
-                    await client.destroy(mk_tbk(tbk))
-            else:
-                print("Nothing deleted.")
-
-    tractor.run(main)
+# @cli.command()
+# @click.option(
+#     '--url',
+#     default=_url,
+#     help='HTTP URL of marketstore instance'
+# )
+# @click.argument('names', nargs=-1)
+# @click.pass_obj
+# def ms_destroy(config: dict, names: list[str], url: str) -> None:
+#     """Destroy symbol entries in the local marketstore instance.
+#     """
+#     async def main():
+#         nonlocal names
+#         async with get_client(url) as client:
+# 
+#             if not names:
+#                 names = await client.list_symbols()
+# 
+#             # default is to wipe db entirely.
+#             answer = input(
+#                 "This will entirely wipe you local marketstore db @ "
+#                 f"{url} of the following symbols:\n {pformat(names)}"
+#                 "\n\nDelete [N/y]?\n")
+# 
+#             if answer == 'y':
+#                 for sym in names:
+#                     # tbk = _tick_tbk.format(sym)
+#                     tbk = tuple(sym, *_tick_tbk_ids)
+#                     print(f"Destroying {tbk}..")
+#                     await client.destroy(mk_tbk(tbk))
+#             else:
+#                 print("Nothing deleted.")
+# 
+#     tractor.run(main)
 
 
 @cli.command()
@@ -102,41 +117,53 @@ def ms_destroy(config: dict, names: List[str], url: str) -> None:
     is_flag=True,
     help='Enable tractor logging')
 @click.option(
-    '--url',
-    default=_url,
-    help='HTTP URL of marketstore instance'
+    '--host',
+    default='localhost'
 )
-@click.argument('name', nargs=1, required=True)
+@click.option(
+    '--port',
+    default=5993
+)
+@click.argument('symbols', nargs=-1)
 @click.pass_obj
-def ms_shell(config, name, tl, url):
-    """Start an IPython shell ready to query the local marketstore db.
-    """
-    async def main():
-        async with get_client(url) as client:
-            query = client.query  # noqa
-            # TODO: write magics to query marketstore
-            from IPython import embed
-            embed()
+def storesh(
+    config,
+    tl,
+    host,
+    port,
+    symbols: list[str],
+):
+    '''
+    Start an IPython shell ready to query the local marketstore db.
 
-    tractor.run(main)
+    '''
+    from piker.data.marketstore import tsdb_history_update
+    from piker._daemon import open_piker_runtime
+
+    async def main():
+        nonlocal symbols
+
+        async with open_piker_runtime(
+            'storesh',
+            enable_modules=['piker.data._ahab'],
+        ):
+            symbol = symbols[0]
+            await tsdb_history_update(symbol)
+
+    trio.run(main)
 
 
 @cli.command()
 @click.option('--test-file', '-t', help='Test quote stream file')
 @click.option('--tl', is_flag=True, help='Enable tractor logging')
-@click.option('--tl', is_flag=True, help='Enable tractor logging')
-@click.option(
-    '--url',
-    default=_url,
-    help='HTTP URL of marketstore instance'
-)
 @click.argument('name', nargs=1, required=True)
 @click.pass_obj
-def ingest(config, name, test_file, tl, url):
-    """Ingest real-time broker quotes and ticks to a marketstore instance.
-    """
+def ingest(config, name, test_file, tl):
+    '''
+    Ingest real-time broker quotes and ticks to a marketstore instance.
+
+    '''
     # global opts
-    brokermod = config['brokermod']
     loglevel = config['loglevel']
     tractorloglevel = config['tractorloglevel']
     # log = config['log']
@@ -145,15 +172,25 @@ def ingest(config, name, test_file, tl, url):
     watchlists = wl.merge_watchlist(watchlist_from_file, wl._builtins)
     symbols = watchlists[name]
 
-    tractor.run(
-        partial(
-            ingest_quote_stream,
-            symbols,
-            brokermod.name,
-            tries=1,
-            loglevel=loglevel,
-        ),
-        name='ingest_marketstore',
-        loglevel=tractorloglevel,
-        debug_mode=True,
-    )
+    grouped_syms = {}
+    for sym in symbols:
+        symbol, _, provider = sym.rpartition('.')
+        if provider not in grouped_syms:
+            grouped_syms[provider] = []
+
+        grouped_syms[provider].append(symbol)
+
+    async def entry_point():
+        async with tractor.open_nursery() as n:
+            for provider, symbols in grouped_syms.items(): 
+                await n.run_in_actor(
+                    ingest_quote_stream,
+                    name='ingest_marketstore',
+                    symbols=symbols,
+                    brokername=provider,
+                    tries=1,
+                    actorloglevel=loglevel,
+                    loglevel=tractorloglevel
+                )
+
+    tractor.run(entry_point)

piker/data/marketstore.py

@@ -14,36 +14,200 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-"""
+'''
 ``marketstore`` integration.
 
 - client management routines
 - ticK data ingest routines
 - websocket client for subscribing to write triggers
 - todo: tick sequence stream-cloning for testing
-- todo: docker container management automation
-"""
-from contextlib import asynccontextmanager
-from typing import Dict, Any, List, Callable, Tuple
+
+'''
+from __future__ import annotations
+from contextlib import asynccontextmanager as acm
+from datetime import datetime
+from pprint import pformat
+from typing import (
+    Any,
+    Optional,
+    Union,
+    TYPE_CHECKING,
+)
 import time
 from math import isnan
 
+from bidict import bidict
 import msgpack
+import pyqtgraph as pg
 import numpy as np
-import pandas as pd
-import pymarketstore as pymkts
 import tractor
 from trio_websocket import open_websocket_url
+from anyio_marketstore import (
+    open_marketstore_client,
+    MarketstoreClient,
+    Params,
+)
+import pendulum
+import purerpc
 
+if TYPE_CHECKING:
+    import docker
+    from ._ahab import DockerContainer
+
+from .feed import maybe_open_feed
 from ..log import get_logger, get_console_log
-from ..data import open_feed
 
 
 log = get_logger(__name__)
 
-_tick_tbk_ids: Tuple[str, str] = ('1Sec', 'TICK')
+
+# container level config
+_config = {
+    'grpc_listen_port': 5995,
+    'ws_listen_port': 5993,
+    'log_level': 'debug',
+}
+
+_yaml_config = '''
+# piker's ``marketstore`` config.
+
+# mount this config using:
+# sudo docker run --mount \
+# type=bind,source="$HOME/.config/piker/",target="/etc" -i -p \
+# 5993:5993 alpacamarkets/marketstore:latest
+
+root_directory: data
+listen_port: {ws_listen_port}
+grpc_listen_port: {grpc_listen_port}
+log_level: {log_level}
+queryable: true
+stop_grace_period: 0
+wal_rotate_interval: 5
+stale_threshold: 5
+enable_add: true
+enable_remove: false
+
+triggers:
+  - module: ondiskagg.so
+    on: "*/1Sec/OHLCV"
+    config:
+        # filter: "nasdaq"
+        destinations:
+            - 1Min
+            - 5Min
+            - 15Min
+            - 1H
+            - 1D
+
+  - module: stream.so
+    on: '*/*/*'
+    # config:
+    #     filter: "nasdaq"
+
+'''.format(**_config)
+
+
+def start_marketstore(
+    client: docker.DockerClient,
+
+    **kwargs,
+
+) -> tuple[DockerContainer, dict[str, Any]]:
+    '''
+    Start and supervise a marketstore instance with its config bind-mounted
+    in from the piker config directory on the system.
+
+    The equivalent cli cmd to this code is:
+
+        sudo docker run --mount \
+        type=bind,source="$HOME/.config/piker/",target="/etc" -i -p \
+        5993:5993 alpacamarkets/marketstore:latest
+
+    '''
+    import os
+    import docker
+    from .. import config
+    get_console_log('info', name=__name__)
+
+    mktsdir = os.path.join(config._config_dir, 'marketstore')
+
+    # create when dne
+    if not os.path.isdir(mktsdir):
+        os.mkdir(mktsdir)
+
+    yml_file = os.path.join(mktsdir, 'mkts.yml')
+    if not os.path.isfile(yml_file):
+        log.warning(
+            f'No `marketstore` config exists?: {yml_file}\n'
+            'Generating new file from template:\n'
+            f'{_yaml_config}\n'
+        )
+        with open(yml_file, 'w') as yf:
+            yf.write(_yaml_config)
+
+    # create a mount from user's local piker config dir into container
+    config_dir_mnt = docker.types.Mount(
+        target='/etc',
+        source=mktsdir,
+        type='bind',
+    )
+
+    # create a user config subdir where the marketstore
+    # backing filesystem database can be persisted.
+    persistent_data_dir = os.path.join(
+        mktsdir, 'data',
+    )
+    if not os.path.isdir(persistent_data_dir):
+        os.mkdir(persistent_data_dir)
+
+    data_dir_mnt = docker.types.Mount(
+        target='/data',
+        source=persistent_data_dir,
+        type='bind',
+    )
+
+    dcntr: DockerContainer = client.containers.run(
+        'alpacamarkets/marketstore:latest',
+        # do we need this for cmds?
+        # '-i',
+
+        # '-p 5993:5993',
+        ports={
+            '5993/tcp': 5993,  # jsonrpc / ws?
+            '5995/tcp': 5995,  # grpc
+        },
+        mounts=[
+            config_dir_mnt,
+            data_dir_mnt,
+        ],
+        detach=True,
+        # stop_signal='SIGINT',
+        init=True,
+        # remove=True,
+    )
+    return (
+        dcntr,
+        _config,
+
+        # expected startup and stop msgs
+        "launching tcp listener for all services...",
+        "exiting...",
+    )
+
+
+_tick_tbk_ids: tuple[str, str] = ('1Sec', 'TICK')
 _tick_tbk: str = '{}/' + '/'.join(_tick_tbk_ids)
-_url: str = 'http://localhost:5993/rpc'
+
+_tick_dt = [
+    # these two are required for as a "primary key"
+    ('Epoch', 'i8'),
+    ('Nanoseconds', 'i4'),
+    ('IsTrade', 'i1'),
+    ('IsBid', 'i1'),
+    ('Price', 'f4'),
+    ('Size', 'f4')
+]
+
 _quote_dt = [
     # these two are required for as a "primary key"
     ('Epoch', 'i8'),
@@ -61,6 +225,7 @@ _quote_dt = [
     # ('brokerd_ts', 'i64'),
     # ('VWAP', 'f4')
 ]
+
 _quote_tmp = {}.fromkeys(dict(_quote_dt).keys(), np.nan)
 _tick_map = {
     'Up': 1,
@@ -69,31 +234,52 @@ _tick_map = {
     None: np.nan,
 }
 
+_ohlcv_dt = [
+    # these two are required for as a "primary key"
+    ('Epoch', 'i8'),
+    # ('Nanoseconds', 'i4'),
 
-class MarketStoreError(Exception):
-    "Generic marketstore client error"
+    # ohlcv sampling
+    ('Open', 'f4'),
+    ('High', 'f4'),
+    ('Low', 'f4'),
+    ('Close', 'f4'),
+    ('Volume', 'f4'),
+]
 
 
-def err_on_resp(response: dict) -> None:
-    """Raise any errors found in responses from client request.
-    """
-    responses = response['responses']
-    if responses is not None:
-        for r in responses:
-            err = r['error']
-            if err:
-                raise MarketStoreError(err)
+ohlc_key_map = bidict({
+    'Epoch': 'time',
+    'Open': 'open',
+    'High': 'high',
+    'Low': 'low',
+    'Close': 'close',
+    'Volume': 'volume',
+})
+
+
+def mk_tbk(keys: tuple[str, str, str]) -> str:
+    '''
+    Generate a marketstore table key from a tuple.
+    Converts,
+        ``('SPY', '1Sec', 'TICK')`` -> ``"SPY/1Sec/TICK"```
+
+    '''
+    return '/'.join(keys)
 
 
 def quote_to_marketstore_structarray(
-    quote: Dict[str, Any],
-    last_fill: str,
+    quote: dict[str, Any],
+    last_fill: Optional[float]
+
 ) -> np.array:
-    """Return marketstore writeable structarray from quote ``dict``.
-    """
+    '''
+    Return marketstore writeable structarray from quote ``dict``.
+
+    '''
     if last_fill:
         # new fill bby
-        now = timestamp(last_fill)
+        now = int(pendulum.parse(last_fill).timestamp)
     else:
         # this should get inserted upstream by the broker-client to
         # subtract from IPC latency
@@ -101,7 +287,7 @@ def quote_to_marketstore_structarray(
 
     secs, ns = now / 10**9, now % 10**9
 
-    # pack into List[Tuple[str, Any]]
+    # pack into list[tuple[str, Any]]
     array_input = []
 
     # insert 'Epoch' entry first and then 'Nanoseconds'.
@@ -123,146 +309,467 @@ def quote_to_marketstore_structarray(
     return np.array([tuple(array_input)], dtype=_quote_dt)
 
 
-def timestamp(datestr: str) -> int:
-    """Return marketstore compatible 'Epoch' integer in nanoseconds
-    from a date formatted str.
-    """
-    return int(pd.Timestamp(datestr).value)
-
-
-def mk_tbk(keys: Tuple[str, str, str]) -> str:
-    """Generate a marketstore table key from a tuple.
-
-    Converts,
-        ``('SPY', '1Sec', 'TICK')`` -> ``"SPY/1Sec/TICK"```
-    """
-    return '{}/' + '/'.join(keys)
-
-
-class Client:
-    """Async wrapper around the alpaca ``pymarketstore`` sync client.
-
-    This will server as the shell for building out a proper async client
-    that isn't horribly documented and un-tested..
-    """
-    def __init__(self, url: str):
-        self._client = pymkts.Client(url)
-
-    async def _invoke(
-        self,
-        meth: Callable,
-        *args,
-        **kwargs,
-    ) -> Any:
-        return err_on_resp(meth(*args, **kwargs))
-
-    async def destroy(
-        self,
-        tbk: Tuple[str, str, str],
-    ) -> None:
-        return await self._invoke(self._client.destroy, mk_tbk(tbk))
-
-    async def list_symbols(
-        self,
-        tbk: str,
-    ) -> List[str]:
-        return await self._invoke(self._client.list_symbols, mk_tbk(tbk))
-
-    async def write(
-        self,
-        symbol: str,
-        array: np.ndarray,
-    ) -> None:
-        start = time.time()
-        await self._invoke(
-            self._client.write,
-            array,
-            _tick_tbk.format(symbol),
-            isvariablelength=True
-        )
-        log.debug(f"{symbol} write time (s): {time.time() - start}")
-
-    def query(
-        self,
-        symbol,
-        tbk: Tuple[str, str] = _tick_tbk_ids,
-    ) -> pd.DataFrame:
-        # XXX: causes crash
-        # client.query(pymkts.Params(symbol, '*', 'OHCLV'
-        result = self._client.query(
-            pymkts.Params(symbol, *tbk),
-        )
-        return result.first().df()
-
-
-@asynccontextmanager
+@acm
 async def get_client(
-    url: str = _url,
-) -> Client:
-    yield Client(url)
+    host: str = 'localhost',
+    port: int = 5995
+
+) -> MarketstoreClient:
+    '''
+    Load a ``anyio_marketstore`` grpc client connected
+    to an existing ``marketstore`` server.
+
+    '''
+    async with open_marketstore_client(
+        host,
+        port
+    ) as client:
+        yield client
+
+
+class MarketStoreError(Exception):
+    "Generic marketstore client error"
+
+
+# def err_on_resp(response: dict) -> None:
+#     """Raise any errors found in responses from client request.
+#     """
+#     responses = response['responses']
+#     if responses is not None:
+#         for r in responses:
+#             err = r['error']
+#             if err:
+#                 raise MarketStoreError(err)
+
+
+# map of seconds ints to "time frame" accepted keys
+tf_in_1s = bidict({
+    1: '1Sec',
+    60: '1Min',
+    60*5: '5Min',
+    60*15: '15Min',
+    60*30: '30Min',
+    60*60: '1H',
+    60*60*24: '1D',
+})
+
+
+class Storage:
+    '''
+    High level storage api for both real-time and historical ingest.
+
+    '''
+    def __init__(
+        self,
+        client: MarketstoreClient,
+
+    ) -> None:
+        # TODO: eventually this should be an api/interface type that
+        # ensures we can support multiple tsdb backends.
+        self.client = client
+
+        # series' cache from tsdb reads
+        self._arrays: dict[str, np.ndarray] = {}
+
+    async def list_keys(self) -> list[str]:
+        return await self.client.list_symbols()
+
+    async def search_keys(self, pattern: str) -> list[str]:
+        '''
+        Search for time series key in the storage backend.
+
+        '''
+        ...
+
+    async def write_ticks(self, ticks: list) -> None:
+        ...
+
+    async def load(
+        self,
+        fqsn: str,
+
+    ) -> tuple[
+        dict[int, np.ndarray],  # timeframe (in secs) to series
+        Optional[datetime],  # first dt
+        Optional[datetime],  # last dt
+    ]:
+
+        first_tsdb_dt, last_tsdb_dt = None, None
+        tsdb_arrays = await self.read_ohlcv(
+            fqsn,
+            # on first load we don't need to pull the max
+            # history per request size worth.
+            limit=3000,
+        )
+        log.info(f'Loaded tsdb history {tsdb_arrays}')
+
+        if tsdb_arrays:
+            fastest = list(tsdb_arrays.values())[0]
+            times = fastest['Epoch']
+            first, last = times[0], times[-1]
+            first_tsdb_dt, last_tsdb_dt = map(
+                pendulum.from_timestamp, [first, last]
+            )
+
+        return tsdb_arrays, first_tsdb_dt, last_tsdb_dt
+
+    async def read_ohlcv(
+        self,
+        fqsn: str,
+        timeframe: Optional[Union[int, str]] = None,
+        end: Optional[int] = None,
+        limit: int = int(800e3),
+
+    ) -> tuple[
+        MarketstoreClient,
+        Union[dict, np.ndarray]
+    ]:
+        client = self.client
+        syms = await client.list_symbols()
+
+        if fqsn not in syms:
+            return {}
+
+        tfstr = tf_in_1s[1]
+
+        params = Params(
+            symbols=fqsn,
+            timeframe=tfstr,
+            attrgroup='OHLCV',
+            end=end,
+            # limit_from_start=True,
+
+            # TODO: figure the max limit here given the
+            # ``purepc`` msg size limit of purerpc: 33554432
+            limit=limit,
+        )
+
+        if timeframe is None:
+            log.info(f'starting {fqsn} tsdb granularity scan..')
+            # loop through and try to find highest granularity
+            for tfstr in tf_in_1s.values():
+                try:
+                    log.info(f'querying for {tfstr}@{fqsn}')
+                    params.set('timeframe', tfstr)
+                    result = await client.query(params)
+                    break
+
+                except purerpc.grpclib.exceptions.UnknownError:
+                    # XXX: this is already logged by the container and
+                    # thus shows up through `marketstored` logs relay.
+                    # log.warning(f'{tfstr}@{fqsn} not found')
+                    continue
+            else:
+                return {}
+
+        else:
+            result = await client.query(params)
+
+        # TODO: it turns out column access on recarrays is actually slower:
+        # https://jakevdp.github.io/PythonDataScienceHandbook/02.09-structured-data-numpy.html#RecordArrays:-Structured-Arrays-with-a-Twist
+        # it might make sense to make these structured arrays?
+        # Fill out a `numpy` array-results map
+        arrays = {}
+        for fqsn, data_set in result.by_symbols().items():
+            arrays.setdefault(fqsn, {})[
+                tf_in_1s.inverse[data_set.timeframe]
+            ] = data_set.array
+
+        return arrays[fqsn][timeframe] if timeframe else arrays[fqsn]
+
+    async def delete_ts(
+        self,
+        key: str,
+        timeframe: Optional[Union[int, str]] = None,
+
+    ) -> bool:
+
+        client = self.client
+        syms = await client.list_symbols()
+        print(syms)
+        # if key not in syms:
+        #     raise KeyError(f'`{fqsn}` table key not found?')
+
+        return await client.destroy(tbk=key)
+
+    async def write_ohlcv(
+        self,
+        fqsn: str,
+        ohlcv: np.ndarray,
+        append_and_duplicate: bool = True,
+        limit: int = int(800e3),
+
+    ) -> None:
+        # build mkts schema compat array for writing
+        mkts_dt = np.dtype(_ohlcv_dt)
+        mkts_array = np.zeros(
+            len(ohlcv),
+            dtype=mkts_dt,
+        )
+        # copy from shm array (yes it's this easy):
+        # https://numpy.org/doc/stable/user/basics.rec.html#assignment-from-other-structured-arrays
+        mkts_array[:] = ohlcv[[
+            'time',
+            'open',
+            'high',
+            'low',
+            'close',
+            'volume',
+        ]]
+
+        m, r = divmod(len(mkts_array), limit)
+
+        for i in range(m, 1):
+            to_push = mkts_array[i-1:i*limit]
+
+            # write to db
+            resp = await self.client.write(
+                to_push,
+                tbk=f'{fqsn}/1Sec/OHLCV',
+
+                # NOTE: will will append duplicates
+                # for the same timestamp-index.
+                # TODO: pre deduplicate?
+                isvariablelength=append_and_duplicate,
+            )
+
+            log.info(
+                f'Wrote {mkts_array.size} datums to tsdb\n'
+            )
+
+            for resp in resp.responses:
+                err = resp.error
+                if err:
+                    raise MarketStoreError(err)
+
+        if r:
+            to_push = mkts_array[m*limit:]
+
+            # write to db
+            resp = await self.client.write(
+                to_push,
+                tbk=f'{fqsn}/1Sec/OHLCV',
+
+                # NOTE: will will append duplicates
+                # for the same timestamp-index.
+                # TODO: pre deduplicate?
+                isvariablelength=append_and_duplicate,
+            )
+
+            log.info(
+                f'Wrote {mkts_array.size} datums to tsdb\n'
+            )
+
+            for resp in resp.responses:
+                err = resp.error
+                if err:
+                    raise MarketStoreError(err)
+
+    # XXX: currently the only way to do this is through the CLI:
+
+    # sudo ./marketstore connect --dir ~/.config/piker/data
+    # >> \show mnq.globex.20220617.ib/1Sec/OHLCV 2022-05-15
+    # and this seems to block and use up mem..
+    # >> \trim mnq.globex.20220617.ib/1Sec/OHLCV 2022-05-15
+
+    # relevant source code for this is here:
+    # https://github.com/alpacahq/marketstore/blob/master/cmd/connect/session/trim.go#L14
+    # def delete_range(self, start_dt, end_dt) -> None:
+    #     ...
+
+@acm
+async def open_storage_client(
+    fqsn: str,
+    period: Optional[Union[int, str]] = None,  # in seconds
+
+) -> tuple[Storage, dict[str, np.ndarray]]:
+    '''
+    Load a series by key and deliver in ``numpy`` struct array format.
+
+    '''
+    async with (
+        # eventually a storage backend endpoint
+        get_client() as client,
+    ):
+        # slap on our wrapper api
+        yield Storage(client)
+
+
+async def tsdb_history_update(
+    fqsn: Optional[str] = None,
+
+) -> list[str]:
+
+    # TODO: real-time dedicated task for ensuring
+    # history consistency between the tsdb, shm and real-time feed..
+
+    # update sequence design notes:
+
+    # - load existing highest frequency data from mkts
+    #   * how do we want to offer this to the UI?
+    #    - lazy loading?
+    #    - try to load it all and expect graphics caching/diffing
+    #      to  hide extra bits that aren't in view?
+
+    # - compute the diff between latest data from broker and shm
+    #   * use sql api in mkts to determine where the backend should
+    #     start querying for data?
+    #   * append any diff with new shm length
+    #   * determine missing (gapped) history by scanning
+    #   * how far back do we look?
+
+    # - begin rt update ingest and aggregation
+    #   * could start by always writing ticks to mkts instead of
+    #     worrying about a shm queue for now.
+    #   * we have a short list of shm queues worth groking:
+    #     - https://github.com/pikers/piker/issues/107
+    #   * the original data feed arch blurb:
+    #     - https://github.com/pikers/piker/issues/98
+    #
+    profiler = pg.debug.Profiler(
+        disabled=False,  # not pg_profile_enabled(),
+        delayed=False,
+    )
+
+    async with (
+        open_storage_client(fqsn) as storage,
+
+        maybe_open_feed(
+            [fqsn],
+            start_stream=False,
+
+        ) as (feed, stream),
+    ):
+        profiler(f'opened feed for {fqsn}')
+
+        to_append = feed.shm.array
+        to_prepend = None
+
+        if fqsn:
+            symbol = feed.symbols.get(fqsn)
+            if symbol:
+                fqsn = symbol.front_fqsn()
+
+            # diff db history with shm and only write the missing portions
+            ohlcv = feed.shm.array
+
+            # TODO: use pg profiler
+            tsdb_arrays = await storage.read_ohlcv(fqsn)
+            # hist diffing
+            if tsdb_arrays:
+                for secs in (1, 60):
+                    ts = tsdb_arrays.get(secs)
+                    if ts is not None and len(ts):
+                        # these aren't currently used but can be referenced from
+                        # within the embedded ipython shell below.
+                        to_append = ohlcv[ohlcv['time'] > ts['Epoch'][-1]]
+                        to_prepend = ohlcv[ohlcv['time'] < ts['Epoch'][0]]
+
+            profiler('Finished db arrays diffs')
+
+        syms = await storage.client.list_symbols()
+        log.info(f'Existing tsdb symbol set:\n{pformat(syms)}')
+        profiler(f'listed symbols {syms}')
+
+        # TODO: ask if user wants to write history for detected
+        # available shm buffers?
+        from tractor.trionics import ipython_embed
+        await ipython_embed()
+
+        # for array in [to_append, to_prepend]:
+        #     if array is None:
+        #         continue
+
+        #     log.info(
+        #         f'Writing datums {array.size} -> to tsdb from shm\n'
+        #     )
+        #     await storage.write_ohlcv(fqsn, array)
+
+        # profiler('Finished db writes')
 
 
 async def ingest_quote_stream(
-    symbols: List[str],
+    symbols: list[str],
     brokername: str,
     tries: int = 1,
     loglevel: str = None,
+
 ) -> None:
-    """Ingest a broker quote stream into marketstore in (sampled) tick format.
-    """
-    async with open_feed(
-        brokername,
-        symbols,
-        loglevel=loglevel,
-    ) as (first_quotes, qstream):
+    '''
+    Ingest a broker quote stream into a ``marketstore`` tsdb.
 
-        quote_cache = first_quotes.copy()
+    '''
+    async with (
+        maybe_open_feed(brokername, symbols, loglevel=loglevel) as feed,
+        get_client() as ms_client,
+    ):
+        async for quotes in feed.stream:
+            log.info(quotes)
+            for symbol, quote in quotes.items():
+                for tick in quote.get('ticks', ()):
+                    ticktype = tick.get('type', 'n/a')
 
-        async with get_client() as ms_client:
+            # techtonic tick write
+            array = quote_to_marketstore_structarray({
+                'IsTrade': 1 if ticktype == 'trade' else 0,
+                'IsBid': 1 if ticktype in ('bid', 'bsize') else 0,
+                'Price': tick.get('price'),
+                'Size': tick.get('size')
+            }, last_fill=quote.get('broker_ts', None))
 
-            # start ingest to marketstore
-            async for quotes in qstream:
-                log.info(quotes)
-                for symbol, quote in quotes.items():
+            await ms_client.write(array, _tick_tbk)
 
-                    # remap tick strs to ints
-                    quote['tick'] = _tick_map[quote.get('tick', 'Equal')]
+            # LEGACY WRITE LOOP (using old tick dt)
+            # quote_cache = {
+            #     'size': 0,
+            #     'tick': 0
+            # }
 
-                    # check for volume update (i.e. did trades happen
-                    # since last quote)
-                    new_vol = quote.get('volume', None)
-                    if new_vol is None:
-                        log.debug(f"No fills for {symbol}")
-                        if new_vol == quote_cache.get('volume'):
-                            # should never happen due to field diffing
-                            # on sender side
-                            log.error(
-                                f"{symbol}: got same volume as last quote?")
+            # async for quotes in qstream:
+            #     log.info(quotes)
+            #     for symbol, quote in quotes.items():
 
-                    quote_cache.update(quote)
+            #         # remap tick strs to ints
+            #         quote['tick'] = _tick_map[quote.get('tick', 'Equal')]
 
-                    a = quote_to_marketstore_structarray(
-                        quote,
-                        # TODO: check this closer to the broker query api
-                        last_fill=quote.get('fill_time', '')
-                    )
-                    await ms_client.write(symbol, a)
+            #         # check for volume update (i.e. did trades happen
+            #         # since last quote)
+            #         new_vol = quote.get('volume', None)
+            #         if new_vol is None:
+            #             log.debug(f"No fills for {symbol}")
+            #             if new_vol == quote_cache.get('volume'):
+            #                 # should never happen due to field diffing
+            #                 # on sender side
+            #                 log.error(
+            #                     f"{symbol}: got same volume as last quote?")
+
+            #         quote_cache.update(quote)
+
+            #         a = quote_to_marketstore_structarray(
+            #             quote,
+            #             # TODO: check this closer to the broker query api
+            #             last_fill=quote.get('fill_time', '')
+            #         )
+            #         await ms_client.write(symbol, a)
 
 
 async def stream_quotes(
-    symbols: List[str],
+    symbols: list[str],
     host: str = 'localhost',
     port: int = 5993,
     diff_cached: bool = True,
     loglevel: str = None,
+
 ) -> None:
-    """Open a symbol stream from a running instance of marketstore and
+    '''
+    Open a symbol stream from a running instance of marketstore and
     log to console.
-    """
+
+    '''
     # XXX: required to propagate ``tractor`` loglevel to piker logging
     get_console_log(loglevel or tractor.current_actor().loglevel)
 
-    tbks: Dict[str, str] = {sym: f"{sym}/*/*" for sym in symbols}
+    tbks: dict[str, str] = {sym: f"{sym}/*/*" for sym in symbols}
 
     async with open_websocket_url(f'ws://{host}:{port}/ws') as ws:
         # send subs topics to server
@@ -271,7 +778,7 @@ async def stream_quotes(
         )
         log.info(resp)
 
-        async def recv() -> Dict[str, Any]:
+        async def recv() -> dict[str, Any]:
             return msgpack.loads((await ws.get_message()), encoding='utf-8')
 
         streams = (await recv())['streams']

piker/fsp/__init__.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+# Copyright (C) Tyler Goodlet (in stewardship of piker0)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,33 +14,17 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-"""
-Financial signal processing for the peeps.
-"""
-from functools import partial
-from typing import AsyncIterator, Callable, Tuple
+'''
+Fin-sig-proc for the peeps!
+
+'''
+from typing import AsyncIterator
 
-import trio
-from trio_typing import TaskStatus
-import tractor
 import numpy as np
 
-from ..log import get_logger
-from .. import data
-from ._momo import _rsi, _wma
-from ._volume import _tina_vwap
-from ..data import attach_shm_array
-from ..data.feed import Feed
-from ..data._sharedmem import ShmArray
+from ._engine import cascade
 
-log = get_logger(__name__)
-
-
-_fsps = {
-    'rsi': _rsi,
-    'wma': _wma,
-    'vwap': _tina_vwap,
-}
+__all__ = ['cascade']
 
 
 async def latency(
@@ -63,163 +47,3 @@ async def latency(
             # stack tracing.
             value = quote['brokerd_ts'] - quote['broker_ts']
             yield value
-
-
-async def fsp_compute(
-    ctx: tractor.Context,
-    symbol: str,
-    feed: Feed,
-
-    src: ShmArray,
-    dst: ShmArray,
-
-    fsp_func_name: str,
-    func: Callable,
-
-    task_status: TaskStatus[None] = trio.TASK_STATUS_IGNORED,
-
-) -> None:
-
-    # TODO: load appropriate fsp with input args
-
-    async def filter_by_sym(
-        sym: str,
-        stream,
-    ):
-
-        # TODO: make this the actualy first quote from feed
-        # XXX: this allows for a single iteration to run for history
-        # processing without waiting on the real-time feed for a new quote
-        yield {}
-
-        # task cancellation won't kill the channel
-        with stream.shield():
-            async for quotes in stream:
-                for symbol, quotes in quotes.items():
-                    if symbol == sym:
-                        yield quotes
-
-    out_stream = func(
-        filter_by_sym(symbol, feed.stream),
-        feed.shm,
-    )
-
-    # TODO: XXX:
-    # THERE'S A BIG BUG HERE WITH THE `index` field since we're
-    # prepending a copy of the first value a few times to make
-    # sub-curves align with the parent bar chart.
-    # This likely needs to be fixed either by,
-    # - manually assigning the index and historical data
-    #   seperately to the shm array (i.e. not using .push())
-    # - developing some system on top of the shared mem array that
-    #   is `index` aware such that historical data can be indexed
-    #   relative to the true first datum? Not sure if this is sane
-    #   for incremental compuations.
-    dst._first.value = src._first.value
-    dst._last.value = src._first.value
-
-    # Conduct a single iteration of fsp with historical bars input
-    # and get historical output
-    history_output = await out_stream.__anext__()
-
-    # build a struct array which includes an 'index' field to push
-    # as history
-    history = np.array(
-        np.arange(len(history_output)),
-        dtype=dst.array.dtype
-    )
-    history[fsp_func_name] = history_output
-
-    # check for data length mis-allignment and fill missing values
-    diff = len(src.array) - len(history)
-    if diff >= 0:
-        print(f"WTF DIFF SIGNAL to HISTORY {diff}")
-        for _ in range(diff):
-            dst.push(history[:1])
-
-    # compare with source signal and time align
-    index = dst.push(history)
-
-    await ctx.send_yield(index)
-
-    # setup a respawn handle
-    with trio.CancelScope() as cs:
-        task_status.started(cs)
-
-        # rt stream
-        async for processed in out_stream:
-            log.debug(f"{fsp_func_name}: {processed}")
-            index = src.index
-            dst.array[-1][fsp_func_name] = processed
-
-            # stream latest shm array index entry
-            await ctx.send_yield(index)
-
-
-@tractor.stream
-async def cascade(
-    ctx: tractor.Context,
-    brokername: str,
-    src_shm_token: dict,
-    dst_shm_token: Tuple[str, np.dtype],
-    symbol: str,
-    fsp_func_name: str,
-) -> AsyncIterator[dict]:
-    """Chain streaming signal processors and deliver output to
-    destination mem buf.
-
-    """
-    src = attach_shm_array(token=src_shm_token)
-    dst = attach_shm_array(readonly=False, token=dst_shm_token)
-
-    func: Callable = _fsps[fsp_func_name]
-
-    # open a data feed stream with requested broker
-    async with data.open_feed(brokername, [symbol]) as feed:
-
-        assert src.token == feed.shm.token
-
-        last_len = new_len = len(src.array)
-
-        fsp_target = partial(
-            fsp_compute,
-            ctx=ctx,
-            symbol=symbol,
-            feed=feed,
-
-            src=src,
-            dst=dst,
-
-            fsp_func_name=fsp_func_name,
-            func=func
-        )
-
-        async with trio.open_nursery() as n:
-
-            cs = await n.start(fsp_target)
-
-            # Increment the underlying shared memory buffer on every
-            # "increment" msg received from the underlying data feed.
-
-            async with feed.index_stream() as stream:
-                async for msg in stream:
-
-                    new_len = len(src.array)
-
-                    if new_len > last_len + 1:
-                        # respawn the signal compute task if the source
-                        # signal has been updated
-                        cs.cancel()
-                        cs = await n.start(fsp_target)
-
-                        # TODO: adopt an incremental update engine/approach
-                        # where possible here eventually!
-
-                    # read out last shm row
-                    array = dst.array
-                    last = array[-1:].copy()
-
-                    # write new row to the shm buffer
-                    dst.push(last)
-
-                    last_len = new_len

piker/fsp/_api.py

@@ -0,0 +1,199 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship of pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+FSP (financial signal processing) apis.
+
+'''
+
+# TODO: things to figure the heck out:
+# - how to handle non-plottable values (pyqtgraph has facility for this
+#   now in `arrayToQPath()`)
+# - composition of fsps / implicit chaining syntax (we need an issue)
+
+from __future__ import annotations
+from functools import partial
+from typing import (
+    Any,
+    Callable,
+    Awaitable,
+    Optional,
+)
+
+import numpy as np
+import tractor
+from tractor.msg import NamespacePath
+
+from ..data._sharedmem import (
+    ShmArray,
+    maybe_open_shm_array,
+    attach_shm_array,
+    _Token,
+)
+from ..log import get_logger
+
+log = get_logger(__name__)
+
+# global fsp registry filled out by @fsp decorator below
+_fsp_registry = {}
+
+
+def _load_builtins() -> dict[tuple, Callable]:
+
+    # import to implicity trigger registration via ``@fsp``
+    from . import _momo  # noqa
+    from . import _volume  # noqa
+
+    return _fsp_registry
+
+
+class Fsp:
+    '''
+    "Financial signal processor" decorator wrapped async function.
+
+    '''
+
+    # TODO: checkout the advanced features from ``wrapt``:
+    # - dynamic enable toggling,
+    # https://wrapt.readthedocs.io/en/latest/decorators.html#dynamically-disabling-decorators
+    # - custom object proxies, might be useful for implementing n-compose
+    # https://wrapt.readthedocs.io/en/latest/wrappers.html#custom-object-proxies
+    # - custom function wrappers,
+    # https://wrapt.readthedocs.io/en/latest/wrappers.html#custom-function-wrappers
+
+    # actor-local map of source flow shm tokens
+    # + the consuming fsp *to* the consumers output
+    # shm flow.
+    _flow_registry: dict[
+        tuple[_Token, str], _Token,
+    ] = {}
+
+    def __init__(
+        self,
+        func: Callable[..., Awaitable],
+        *,
+        outputs: tuple[str] = (),
+        display_name: Optional[str] = None,
+        **config,
+
+    ) -> None:
+
+        # TODO (maybe):
+        # - type introspection?
+        # - should we make this a wrapt object proxy?
+        self.func = func
+        self.__name__ = func.__name__  # XXX: must have func-object name
+
+        self.ns_path: tuple[str, str] = NamespacePath.from_ref(func)
+        self.outputs = outputs
+        self.config: dict[str, Any] = config
+
+        # register with declared set.
+        _fsp_registry[self.ns_path] = self
+
+    @property
+    def name(self) -> str:
+        return self.__name__
+
+    def __call__(
+        self,
+
+        # TODO: when we settle on py3.10 we should probably use the new
+        # type annots from pep 612:
+        # https://www.python.org/dev/peps/pep-0612/
+        # instance,
+        *args,
+        **kwargs
+    ):
+        return self.func(*args, **kwargs)
+
+    # TODO: lru_cache this? prettty sure it'll work?
+    def get_shm(
+        self,
+        src_shm: ShmArray,
+
+    ) -> ShmArray:
+        '''
+        Provide access to allocated shared mem array
+        for this "instance" of a signal processor for
+        the given ``key``.
+
+        '''
+        dst_token = self._flow_registry[
+            (src_shm._token, self.name)
+        ]
+        shm = attach_shm_array(dst_token)
+        return shm
+
+
+def fsp(
+    wrapped=None,
+    *,
+    outputs: tuple[str] = (),
+    display_name: Optional[str] = None,
+    **config,
+
+) -> Fsp:
+
+    if wrapped is None:
+        return partial(
+            Fsp,
+            outputs=outputs,
+            display_name=display_name,
+            **config,
+        )
+
+    return Fsp(wrapped, outputs=(wrapped.__name__,))
+
+
+def mk_fsp_shm_key(
+    sym: str,
+    target: Fsp
+
+) -> str:
+    uid = tractor.current_actor().uid
+    return f'{sym}.fsp.{target.name}.{".".join(uid)}'
+
+
+def maybe_mk_fsp_shm(
+    sym: str,
+    target: Fsp,
+    readonly: bool = True,
+
+) -> (str, ShmArray, bool):
+    '''
+    Allocate a single row shm array for an symbol-fsp pair if none
+    exists, otherwise load the shm already existing for that token.
+
+    '''
+    assert isinstance(sym, str), '`sym` should be file-name-friendly `str`'
+
+    # TODO: load output types from `Fsp`
+    # - should `index` be a required internal field?
+    fsp_dtype = np.dtype(
+        [('index', int)] +
+        [(field_name, float) for field_name in target.outputs]
+    )
+
+    key = mk_fsp_shm_key(sym, target)
+
+    shm, opened = maybe_open_shm_array(
+        key,
+        # TODO: create entry for each time frame
+        dtype=fsp_dtype,
+        readonly=True,
+    )
+    return key, shm, opened

piker/fsp/_engine.py

@@ -0,0 +1,460 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship of pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+core task logic for processing chains
+
+'''
+from dataclasses import dataclass
+from functools import partial
+from typing import (
+    AsyncIterator, Callable, Optional,
+    Union,
+)
+
+import numpy as np
+import pyqtgraph as pg
+import trio
+from trio_typing import TaskStatus
+import tractor
+from tractor.msg import NamespacePath
+
+from ..log import get_logger, get_console_log
+from .. import data
+from ..data import attach_shm_array
+from ..data.feed import Feed
+from ..data._sharedmem import ShmArray
+from ..data._source import Symbol
+from ._api import (
+    Fsp,
+    _load_builtins,
+    _Token,
+)
+
+log = get_logger(__name__)
+
+
+@dataclass
+class TaskTracker:
+    complete: trio.Event
+    cs: trio.CancelScope
+
+
+async def filter_quotes_by_sym(
+
+    sym: str,
+    quote_stream: tractor.MsgStream,
+
+) -> AsyncIterator[dict]:
+    '''
+    Filter quote stream by target symbol.
+
+    '''
+    # TODO: make this the actual first quote from feed
+    # XXX: this allows for a single iteration to run for history
+    # processing without waiting on the real-time feed for a new quote
+    yield {}
+
+    async for quotes in quote_stream:
+        quote = quotes.get(sym)
+        if quote:
+            yield quote
+
+
+async def fsp_compute(
+
+    symbol: Symbol,
+    feed: Feed,
+    quote_stream: trio.abc.ReceiveChannel,
+
+    src: ShmArray,
+    dst: ShmArray,
+
+    func: Callable,
+
+    # attach_stream: bool = False,
+    task_status: TaskStatus[None] = trio.TASK_STATUS_IGNORED,
+
+) -> None:
+
+    profiler = pg.debug.Profiler(
+        delayed=False,
+        disabled=True
+    )
+
+    fqsn = symbol.front_fqsn()
+    out_stream = func(
+
+        # TODO: do we even need this if we do the feed api right?
+        # shouldn't a local stream do this before we get a handle
+        # to the async iterable? it's that or we do some kinda
+        # async itertools style?
+        filter_quotes_by_sym(fqsn, quote_stream),
+
+        # XXX: currently the ``ohlcv`` arg
+        feed.shm,
+    )
+
+    # Conduct a single iteration of fsp with historical bars input
+    # and get historical output
+    history_output: Union[
+        dict[str, np.ndarray],  # multi-output case
+        np.ndarray,  # single output case
+    ]
+    history_output = await out_stream.__anext__()
+
+    func_name = func.__name__
+    profiler(f'{func_name} generated history')
+
+    # build struct array with an 'index' field to push as history
+
+    # TODO: push using a[['f0', 'f1', .., 'fn']] = .. syntax no?
+    # if the output array is multi-field then push
+    # each respective field.
+    fields = getattr(dst.array.dtype, 'fields', None).copy()
+    fields.pop('index')
+    history: Optional[np.ndarray] = None  # TODO: nptyping here!
+
+    if fields and len(fields) > 1 and fields:
+        if not isinstance(history_output, dict):
+            raise ValueError(
+                f'`{func_name}` is a multi-output FSP and should yield a '
+                '`dict[str, np.ndarray]` for history'
+            )
+
+        for key in fields.keys():
+            if key in history_output:
+                output = history_output[key]
+
+                if history is None:
+
+                    if output is None:
+                        length = len(src.array)
+                    else:
+                        length = len(output)
+
+                    # using the first output, determine
+                    # the length of the struct-array that
+                    # will be pushed to shm.
+                    history = np.zeros(
+                        length,
+                        dtype=dst.array.dtype
+                    )
+
+                if output is None:
+                    continue
+
+                history[key] = output
+
+    # single-key output stream
+    else:
+        if not isinstance(history_output, np.ndarray):
+            raise ValueError(
+                f'`{func_name}` is a single output FSP and should yield an '
+                '`np.ndarray` for history'
+            )
+        history = np.zeros(
+            len(history_output),
+            dtype=dst.array.dtype
+        )
+        history[func_name] = history_output
+
+    # TODO: XXX:
+    # THERE'S A BIG BUG HERE WITH THE `index` field since we're
+    # prepending a copy of the first value a few times to make
+    # sub-curves align with the parent bar chart.
+    # This likely needs to be fixed either by,
+    # - manually assigning the index and historical data
+    #   seperately to the shm array (i.e. not using .push())
+    # - developing some system on top of the shared mem array that
+    #   is `index` aware such that historical data can be indexed
+    #   relative to the true first datum? Not sure if this is sane
+    #   for incremental compuations.
+    first = dst._first.value = src._first.value
+
+    # TODO: can we use this `start` flag instead of the manual
+    # setting above?
+    index = dst.push(history, start=first)
+
+    profiler(f'{func_name} pushed history')
+    profiler.finish()
+
+    # setup a respawn handle
+    with trio.CancelScope() as cs:
+
+        # TODO: might be better to just make a "restart" method where
+        # the target task is spawned implicitly and then the event is
+        # set via some higher level api? At that poing we might as well
+        # be writing a one-cancels-one nursery though right?
+        tracker = TaskTracker(trio.Event(), cs)
+        task_status.started((tracker, index))
+
+        profiler(f'{func_name} yield last index')
+
+        # import time
+        # last = time.time()
+
+        try:
+
+            async for processed in out_stream:
+
+                log.debug(f"{func_name}: {processed}")
+                key, output = processed
+                index = src.index
+                dst.array[-1][key] = output
+
+                # NOTE: for now we aren't streaming this to the consumer
+                # stream latest array index entry which basically just acts
+                # as trigger msg to tell the consumer to read from shm
+                # TODO: further this should likely be implemented much
+                # like our `Feed` api where there is one background
+                # "service" task which computes output and then sends to
+                # N-consumers who subscribe for the real-time output,
+                # which we'll likely want to implement using local-mem
+                # chans for the fan out?
+                # if attach_stream:
+                #     await client_stream.send(index)
+
+                # period = time.time() - last
+                # hz = 1/period if period else float('nan')
+                # if hz > 60:
+                #     log.info(f'FSP quote too fast: {hz}')
+                # last = time.time()
+        finally:
+            tracker.complete.set()
+
+
+@tractor.context
+async def cascade(
+
+    ctx: tractor.Context,
+
+    # data feed key
+    fqsn: str,
+
+    src_shm_token: dict,
+    dst_shm_token: tuple[str, np.dtype],
+
+    ns_path: NamespacePath,
+
+    shm_registry: dict[str, _Token],
+
+    zero_on_step: bool = False,
+    loglevel: Optional[str] = None,
+
+) -> None:
+    '''
+    Chain streaming signal processors and deliver output to
+    destination shm array buffer.
+
+    '''
+    profiler = pg.debug.Profiler(
+        delayed=False,
+        disabled=False
+    )
+
+    if loglevel:
+        get_console_log(loglevel)
+
+    src = attach_shm_array(token=src_shm_token)
+    dst = attach_shm_array(readonly=False, token=dst_shm_token)
+
+    reg = _load_builtins()
+    lines = '\n'.join([f'{key.rpartition(":")[2]} => {key}' for key in reg])
+    log.info(
+        f'Registered FSP set:\n{lines}'
+    )
+
+    # update actorlocal flows table which registers
+    # readonly "instances" of this fsp for symbol/source
+    # so that consumer fsps can look it up by source + fsp.
+    # TODO: ugh i hate this wind/unwind to list over the wire
+    # but not sure how else to do it.
+    for (token, fsp_name, dst_token) in shm_registry:
+        Fsp._flow_registry[
+            (_Token.from_msg(token), fsp_name)
+        ] = _Token.from_msg(dst_token)
+
+    fsp: Fsp = reg.get(
+        NamespacePath(ns_path)
+    )
+    func = fsp.func
+
+    if not func:
+        # TODO: assume it's a func target path
+        raise ValueError(f'Unknown fsp target: {ns_path}')
+
+    # open a data feed stream with requested broker
+    async with data.feed.maybe_open_feed(
+        [fqsn],
+
+        # TODO throttle tick outputs from *this* daemon since
+        # it'll emit tons of ticks due to the throttle only
+        # limits quote arrival periods, so the consumer of *this*
+        # needs to get throttled the ticks we generate.
+        # tick_throttle=60,
+
+    ) as (feed, quote_stream):
+        symbol = feed.symbols[fqsn]
+
+        profiler(f'{func}: feed up')
+
+        assert src.token == feed.shm.token
+        # last_len = new_len = len(src.array)
+
+        func_name = func.__name__
+        async with (
+            trio.open_nursery() as n,
+        ):
+
+            fsp_target = partial(
+
+                fsp_compute,
+                symbol=symbol,
+                feed=feed,
+                quote_stream=quote_stream,
+
+                # shm
+                src=src,
+                dst=dst,
+
+                # target
+                func=func
+            )
+
+            tracker, index = await n.start(fsp_target)
+
+            if zero_on_step:
+                last = dst.array[-1:]
+                zeroed = np.zeros(last.shape, dtype=last.dtype)
+
+            profiler(f'{func_name}: fsp up')
+
+            # sync client
+            await ctx.started(index)
+
+            # XXX:  rt stream with client which we MUST
+            # open here (and keep it open) in order to make
+            # incremental "updates" as history prepends take
+            # place.
+            async with ctx.open_stream() as client_stream:
+
+                # TODO: these likely should all become
+                # methods of this ``TaskLifetime`` or wtv
+                # abstraction..
+                async def resync(
+                    tracker: TaskTracker,
+
+                ) -> tuple[TaskTracker, int]:
+                    # TODO: adopt an incremental update engine/approach
+                    # where possible here eventually!
+                    log.debug(f're-syncing fsp {func_name} to source')
+                    tracker.cs.cancel()
+                    await tracker.complete.wait()
+                    tracker, index = await n.start(fsp_target)
+
+                    # always trigger UI refresh after history update,
+                    # see ``piker.ui._fsp.FspAdmin.open_chain()`` and
+                    # ``piker.ui._display.trigger_update()``.
+                    await client_stream.send({
+                        'fsp_update': {
+                            'key': dst_shm_token,
+                            'first': dst._first.value,
+                            'last': dst._last.value,
+                    }})
+                    return tracker, index
+
+                def is_synced(
+                    src: ShmArray,
+                    dst: ShmArray
+                ) -> tuple[bool, int, int]:
+                    '''Predicate to dertmine if a destination FSP
+                    output array is aligned to its source array.
+
+                    '''
+                    step_diff = src.index - dst.index
+                    len_diff = abs(len(src.array) - len(dst.array))
+                    return not (
+                        # the source is likely backfilling and we must
+                        # sync history calculations
+                        len_diff > 2 or
+
+                        # we aren't step synced to the source and may be
+                        # leading/lagging by a step
+                        step_diff > 1 or
+                        step_diff < 0
+                    ), step_diff, len_diff
+
+                async def poll_and_sync_to_step(
+
+                    tracker: TaskTracker,
+                    src: ShmArray,
+                    dst: ShmArray,
+
+                ) -> tuple[TaskTracker, int]:
+
+                    synced, step_diff, _ = is_synced(src, dst)
+                    while not synced:
+                        tracker, index = await resync(tracker)
+                        synced, step_diff, _ = is_synced(src, dst)
+
+                    return tracker, step_diff
+
+                s, step, ld = is_synced(src, dst)
+
+                # detect sample period step for subscription to increment
+                # signal
+                times = src.array['time']
+                delay_s = times[-1] - times[times != times[-1]][-1]
+
+                # Increment the underlying shared memory buffer on every
+                # "increment" msg received from the underlying data feed.
+                async with feed.index_stream(
+                    int(delay_s)
+                ) as istream:
+
+                    profiler(f'{func_name}: sample stream up')
+                    profiler.finish()
+
+                    async for _ in istream:
+
+                        # respawn the compute task if the source
+                        # array has been updated such that we compute
+                        # new history from the (prepended) source.
+                        synced, step_diff, _ = is_synced(src, dst)
+                        if not synced:
+                            tracker, step_diff = await poll_and_sync_to_step(
+                                tracker,
+                                src,
+                                dst,
+                            )
+
+                            # skip adding a last bar since we should already
+                            # be step alinged
+                            if step_diff == 0:
+                                continue
+
+                        # read out last shm row, copy and write new row
+                        array = dst.array
+
+                        # some metrics like vlm should be reset
+                        # to zero every step.
+                        if zero_on_step:
+                            last = zeroed
+                        else:
+                            last = array[-1:].copy()
+
+                        dst.push(last)

piker/fsp/_momo.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+# Copyright (C) Tyler Goodlet (in stewardship of pikers)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -16,19 +16,18 @@
 
 """
 Momentum bby.
+
 """
 from typing import AsyncIterator, Optional
 
 import numpy as np
 from numba import jit, float64, optional, int64
 
+from ._api import fsp
 from ..data._normalize import iterticks
+from ..data._sharedmem import ShmArray
 
 
-# TODO: things to figure the fuck out:
-# - how to handle non-plottable values
-# - composition of fsps / implicit chaining
-
 @jit(
     float64[:](
         float64[:],
@@ -39,11 +38,14 @@ from ..data._normalize import iterticks
     nogil=True
 )
 def ema(
+
     y: 'np.ndarray[float64]',
     alpha: optional(float64) = None,
     ylast: optional(float64) = None,
+
 ) -> 'np.ndarray[float64]':
-    r"""Exponential weighted moving average owka 'Exponential smoothing'.
+    r'''
+    Exponential weighted moving average owka 'Exponential smoothing'.
 
     - https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
     - https://en.wikipedia.org/wiki/Exponential_smoothing
@@ -68,7 +70,8 @@ def ema(
 
     More discussion here:
     https://stackoverflow.com/questions/42869495/numpy-version-of-exponential-weighted-moving-average-equivalent-to-pandas-ewm
-    """
+
+    '''
     n = y.shape[0]
 
     if alpha is None:
@@ -104,15 +107,22 @@ def ema(
 #     nopython=True,
 #     nogil=True
 # )
-def rsi(
+def _rsi(
+
+    # TODO: use https://github.com/ramonhagenaars/nptyping
     signal: 'np.ndarray[float64]',
     period: int64 = 14,
     up_ema_last: float64 = None,
     down_ema_last: float64 = None,
+
 ) -> 'np.ndarray[float64]':
+    '''
+    relative strengggth.
+
+    '''
     alpha = 1/period
 
-    df = np.diff(signal)
+    df = np.diff(signal, prepend=0)
 
     up = np.where(df > 0, df, 0)
     up_ema = ema(up, alpha, up_ema_last)
@@ -120,11 +130,12 @@ def rsi(
     down = np.where(df < 0, -df, 0)
     down_ema = ema(down, alpha, down_ema_last)
 
-    # avoid dbz errors
+    # avoid dbz errors, this leaves the first
+    # index == 0 right?
     rs = np.divide(
         up_ema,
         down_ema,
-        out=np.zeros_like(up_ema),
+        out=np.zeros_like(signal),
         where=down_ema != 0
     )
 
@@ -136,11 +147,19 @@ def rsi(
     return rsi, up_ema[-1], down_ema[-1]
 
 
-def wma(
+def _wma(
+
     signal: np.ndarray,
     length: int,
     weights: Optional[np.ndarray] = None,
+
 ) -> np.ndarray:
+    '''
+    Compute a windowed moving average of ``signal`` with window
+    ``length`` and optional ``weights`` (must be same size as
+    ``signal``).
+
+    '''
     if weights is None:
         # default is a standard arithmetic mean
         seq = np.full((length,), 1)
@@ -148,29 +167,59 @@ def wma(
 
     assert length == len(weights)
 
+    # lol, for long sequences this is nutso slow and expensive..
     return np.convolve(signal, weights, 'valid')
 
 
-# @piker.fsp.signal(
-#     timeframes=['1s', '5s', '15s', '1m', '5m', '1H'],
-# )
-async def _rsi(
+@fsp
+async def wma(
+
+    source,  #: AsyncStream[np.ndarray],
+    length: int,
+    ohlcv: np.ndarray,  # price time-frame "aware"
+
+) -> AsyncIterator[np.ndarray]:  # maybe something like like FspStream?
+    '''
+    Streaming weighted moving average.
+
+    ``weights`` is a sequence of already scaled values. As an example
+    for the WMA often found in "techincal analysis":
+    ``weights = np.arange(1, N) * N*(N-1)/2``.
+
+    '''
+    # deliver historical output as "first yield"
+    yield _wma(ohlcv.array['close'], length)
+
+    # begin real-time section
+
+    async for quote in source:
+        for tick in iterticks(quote, type='trade'):
+            yield _wma(ohlcv.last(length))
+
+
+@fsp
+async def rsi(
+
     source: 'QuoteStream[Dict[str, Any]]',  # noqa
-    ohlcv: "ShmArray[T<'close'>]",
+    ohlcv: ShmArray,
     period: int = 14,
+
 ) -> AsyncIterator[np.ndarray]:
-    """Multi-timeframe streaming RSI.
+    '''
+    Multi-timeframe streaming RSI.
 
     https://en.wikipedia.org/wiki/Relative_strength_index
-    """
+
+    '''
     sig = ohlcv.array['close']
 
     # wilder says to seed the RSI EMAs with the SMA for the "period"
-    seed = wma(ohlcv.last(period)['close'], period)[0]
+    seed = _wma(ohlcv.last(period)['close'], period)[0]
 
     # TODO: the emas here should be seeded with a period SMA as per
     # wilder's original formula..
-    rsi_h, last_up_ema_close, last_down_ema_close = rsi(sig, period, seed, seed)
+    rsi_h, last_up_ema_close, last_down_ema_close = _rsi(
+        sig, period, seed, seed)
     up_ema_last = last_up_ema_close
     down_ema_last = last_down_ema_close
 
@@ -178,7 +227,6 @@ async def _rsi(
     yield rsi_h
 
     index = ohlcv.index
-
     async for quote in source:
         # tick based updates
         for tick in iterticks(quote):
@@ -196,31 +244,10 @@ async def _rsi(
                 last_down_ema_close = down_ema_last
                 index = ohlcv.index
 
-            rsi_out, up_ema_last, down_ema_last = rsi(
+            rsi_out, up_ema_last, down_ema_last = _rsi(
                 sig,
                 period=period,
                 up_ema_last=last_up_ema_close,
                 down_ema_last=last_down_ema_close,
             )
             yield rsi_out[-1:]
-
-
-async def _wma(
-    source,  #: AsyncStream[np.ndarray],
-    length: int,
-    ohlcv: np.ndarray,  # price time-frame "aware"
-) -> AsyncIterator[np.ndarray]:  # maybe something like like FspStream?
-    """Streaming weighted moving average.
-
-    ``weights`` is a sequence of already scaled values. As an example
-    for the WMA often found in "techincal analysis":
-    ``weights = np.arange(1, N) * N*(N-1)/2``.
-    """
-    # deliver historical output as "first yield"
-    yield wma(ohlcv.array['close'], length)
-
-    # begin real-time section
-
-    async for quote in source:
-        for tick in iterticks(quote, type='trade'):
-            yield wma(ohlcv.last(length))

piker/fsp/_volume.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+# Copyright (C) Tyler Goodlet (in stewardship of pikers)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,20 +14,33 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from typing import AsyncIterator, Optional
+from typing import AsyncIterator, Optional, Union
 
 import numpy as np
+from tractor.trionics._broadcast import AsyncReceiver
 
+from ._api import fsp
 from ..data._normalize import iterticks
+from ..data._sharedmem import ShmArray
+from ._momo import _wma
+from ..log import get_logger
+
+log = get_logger(__name__)
 
 
+# NOTE: is the same as our `wma` fsp, and if so which one is faster?
+# Ohhh, this is an IIR style i think? So it has an anchor point
+# effectively instead of a moving window/FIR style?
 def wap(
+
     signal: np.ndarray,
     weights: np.ndarray,
-) -> np.ndarray:
-    """Weighted average price from signal and weights.
 
-    """
+) -> np.ndarray:
+    '''
+    Weighted average price from signal and weights.
+
+    '''
     cum_weights = np.cumsum(weights)
     cum_weighted_input = np.cumsum(signal * weights)
 
@@ -46,16 +59,25 @@ def wap(
     )
 
 
-async def _tina_vwap(
-    source,  #: AsyncStream[np.ndarray],
-    ohlcv: np.ndarray,  # price time-frame "aware"
+@fsp
+async def tina_vwap(
+
+    source: AsyncReceiver[dict],
+    ohlcv: ShmArray,  # OHLC sampled history
+
+    # TODO: anchor logic (eg. to session start)
     anchors: Optional[np.ndarray] = None,
-) -> AsyncIterator[np.ndarray]:  # maybe something like like FspStream?
-    """Streaming volume weighted moving average.
+
+) -> Union[
+    AsyncIterator[np.ndarray],
+    float
+]:
+    '''
+    Streaming volume weighted moving average.
 
     Calling this "tina" for now since we're using HLC3 instead of tick.
 
-    """
+    '''
     if anchors is None:
         # TODO:
         # anchor to session start of data if possible
@@ -75,8 +97,10 @@ async def _tina_vwap(
     # vwap_tot = h_vwap[-1]
 
     async for quote in source:
-
-        for tick in iterticks(quote, types=['trade']):
+        for tick in iterticks(
+            quote,
+            types=['trade'],
+        ):
 
             # c, h, l, v = ohlcv.array[-1][
             #     ['closes', 'high', 'low', 'volume']
@@ -90,4 +114,245 @@ async def _tina_vwap(
             w_tot += price * size
 
             # yield ((((o + h + l) / 3) * v) weights_tot) / v_tot
-            yield w_tot / v_tot
+            yield 'tina_vwap', w_tot / v_tot
+
+
+@fsp(
+    outputs=(
+        'dolla_vlm',
+        'dark_vlm',
+        'trade_count',
+        'dark_trade_count',
+    ),
+    curve_style='step',
+)
+async def dolla_vlm(
+    source: AsyncReceiver[dict],
+    ohlcv: ShmArray,  # OHLC sampled history
+
+) -> AsyncIterator[
+    tuple[str, Union[np.ndarray, float]],
+]:
+    '''
+    "Dollar Volume", aka the volume in asset-currency-units (usually
+    a fiat) computed from some price function for the sample step
+    *multiplied* (*) by the asset unit volume.
+
+    Useful for comparing cross asset "money flow" in #s that are
+    asset-currency-independent.
+
+    '''
+    a = ohlcv.array
+    chl3 = (a['close'] + a['high'] + a['low']) / 3
+    v = a['volume']
+
+    # on first iteration yield history
+    yield {
+        'dolla_vlm': chl3 * v,
+        'dark_vlm': None,
+    }
+
+    i = ohlcv.index
+    dvlm = vlm = 0
+    dark_trade_count = trade_count = 0
+
+    async for quote in source:
+        for tick in iterticks(
+            quote,
+            types=(
+                'trade',
+                'dark_trade',
+            ),
+            deduplicate_darks=True,
+        ):
+
+            # this computes tick-by-tick weightings from here forward
+            size = tick['size']
+            price = tick['price']
+
+            li = ohlcv.index
+            if li > i:
+                i = li
+                trade_count = dark_trade_count = dvlm = vlm = 0
+
+            # TODO: for marginned instruments (futes, etfs?) we need to
+            # show the margin $vlm by multiplying by whatever multiplier
+            # is reported in the sym info.
+
+            ttype = tick.get('type')
+
+            if ttype == 'dark_trade':
+                dvlm += price * size
+                yield 'dark_vlm', dvlm
+
+                dark_trade_count += 1
+                yield 'dark_trade_count', dark_trade_count
+
+                # print(f'{dark_trade_count}th dark_trade: {tick}')
+
+            else:
+                # print(f'vlm: {tick}')
+                vlm += price * size
+                yield 'dolla_vlm', vlm
+
+                trade_count += 1
+                yield 'trade_count', trade_count
+
+            # TODO: plot both to compare?
+            # c, h, l, v = ohlcv.last()[
+            #     ['close', 'high', 'low', 'volume']
+            # ][0]
+            # tina_lvlm = c+h+l/3 * v
+            # print(f' tinal vlm: {tina_lvlm}')
+
+
+@fsp(
+    # TODO: eventually I guess we should support some kinda declarative
+    # graphics config syntax per output yah? That seems like a clean way
+    # to let users configure things? Not sure how exactly to offer that
+    # api as well as how to expose such a thing *inside* the body?
+    outputs=(
+        # pulled verbatim from `ib` for now
+        '1m_trade_rate',
+        '1m_vlm_rate',
+
+        # our own instantaneous rate calcs which are all
+        # parameterized by a samples count (bars) period
+        'trade_rate',
+        'dark_trade_rate',
+
+        'dvlm_rate',
+        'dark_dvlm_rate',
+    ),
+    curve_style='line',
+)
+async def flow_rates(
+    source: AsyncReceiver[dict],
+    ohlcv: ShmArray,  # OHLC sampled history
+
+    # TODO (idea): a dynamic generic / boxing type that can be updated by other
+    # FSPs, user input, and possibly any general event stream in
+    # real-time. Hint: ideally implemented with caching until mutated
+    # ;)
+    period: 'Param[int]' = 6,  # noqa
+
+    # TODO: support other means by providing a map
+    # to weights `partial()`-ed with `wma()`?
+    mean_type: str = 'arithmetic',
+
+    # TODO (idea): a generic for declaring boxed fsps much like ``pytest``
+    # fixtures? This probably needs a lot of thought if we want to offer
+    # a higher level composition syntax eventually (oh right gotta make
+    # an issue for that).
+    # ideas for how to allow composition / intercalling:
+    # - offer a `Fsp.get_history()` to do the first yield output?
+    #  * err wait can we just have shm access directly?
+    # - how would it work if some consumer fsp wanted to dynamically
+    # change params which are input to the callee fsp? i guess we could
+    # lazy copy in that case?
+    # dvlm: 'Fsp[dolla_vlm]'
+
+) -> AsyncIterator[
+    tuple[str, Union[np.ndarray, float]],
+]:
+    # generally no history available prior to real-time calcs
+    yield {
+        # from ib
+        '1m_trade_rate': None,
+        '1m_vlm_rate': None,
+
+        'trade_rate': None,
+        'dark_trade_rate': None,
+
+        'dvlm_rate': None,
+        'dark_dvlm_rate': None,
+    }
+
+    # TODO: 3.10 do ``anext()``
+    quote = await source.__anext__()
+
+    # ltr = 0
+    # lvr = 0
+    tr = quote.get('tradeRate')
+    yield '1m_trade_rate', tr or 0
+    vr = quote.get('volumeRate')
+    yield '1m_vlm_rate', vr or 0
+
+    yield 'trade_rate', 0
+    yield 'dark_trade_rate', 0
+    yield 'dvlm_rate', 0
+    yield 'dark_dvlm_rate', 0
+
+    # NOTE: in theory we could dynamically allocate a cascade based on
+    # this call but not sure if that's too "dynamic" in terms of
+    # validating cascade flows from message typing perspective.
+
+    # attach to ``dolla_vlm`` fsp running
+    # on this same source flow.
+    dvlm_shm = dolla_vlm.get_shm(ohlcv)
+
+    # precompute arithmetic mean weights (all ones)
+    seq = np.full((period,), 1)
+    weights = seq / seq.sum()
+
+    async for quote in source:
+        if not quote:
+            log.error("OH WTF NO QUOTE IN FSP")
+            continue
+
+        # dvlm_wma = _wma(
+        #     dvlm_shm.array['dolla_vlm'],
+        #     period,
+        #     weights=weights,
+        # )
+        # yield 'dvlm_rate', dvlm_wma[-1]
+
+        if period > 1:
+            trade_rate_wma = _wma(
+                dvlm_shm.array['trade_count'][-period:],
+                period,
+                weights=weights,
+            )
+            trade_rate = trade_rate_wma[-1]
+            # print(trade_rate)
+            yield 'trade_rate', trade_rate
+        else:
+            # instantaneous rate per sample step
+            count = dvlm_shm.array['trade_count'][-1]
+            yield 'trade_rate', count
+
+        # TODO: skip this if no dark vlm is declared
+        # by symbol info (eg. in crypto$)
+        # dark_dvlm_wma = _wma(
+        #     dvlm_shm.array['dark_vlm'],
+        #     period,
+        #     weights=weights,
+        # )
+        # yield 'dark_dvlm_rate', dark_dvlm_wma[-1]
+
+        if period > 1:
+            dark_trade_rate_wma = _wma(
+                dvlm_shm.array['dark_trade_count'][-period:],
+                period,
+                weights=weights,
+            )
+            yield 'dark_trade_rate', dark_trade_rate_wma[-1]
+        else:
+            # instantaneous rate per sample step
+            dark_count = dvlm_shm.array['dark_trade_count'][-1]
+            yield 'dark_trade_rate', dark_count
+
+        # XXX: ib specific schema we should
+        # probably pre-pack ourselves.
+
+        # tr = quote.get('tradeRate')
+        # if tr is not None and tr != ltr:
+        #     # print(f'trade rate: {tr}')
+        #     yield '1m_trade_rate', tr
+        #     ltr = tr
+
+        # vr = quote.get('volumeRate')
+        # if vr is not None and vr != lvr:
+        #     # print(f'vlm rate: {vr}')
+        #     yield '1m_vlm_rate', vr
+        #     lvr = vr

piker/log.py

@@ -25,10 +25,13 @@ from pygments import highlight, lexers, formatters
 
 # Makes it so we only see the full module name when using ``__name__``
 # without the extra "piker." prefix.
-_proj_name = 'piker'
+_proj_name: str = 'piker'
 
 
-def get_logger(name: str = None) -> logging.Logger:
+def get_logger(
+    name: str = None,
+
+) -> logging.Logger:
     '''Return the package log or a sub-log for `name` if provided.
     '''
     return tractor.log.get_logger(name=name, _root_name=_proj_name)

piker/trionics.py

@@ -0,0 +1,80 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship of piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+sugarz for trio/tractor conc peeps.
+
+'''
+from typing import AsyncContextManager
+from typing import TypeVar
+from contextlib import asynccontextmanager as acm
+
+import trio
+
+
+# A regular invariant generic type
+T = TypeVar("T")
+
+
+async def _enter_and_sleep(
+
+    mngr: AsyncContextManager[T],
+    to_yield: dict[int, T],
+    all_entered: trio.Event,
+    # task_status: TaskStatus[T] = trio.TASK_STATUS_IGNORED,
+
+) -> T:
+    '''Open the async context manager deliver it's value
+    to this task's spawner and sleep until cancelled.
+
+    '''
+    async with mngr as value:
+        to_yield[id(mngr)] = value
+
+        if all(to_yield.values()):
+            all_entered.set()
+
+        # sleep until cancelled
+        await trio.sleep_forever()
+
+
+@acm
+async def async_enter_all(
+
+    *mngrs: list[AsyncContextManager[T]],
+
+) -> tuple[T]:
+
+    to_yield = {}.fromkeys(id(mngr) for mngr in mngrs)
+
+    all_entered = trio.Event()
+
+    async with trio.open_nursery() as n:
+        for mngr in mngrs:
+            n.start_soon(
+                _enter_and_sleep,
+                mngr,
+                to_yield,
+                all_entered,
+            )
+
+        # deliver control once all managers have started up
+        await all_entered.wait()
+        yield tuple(to_yield.values())
+
+        # tear down all sleeper tasks thus triggering individual
+        # mngr ``__aexit__()``s.
+        n.cancel_scope.cancel()

piker/ui/_anchors.py

@@ -0,0 +1,130 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Anchor funtions for UI placement of annotions.
+
+'''
+from __future__ import annotations
+from typing import Callable, TYPE_CHECKING
+
+from PyQt5.QtCore import QPointF
+from PyQt5.QtWidgets import QGraphicsPathItem
+
+if TYPE_CHECKING:
+    from ._chart import ChartPlotWidget
+    from ._label import Label
+
+
+def vbr_left(
+    label: Label,
+
+) -> Callable[..., float]:
+    '''
+    Return a closure which gives the scene x-coordinate for the leftmost
+    point of the containing view box.
+
+    '''
+    return label.vbr().left
+
+
+def right_axis(
+
+    chart: ChartPlotWidget,  # noqa
+    label: Label,
+
+    side: str = 'left',
+    offset: float = 10,
+    avoid_book: bool = True,
+    # width: float = None,
+
+) -> Callable[..., float]:
+    '''Return a position closure which gives the scene x-coordinate for
+    the x point on the right y-axis minus the width of the label given
+    it's contents.
+
+    '''
+    ryaxis = chart.getAxis('right')
+
+    if side == 'left':
+
+        if avoid_book:
+            def right_axis_offset_by_w() -> float:
+
+                # l1 spread graphics x-size
+                l1_len = chart._max_l1_line_len
+
+                # sum of all distances "from" the y-axis
+                right_offset = l1_len + label.w + offset
+
+                return ryaxis.pos().x() - right_offset
+
+        else:
+            def right_axis_offset_by_w() -> float:
+
+                return ryaxis.pos().x() - (label.w + offset)
+
+        return right_axis_offset_by_w
+
+    elif 'right':
+
+        # axis_offset = ryaxis.style['tickTextOffset'][0]
+
+        def on_axis() -> float:
+
+            return ryaxis.pos().x()  # + axis_offset - 2
+
+        return on_axis
+
+
+def gpath_pin(
+
+    gpath: QGraphicsPathItem,
+    label: Label,  # noqa
+
+    location_description: str = 'right-of-path-centered',
+    use_right_of_pp_label: bool = False,
+
+) -> QPointF:
+
+    # get actual arrow graphics path
+    path_br = gpath.mapToScene(gpath.path()).boundingRect()
+
+    # label.vb.locate(label.txt)  #, children=True)
+
+    if location_description == 'right-of-path-centered':
+        return path_br.topRight() - QPointF(label.h/16, label.h / 3)
+
+    if location_description == 'left-of-path-centered':
+        return path_br.topLeft() - QPointF(label.w, label.h / 6)
+
+    elif location_description == 'below-path-left-aligned':
+        return path_br.bottomLeft() - QPointF(0, label.h / 6)
+
+    elif location_description == 'below-path-right-aligned':
+        return path_br.bottomRight() - QPointF(label.w, label.h / 6)
+
+
+def pp_tight_and_right(
+    label: Label
+
+) -> QPointF:
+    '''
+    Place *just* right of the pp label.
+
+    '''
+    # txt = label.txt
+    return label.txt.pos() + QPointF(label.w - label.h/3, 0)

piker/ui/_annotate.py

@@ -18,17 +18,19 @@
 Annotations for ur faces.
 
 """
-import PyQt5
-from PyQt5 import QtCore, QtGui
+from typing import Callable, Optional
+
+from PyQt5 import QtCore, QtGui, QtWidgets
+from PyQt5.QtCore import QPointF, QRectF
 from PyQt5.QtWidgets import QGraphicsPathItem
 from pyqtgraph import Point, functions as fn, Color
 import numpy as np
 
 
-def mk_marker(
-    style,
-    size: float = 20.0,
-    use_qgpath: bool = True,
+def mk_marker_path(
+
+    style: str,
+
 ) -> QGraphicsPathItem:
     """Add a marker to be displayed on the line wrapped in a ``QGraphicsPathItem``
     ready to be placed using scene coordinates (not view).
@@ -37,7 +39,7 @@ def mk_marker(
     style        String indicating the style of marker to add:
                   ``'<|'``, ``'|>'``, ``'>|'``, ``'|<'``, ``'<|>'``,
                   ``'>|<'``, ``'^'``, ``'v'``, ``'o'``
-    size          Size of the marker in pixels. Default is 10.0.
+    size          Size of the marker in pixels.
 
     """
     path = QtGui.QPainterPath()
@@ -81,13 +83,148 @@ def mk_marker(
 
     # self._maxMarkerSize = max([m[2] / 2. for m in self.markers])
 
-    if use_qgpath:
-        path = QGraphicsPathItem(path)
-        path.scale(size, size)
-
     return path
 
 
+class LevelMarker(QGraphicsPathItem):
+    '''An arrow marker path graphich which redraws itself
+    to the specified view coordinate level on each paint cycle.
+
+    '''
+    def __init__(
+        self,
+        chart: 'ChartPlotWidget',  # noqa
+        style: str,
+        get_level: Callable[..., float],
+        size: float = 20,
+        keep_in_view: bool = True,
+        on_paint: Optional[Callable] = None,
+
+    ) -> None:
+
+        # get polygon and scale
+        super().__init__()
+        self.scale(size, size)
+
+        # interally generates path
+        self._style = None
+        self.style = style
+
+        self.chart = chart
+
+        self.get_level = get_level
+        self._on_paint = on_paint
+        self.scene_x = lambda: chart.marker_right_points()[1]
+        self.level: float = 0
+        self.keep_in_view = keep_in_view
+
+    @property
+    def style(self) -> str:
+        return self._style
+
+    @style.setter
+    def style(self, value: str) -> None:
+        if self._style != value:
+            polygon = mk_marker_path(value)
+            self.setPath(polygon)
+            self._style = value
+
+    def path_br(self) -> QRectF:
+        '''Return the bounding rect for the opaque path part
+        of this item.
+
+        '''
+        return self.mapToScene(
+            self.path()
+        ).boundingRect()
+
+    def delete(self) -> None:
+        self.scene().removeItem(self)
+
+    @property
+    def h(self) -> float:
+        return self.path_br().height()
+
+    @property
+    def w(self) -> float:
+        return self.path_br().width()
+
+    def position_in_view(
+        self,
+        # level: float,
+
+    ) -> None:
+        '''Show a pp off-screen indicator for a level label.
+
+        This is like in fps games where you have a gps "nav" indicator
+        but your teammate is outside the range of view, except in 2D, on
+        the y-dimension.
+
+        '''
+        level = self.get_level()
+
+        view = self.chart.getViewBox()
+        vr = view.state['viewRange']
+        ymn, ymx = vr[1]
+
+        # _, marker_right, _ = line._chart.marker_right_points()
+        x = self.scene_x()
+
+        if self.style == '>|':  # short style, points "down-to" line
+            top_offset = self.h
+            bottom_offset = 0
+        else:
+            top_offset = 0
+            bottom_offset = self.h
+
+        if level > ymx:  # pin to top of view
+            self.setPos(
+                QPointF(
+                    x,
+                    top_offset + self.h/3,
+                )
+            )
+
+        elif level < ymn:  # pin to bottom of view
+
+            self.setPos(
+                QPointF(
+                    x,
+                    view.height() - (bottom_offset + self.h/3),
+                )
+            )
+
+        else:
+            # pp line is viewable so show marker normally
+            self.setPos(
+                x,
+                self.chart.view.mapFromView(
+                    QPointF(0, self.get_level())
+                ).y()
+            )
+
+    def paint(
+        self,
+
+        p: QtGui.QPainter,
+        opt: QtWidgets.QStyleOptionGraphicsItem,
+        w: QtWidgets.QWidget
+
+    ) -> None:
+        '''Core paint which we override to always update
+        our marker position in scene coordinates from a
+        view cooridnate "level".
+
+        '''
+        if self.keep_in_view:
+            self.position_in_view()
+
+        super().paint(p, opt, w)
+
+        if self._on_paint:
+            self._on_paint(self)
+
+
 def qgo_draw_markers(
 
     markers: list,

piker/ui/_app.py

@@ -0,0 +1,183 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Main app startup and run.
+
+'''
+from functools import partial
+
+from PyQt5.QtCore import QEvent
+import trio
+
+from .._daemon import maybe_spawn_brokerd
+from ..brokers import get_brokermod
+from . import _event
+from ._exec import run_qtractor
+from ..data.feed import install_brokerd_search
+from . import _search
+from ._chart import GodWidget
+from ..log import get_logger
+
+log = get_logger(__name__)
+
+
+async def load_provider_search(
+
+    broker: str,
+    loglevel: str,
+
+) -> None:
+
+    log.info(f'loading brokerd for {broker}..')
+
+    async with (
+
+        maybe_spawn_brokerd(
+            broker,
+            loglevel=loglevel
+        ) as portal,
+
+        install_brokerd_search(
+            portal,
+            get_brokermod(broker),
+        ),
+    ):
+
+        # keep search engine stream up until cancelled
+        await trio.sleep_forever()
+
+
+async def _async_main(
+
+    # implicit required argument provided by ``qtractor_run()``
+    main_widget: GodWidget,
+
+    sym: str,
+    brokernames: str,
+    loglevel: str,
+
+) -> None:
+    """
+    Main Qt-trio routine invoked by the Qt loop with the widgets ``dict``.
+
+    Provision the "main" widget with initial symbol data and root nursery.
+
+    """
+    from . import _display
+
+    godwidget = main_widget
+
+    # attempt to configure DPI aware font size
+    screen = godwidget.window.current_screen()
+
+    # configure graphics update throttling based on display refresh rate
+    _display._quote_throttle_rate = min(
+        round(screen.refreshRate()),
+        _display._quote_throttle_rate,
+    )
+    log.info(f'Set graphics update rate to {_display._quote_throttle_rate} Hz')
+
+    # TODO: do styling / themeing setup
+    # _style.style_ze_sheets(godwidget)
+
+    sbar = godwidget.window.status_bar
+    starting_done = sbar.open_status('starting ze sexy chartz')
+
+    async with (
+        trio.open_nursery() as root_n,
+    ):
+        # set root nursery and task stack for spawning other charts/feeds
+        # that run cached in the bg
+        godwidget._root_n = root_n
+
+        # setup search widget and focus main chart view at startup
+        # search widget is a singleton alongside the godwidget
+        search = _search.SearchWidget(godwidget=godwidget)
+        search.bar.unfocus()
+
+        godwidget.hbox.addWidget(search)
+        godwidget.search = search
+
+        symbol, _, provider = sym.rpartition('.')
+
+        # this internally starts a ``display_symbol_data()`` task above
+        order_mode_ready = await godwidget.load_symbol(
+            provider,
+            symbol,
+            loglevel
+        )
+
+        # spin up a search engine for the local cached symbol set
+        async with _search.register_symbol_search(
+
+            provider_name='cache',
+            search_routine=partial(
+                _search.search_simple_dict,
+                source=godwidget._chart_cache,
+            ),
+            # cache is super fast so debounce on super short period
+            pause_period=0.01,
+
+        ):
+            # load other providers into search **after**
+            # the chart's select cache
+            for broker in brokernames:
+                root_n.start_soon(load_provider_search, broker, loglevel)
+
+            await order_mode_ready.wait()
+
+            # start handling peripherals input for top level widgets
+            async with (
+
+                # search bar kb input handling
+                _event.open_handlers(
+                    [search.bar],
+                    event_types={
+                        QEvent.KeyPress,
+                    },
+                    async_handler=_search.handle_keyboard_input,
+                    filter_auto_repeats=False,  # let repeats passthrough
+                ),
+
+                # completer view mouse click signal handling
+                _event.open_signal_handler(
+                    search.view.pressed,
+                    search.view.on_pressed,
+                ),
+            ):
+                # remove startup status text
+                starting_done()
+                await trio.sleep_forever()
+
+
+def _main(
+    sym: str,
+    brokernames: [str],
+    piker_loglevel: str,
+    tractor_kwargs,
+) -> None:
+    '''
+    Sync entry point to start a chart: a ``tractor`` + Qt runtime
+    entry point
+
+    '''
+    run_qtractor(
+        func=_async_main,
+        args=(sym, brokernames, piker_loglevel),
+        main_widget=GodWidget,
+        tractor_kwargs=tractor_kwargs,
+    )

piker/ui/_axes.py

@@ -18,39 +18,45 @@
 Chart axes graphics and behavior.
 
 """
-from typing import List, Tuple, Optional
+from functools import lru_cache
+from typing import Optional, Callable
 from math import floor
 
-import pandas as pd
+import numpy as np
 import pyqtgraph as pg
 from PyQt5 import QtCore, QtGui, QtWidgets
 from PyQt5.QtCore import QPointF
 
-from ._style import DpiAwareFont, hcolor, _font
 from ..data._source import float_digits
+from ._label import Label
+from ._style import DpiAwareFont, hcolor, _font
+from ._interaction import ChartView
 
 _axis_pen = pg.mkPen(hcolor('bracket'))
 
 
 class Axis(pg.AxisItem):
-    """A better axis that sizes tick contents considering font size.
+    '''
+    A better axis that sizes tick contents considering font size.
 
-    """
+    '''
     def __init__(
         self,
         linkedsplits,
         typical_max_str: str = '100 000.000',
-        min_tick: int = 2,
+        text_color: str = 'bracket',
         **kwargs
-    ) -> None:
 
-        super().__init__(**kwargs)
+    ) -> None:
+        super().__init__(
+            # textPen=textPen,
+            **kwargs
+        )
 
         # XXX: pretty sure this makes things slower
         # self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
 
         self.linkedsplits = linkedsplits
-        self._min_tick = min_tick
         self._dpi_font = _font
 
         self.setTickFont(_font.font)
@@ -72,44 +78,128 @@ class Axis(pg.AxisItem):
         })
 
         self.setTickFont(_font.font)
+
+        # NOTE: this is for surrounding "border"
         self.setPen(_axis_pen)
+
+        # this is the text color
+        # self.setTextPen(pg.mkPen(hcolor(text_color)))
+        self.text_color = text_color
+
         self.typical_br = _font._qfm.boundingRect(typical_max_str)
 
         # size the pertinent axis dimension to a "typical value"
         self.size_to_values()
 
+    @property
+    def text_color(self) -> str:
+        return self._text_color
+
+    @text_color.setter
+    def text_color(self, text_color: str) -> None:
+        self.setTextPen(pg.mkPen(hcolor(text_color)))
+        self._text_color = text_color
+
     def size_to_values(self) -> None:
         pass
 
-    def set_min_tick(self, size: int) -> None:
-        self._min_tick = size
-
-    def txt_offsets(self) -> Tuple[int, int]:
+    def txt_offsets(self) -> tuple[int, int]:
         return tuple(self.style['tickTextOffset'])
 
 
 class PriceAxis(Axis):
 
+    def __init__(
+        self,
+        *args,
+        min_tick: int = 2,
+        title: str = '',
+        formatter: Optional[Callable[[float], str]] = None,
+        **kwargs
+
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self.formatter = formatter
+        self._min_tick: int = min_tick
+        self.title = None
+
+    def set_title(
+        self,
+        title: str,
+        view: Optional[ChartView] = None,
+        color: Optional[str] = None,
+
+    ) -> Label:
+        '''
+        Set a sane UX label using our built-in ``Label``.
+
+        '''
+        # XXX: built-in labels but they're huge, and placed weird..
+        # self.setLabel(title)
+        # self.showLabel()
+
+        label = self.title = Label(
+            view=view or self.linkedView(),
+            fmt_str=title,
+            color=color or self.text_color,
+            parent=self,
+            # update_on_range_change=False,
+        )
+
+        def below_axis() -> QPointF:
+            return QPointF(
+                0,
+                self.size().height(),
+            )
+
+        # XXX: doesn't work? have to pass it above
+        # label.txt.setParent(self)
+        label.scene_anchor = below_axis
+        label.render()
+        label.show()
+        label.update()
+        return label
+
+    def set_min_tick(
+        self,
+        size: int
+    ) -> None:
+        self._min_tick = size
+
     def size_to_values(self) -> None:
+        # self.typical_br = _font._qfm.boundingRect(typical_max_str)
         self.setWidth(self.typical_br.width())
 
     # XXX: drop for now since it just eats up h space
 
-    def tickStrings(self, vals, scale, spacing):
+    def tickStrings(
+        self,
+        vals: tuple[float],
+        scale: float,
+        spacing: float,
 
-        # TODO: figure out how to enforce min tick spacing by passing
-        # it into the parent type
-        digits = max(float_digits(spacing * scale), self._min_tick)
+    ) -> list[str]:
+        # TODO: figure out how to enforce min tick spacing by passing it
+        # into the parent type
+        digits = max(
+            float_digits(spacing * scale),
+            self._min_tick,
+        )
+        if self.title:
+            self.title.update()
 
         # print(f'vals: {vals}\nscale: {scale}\nspacing: {spacing}')
         # print(f'digits: {digits}')
 
-        return [
-            ('{value:,.{digits}f}').format(
-                digits=digits,
-                value=v,
-            ).replace(',', ' ') for v in vals
-        ]
+        if not self.formatter:
+            return [
+                ('{value:,.{digits}f}').format(
+                    digits=digits,
+                    value=v,
+                ).replace(',', ' ') for v in vals
+            ]
+        else:
+            return list(map(self.formatter, vals))
 
 
 class DynamicDateAxis(Axis):
@@ -128,13 +218,14 @@ class DynamicDateAxis(Axis):
 
     def _indexes_to_timestrs(
         self,
-        indexes: List[int],
-    ) -> List[str]:
+        indexes: list[int],
+
+    ) -> list[str]:
 
-        # try:
         chart = self.linkedsplits.chart
-        bars = chart._arrays['ohlc']
-        shm = self.linkedsplits.chart._shm
+        flow = chart._flows[chart.name]
+        shm = flow.shm
+        bars = shm.array
         first = shm._first.value
 
         bars_len = len(bars)
@@ -151,12 +242,27 @@ class DynamicDateAxis(Axis):
         )]
 
         # TODO: **don't** have this hard coded shift to EST
-        dts = pd.to_datetime(epochs, unit='s')  # - 4*pd.offsets.Hour()
+        # delay = times[-1] - times[-2]
+        dts = np.array(epochs, dtype='datetime64[s]')
 
-        delay = times[-1] - times[-2]
-        return dts.strftime(self.tick_tpl[delay])
+        # see units listing:
+        # https://numpy.org/devdocs/reference/arrays.datetime.html#datetime-units
+        return list(np.datetime_as_string(dts))
 
-    def tickStrings(self, values: List[float], scale, spacing):
+        # TODO: per timeframe formatting?
+        # - we probably need this based on zoom now right?
+        # prec = self.np_dt_precision[delay]
+        # return dts.strftime(self.tick_tpl[delay])
+
+    def tickStrings(
+        self,
+        values: tuple[float],
+        scale: float,
+        spacing: float,
+
+    ) -> list[str]:
+        # info = self.tickStrings.cache_info()
+        # print(info)
         return self._indexes_to_timestrs(values)
 
 
@@ -207,6 +313,8 @@ class AxisLabel(pg.GraphicsObject):
         self.path = None
         self.rect = None
 
+        self._pw = self.pixelWidth()
+
     def paint(
         self,
         p: QtGui.QPainter,
@@ -256,9 +364,10 @@ class AxisLabel(pg.GraphicsObject):
 
 
     def boundingRect(self):  # noqa
-        """Size the graphics space from the text contents.
+        '''
+        Size the graphics space from the text contents.
 
-        """
+        '''
         if self.label_str:
             self._size_br_from_str(self.label_str)
 
@@ -274,23 +383,32 @@ class AxisLabel(pg.GraphicsObject):
 
         return QtCore.QRectF()
 
-        # return self.rect or QtCore.QRectF()
+    # TODO: but the input probably needs to be the "len" of
+    # the current text value:
+    @lru_cache
+    def _size_br_from_str(
+        self,
+        value: str
 
-    def _size_br_from_str(self, value: str) -> None:
-        """Do our best to render the bounding rect to a set margin
+    ) -> tuple[float, float]:
+        '''
+        Do our best to render the bounding rect to a set margin
         around provided string contents.
 
-        """
+        '''
         # size the filled rect to text and/or parent axis
         # if not self._txt_br:
-        #     # XXX: this can't be c
+        #     # XXX: this can't be called until stuff is rendered?
         #     self._txt_br = self._dpifont.boundingRect(value)
 
         txt_br = self._txt_br = self._dpifont.boundingRect(value)
         txt_h, txt_w = txt_br.height(), txt_br.width()
+        # print(f'wsw: {self._dpifont.boundingRect(" ")}')
 
         # allow subtypes to specify a static width and height
         h, w = self.size_hint()
+        # print(f'axis size: {self._parent.size()}')
+        # print(f'axis geo: {self._parent.geometry()}')
 
         self.rect = QtCore.QRectF(
             0, 0,
@@ -301,7 +419,7 @@ class AxisLabel(pg.GraphicsObject):
         # hb = self.path.controlPointRect()
         # hb_size = hb.size()
 
-        return self.rect
+        return (self.rect.width(), self.rect.height())
 
 # _common_text_flags = (
 #     QtCore.Qt.TextDontClip |
@@ -320,7 +438,7 @@ class XAxisLabel(AxisLabel):
         | QtCore.Qt.AlignCenter
     )
 
-    def size_hint(self) -> Tuple[float, float]:
+    def size_hint(self) -> tuple[float, float]:
         # size to parent axis height
         return self._parent.height(), None
 
@@ -329,31 +447,34 @@ class XAxisLabel(AxisLabel):
         abs_pos: QPointF,  # scene coords
         value: float,  # data for text
         offset: int = 0  # if have margins, k?
+
     ) -> None:
 
         timestrs = self._parent._indexes_to_timestrs([int(value)])
 
-        if not timestrs.any():
+        if not len(timestrs):
             return
 
         pad = 1*' '
-        self.label_str = pad + timestrs[0] + pad
+        self.label_str = pad + str(timestrs[0]) + pad
 
         _, y_offset = self._parent.txt_offsets()
 
         w = self.boundingRect().width()
 
-        self.setPos(QPointF(
-            abs_pos.x() - w/2,
-            y_offset/2,
-        ))
+        self.setPos(
+            QPointF(
+                abs_pos.x() - w/2 - self._pw,
+                y_offset/2,
+            )
+        )
         self.update()
 
     def _draw_arrow_path(self):
         y_offset = self._parent.style['tickTextOffset'][1]
         path = QtGui.QPainterPath()
         h, w = self.rect.height(), self.rect.width()
-        middle = w/2 - 0.5
+        middle = w/2 - self._pw * 0.5
         aw = h/2
         left = middle - aw
         right = middle + aw
@@ -396,9 +517,13 @@ class YAxisLabel(AxisLabel):
         if getattr(self._parent, 'txt_offsets', False):
             self.x_offset, y_offset = self._parent.txt_offsets()
 
-    def size_hint(self) -> Tuple[float, float]:
-        # size to parent axis width
-        return None, self._parent.width()
+    def size_hint(self) -> tuple[float, float]:
+        # size to parent axis width(-ish)
+        wsh = self._dpifont.boundingRect(' ').height() / 2
+        return (
+            None,
+            self._parent.size().width() - wsh,
+        )
 
     def update_label(
         self,
@@ -419,16 +544,19 @@ class YAxisLabel(AxisLabel):
         br = self.boundingRect()
         h = br.height()
 
-        self.setPos(QPointF(
-            x_offset,
-            abs_pos.y() - h / 2 - self._y_margin / 2
-        ))
+        self.setPos(
+            QPointF(
+                x_offset,
+                abs_pos.y() - h / 2 - self._pw,
+            )
+        )
         self.update()
 
     def update_on_resize(self, vr, r):
-        """Tiis is a ``.sigRangeChanged()`` handler.
+        '''
+        This is a ``.sigRangeChanged()`` handler.
 
-        """
+        '''
         index, last = self._last_datum
         if index is not None:
             self.update_from_data(index, last)
@@ -438,11 +566,13 @@ class YAxisLabel(AxisLabel):
         index: int,
         value: float,
         _save_last: bool = True,
+
     ) -> None:
-        """Update the label's text contents **and** position from
+        '''
+        Update the label's text contents **and** position from
         a view box coordinate datum.
 
-        """
+        '''
         if _save_last:
             self._last_datum = (index, value)
 
@@ -456,7 +586,7 @@ class YAxisLabel(AxisLabel):
         path = QtGui.QPainterPath()
         h = self.rect.height()
         path.moveTo(0, 0)
-        path.lineTo(-x_offset - h/4, h/2.)
+        path.lineTo(-x_offset - h/4, h/2. - self._pw/2)
         path.lineTo(0, h)
         path.closeSubpath()
         self.path = path

piker/ui/_compression.py

@@ -0,0 +1,318 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Graphics related downsampling routines for compressing to pixel
+limits on the display device.
+
+'''
+import math
+from typing import Optional
+
+import numpy as np
+from numpy.lib import recfunctions as rfn
+from numba import (
+    jit,
+    # float64, optional, int64,
+)
+
+from ..log import get_logger
+
+
+log = get_logger(__name__)
+
+
+def hl2mxmn(ohlc: np.ndarray) -> np.ndarray:
+    '''
+    Convert a OHLC struct-array containing 'high'/'low' columns
+    to a "joined" max/min 1-d array.
+
+    '''
+    index = ohlc['index']
+    hls = ohlc[[
+        'low',
+        'high',
+    ]]
+
+    mxmn = np.empty(2*hls.size, dtype=np.float64)
+    x = np.empty(2*hls.size, dtype=np.float64)
+    trace_hl(hls, mxmn, x, index[0])
+    x = x + index[0]
+
+    return mxmn, x
+
+
+@jit(
+    # TODO: the type annots..
+    # float64[:](float64[:],),
+    nopython=True,
+)
+def trace_hl(
+    hl: 'np.ndarray',
+    out: np.ndarray,
+    x: np.ndarray,
+    start: int,
+
+    # the "offset" values in the x-domain which
+    # place the 2 output points around each ``int``
+    # master index.
+    margin: float = 0.43,
+
+) -> None:
+    '''
+    "Trace" the outline of the high-low values of an ohlc sequence
+    as a line such that the maximum deviation (aka disperaion) between
+    bars if preserved.
+
+    This routine is expected to modify input arrays in-place.
+
+    '''
+    last_l = hl['low'][0]
+    last_h = hl['high'][0]
+
+    for i in range(hl.size):
+        row = hl[i]
+        l, h = row['low'], row['high']
+
+        up_diff = h - last_l
+        down_diff = last_h - l
+
+        if up_diff > down_diff:
+            out[2*i + 1] = h
+            out[2*i] = last_l
+        else:
+            out[2*i + 1] = l
+            out[2*i] = last_h
+
+        last_l = l
+        last_h = h
+
+        x[2*i] = int(i) - margin
+        x[2*i + 1] = int(i) + margin
+
+    return out
+
+
+def ohlc_flatten(
+    ohlc: np.ndarray,
+    use_mxmn: bool = True,
+
+) -> tuple[np.ndarray, np.ndarray]:
+    '''
+    Convert an OHLCV struct-array into a flat ready-for-line-plotting
+    1-d array that is 4 times the size with x-domain values distributed
+    evenly (by 0.5 steps) over each index.
+
+    '''
+    index = ohlc['index']
+
+    if use_mxmn:
+        # traces a line optimally over highs to lows
+        # using numba. NOTE: pretty sure this is faster
+        # and looks about the same as the below output.
+        flat, x = hl2mxmn(ohlc)
+
+    else:
+        flat = rfn.structured_to_unstructured(
+            ohlc[['open', 'high', 'low', 'close']]
+        ).flatten()
+
+        x = np.linspace(
+            start=index[0] - 0.5,
+            stop=index[-1] + 0.5,
+            num=len(flat),
+        )
+    return x, flat
+
+
+def ds_m4(
+    x: np.ndarray,
+    y: np.ndarray,
+    # units-per-pixel-x(dimension)
+    uppx: float,
+
+    # XXX: troll zone / easter egg..
+    # want to mess with ur pal, pass in the actual
+    # pixel width here instead of uppx-proper (i.e. pass
+    # in our ``pg.GraphicsObject`` derivative's ``.px_width()``
+    # gto mega-trip-out ur bud). Hint, it used to be implemented
+    # (wrongly) using "pixel width", so check the git history ;)
+
+    xrange: Optional[float] = None,
+
+) -> tuple[int, np.ndarray, np.ndarray]:
+    '''
+    Downsample using the M4 algorithm.
+
+    This is more or less an OHLC style sampling of a line-style series.
+
+    '''
+    # NOTE: this method is a so called "visualization driven data
+    # aggregation" approach. It gives error-free line chart
+    # downsampling, see
+    # further scientific paper resources:
+    # - http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
+    # - http://www.vldb.org/2014/program/papers/demo/p997-jugel.pdf
+
+    # Details on implementation of this algo are based in,
+    # https://github.com/pikers/piker/issues/109
+
+    # XXX: from infinite on downsampling viewable graphics:
+    # "one thing i remembered about the binning - if you are
+    # picking a range within your timeseries the start and end bin
+    # should be one more bin size outside the visual range, then
+    # you get better visual fidelity at the edges of the graph"
+    # "i didn't show it in the sample code, but it's accounted for
+    # in the start and end indices and number of bins"
+
+    # should never get called unless actually needed
+    assert uppx > 1
+
+    # NOTE: if we didn't pre-slice the data to downsample
+    # you could in theory pass these as the slicing params,
+    # do we care though since we can always just pre-slice the
+    # input?
+    x_start = x[0]  # x value start/lowest in domain
+
+    if xrange is None:
+        x_end = x[-1]  # x end value/highest in domain
+        xrange = (x_end - x_start)
+
+    # XXX: always round up on the input pixels
+    # lnx = len(x)
+    # uppx *= max(4 / (1 + math.log(uppx, 2)), 1)
+
+    pxw = math.ceil(xrange / uppx)
+
+    # scale up the frame "width" directly with uppx
+    w = uppx
+
+    # ensure we make more then enough
+    # frames (windows) for the output pixel
+    frames = pxw
+
+    # if we have more and then exact integer's
+    # (uniform quotient output) worth of datum-domain-points
+    # per windows-frame, add one more window to ensure
+    # we have room for all output down-samples.
+    pts_per_pixel, r = divmod(xrange, frames)
+    if r:
+        # while r:
+        frames += 1
+        pts_per_pixel, r = divmod(xrange, frames)
+
+    # print(
+    #     f'uppx: {uppx}\n'
+    #     f'xrange: {xrange}\n'
+    #     f'pxw: {pxw}\n'
+    #     f'frames: {frames}\n'
+    # )
+    assert frames >= (xrange / uppx)
+
+    # call into ``numba``
+    nb, i_win, y_out = _m4(
+        x,
+        y,
+
+        frames,
+
+        # TODO: see func below..
+        # i_win,
+        # y_out,
+
+        # first index in x data to start at
+        x_start,
+        # window size for each "frame" of data to downsample (normally
+        # scaled by the ratio of pixels on screen to data in x-range).
+        w,
+    )
+
+    # filter out any overshoot in the input allocation arrays by
+    # removing zero-ed tail entries which should start at a certain
+    # index.
+    i_win = i_win[i_win != 0]
+    y_out = y_out[:i_win.size]
+
+    return nb, i_win, y_out
+
+
+@jit(
+    nopython=True,
+    nogil=True,
+)
+def _m4(
+
+    xs: np.ndarray,
+    ys: np.ndarray,
+
+    frames: int,
+
+    # TODO: using this approach by having the ``.zeros()`` alloc lines
+    # below, in put python was causing segs faults and alloc crashes..
+    # we might need to see how it behaves with shm arrays and consider
+    # allocating them once at startup?
+
+    # pre-alloc array of x indices mapping to the start
+    # of each window used for downsampling in y.
+    # i_win: np.ndarray,
+    # pre-alloc array of output downsampled y values
+    # y_out: np.ndarray,
+
+    x_start: int,
+    step: float,
+
+) -> int:
+    # nbins = len(i_win)
+    # count = len(xs)
+
+    # these are pre-allocated and mutated by ``numba``
+    # code in-place.
+    y_out = np.zeros((frames, 4), ys.dtype)
+    i_win = np.zeros(frames, xs.dtype)
+
+    bincount = 0
+    x_left = x_start
+
+    # Find the first window's starting value which *includes* the
+    # first value in the x-domain array, i.e. the first
+    # "left-side-of-window" **plus** the downsampling step,
+    # creates a window which includes the first x **value**.
+    while xs[0] >= x_left + step:
+        x_left += step
+
+    # set all bins in the left-most entry to the starting left-most x value
+    # (aka a row broadcast).
+    i_win[bincount] = x_left
+    # set all y-values to the first value passed in.
+    y_out[bincount] = ys[0]
+
+    for i in range(len(xs)):
+        x = xs[i]
+        y = ys[i]
+        if x < x_left + step:   # the current window "step" is [bin, bin+1)
+            y_out[bincount, 1] = min(y, y_out[bincount, 1])
+            y_out[bincount, 2] = max(y, y_out[bincount, 2])
+            y_out[bincount, 3] = y
+        else:
+            # Find the next bin
+            while x >= x_left + step:
+                x_left += step
+
+            bincount += 1
+            i_win[bincount] = x_left
+            y_out[bincount] = y
+
+    return bincount, i_win, y_out

piker/ui/_cursor.py

@@ -24,16 +24,17 @@ from typing import Optional, Callable
 import inspect
 import numpy as np
 import pyqtgraph as pg
-from PyQt5 import QtCore, QtGui, QtWidgets
+from PyQt5 import QtCore, QtWidgets
 from PyQt5.QtCore import QPointF, QRectF
 
-from .._style import (
+from ._style import (
     _xaxis_at,
     hcolor,
     _font_small,
+    _font,
 )
-from .._axes import YAxisLabel, XAxisLabel
-from ...log import get_logger
+from ._axes import YAxisLabel, XAxisLabel
+from ..log import get_logger
 
 
 log = get_logger(__name__)
@@ -41,8 +42,9 @@ log = get_logger(__name__)
 # XXX: these settings seem to result in really decent mouse scroll
 # latency (in terms of perceived lag in cross hair) so really be sure
 # there's an improvement if you want to change it!
+
 _mouse_rate_limit = 60  # TODO; should we calc current screen refresh rate?
-_debounce_delay = 1 / 2e3
+_debounce_delay = 0
 _ch_label_opac = 1
 
 
@@ -52,13 +54,18 @@ class LineDot(pg.CurvePoint):
 
     def __init__(
         self,
+
         curve: pg.PlotCurveItem,
         index: int,
+
         plot: 'ChartPlotWidget',  # type: ingore # noqa
         pos=None,
-        size: int = 6,  # in pxs
         color: str = 'default_light',
+
     ) -> None:
+        # scale from dpi aware font size
+        size = int(_font.px_size * 0.375)
+
         pg.CurvePoint.__init__(
             self,
             curve,
@@ -89,22 +96,32 @@ class LineDot(pg.CurvePoint):
     def event(
         self,
         ev: QtCore.QEvent,
-    ) -> None:
-        if not isinstance(
-            ev, QtCore.QDynamicPropertyChangeEvent
-        ) or self.curve() is None:
+
+    ) -> bool:
+
+        if (
+            not isinstance(ev, QtCore.QDynamicPropertyChangeEvent)
+            or self.curve() is None
+        ):
             return False
 
-        (x, y) = self.curve().getData()
-        index = self.property('index')
-        # first = self._plot._arrays['ohlc'][0]['index']
-        # first = x[0]
-        # i = index - first
-        i = index - x[0]
-        if i > 0 and i < len(y):
-            newPos = (index, y[i])
-            QtWidgets.QGraphicsItem.setPos(self, *newPos)
-            return True
+        # TODO: get rid of this ``.getData()`` and
+        # make a more pythonic api to retreive backing
+        # numpy arrays...
+        # (x, y) = self.curve().getData()
+        # index = self.property('index')
+        # # first = self._plot._arrays['ohlc'][0]['index']
+        # # first = x[0]
+        # # i = index - first
+        # if index:
+        #     i = round(index - x[0])
+        #     if i > 0 and i < len(y):
+        #         newPos = (index, y[i])
+        #         QtWidgets.QGraphicsItem.setPos(
+        #             self,
+        #             *newPos,
+        #         )
+        #         return True
 
         return False
 
@@ -132,8 +149,8 @@ class ContentsLabel(pg.LabelItem):
     }
 
     def __init__(
-
         self,
+
         # chart: 'ChartPlotWidget',  # noqa
         view: pg.ViewBox,
 
@@ -167,8 +184,8 @@ class ContentsLabel(pg.LabelItem):
         self.anchor(itemPos=index, parentPos=index, offset=margins)
 
     def update_from_ohlc(
-
         self,
+
         name: str,
         index: int,
         array: np.ndarray,
@@ -179,6 +196,9 @@ class ContentsLabel(pg.LabelItem):
 
         self.setText(
             "<b>i</b>:{index}<br/>"
+            # NB: these fields must be indexed in the correct order via
+            # the slice syntax below.
+            "<b>epoch</b>:{}<br/>"
             "<b>O</b>:{}<br/>"
             "<b>H</b>:{}<br/>"
             "<b>L</b>:{}<br/>"
@@ -186,7 +206,15 @@ class ContentsLabel(pg.LabelItem):
             "<b>V</b>:{}<br/>"
             "<b>wap</b>:{}".format(
                 *array[index - first][
-                    ['open', 'high', 'low', 'close', 'volume', 'bar_wap']
+                    [
+                        'time',
+                        'open',
+                        'high',
+                        'low',
+                        'close',
+                        'volume',
+                        'bar_wap',
+                    ]
                 ],
                 name=name,
                 index=index,
@@ -194,8 +222,8 @@ class ContentsLabel(pg.LabelItem):
         )
 
     def update_from_value(
-
         self,
+
         name: str,
         index: int,
         array: np.ndarray,
@@ -231,17 +259,20 @@ class ContentsLabels:
     def update_labels(
         self,
         index: int,
-        # array_name: str,
 
     ) -> None:
-        # for name, (label, update) in self._labels.items():
         for chart, name, label, update in self._labels:
 
-            if not (index >= 0 and index < chart._arrays['ohlc'][-1]['index']):
-                # out of range
-                continue
+            flow = chart._flows[name]
+            array = flow.shm.array
 
-            array = chart._arrays[name]
+            if not (
+                index >= 0
+                and index < array[-1]['index']
+            ):
+                # out of range
+                print('WTF out of range?')
+                continue
 
             # call provided update func with data point
             try:
@@ -266,19 +297,22 @@ class ContentsLabels:
     ) -> ContentsLabel:
 
         label = ContentsLabel(
-            view=chart._vb,
+            view=chart.view,
             anchor_at=anchor_at,
         )
         self._labels.append(
             (chart, name, label, partial(update_func, label, name))
         )
-        # label.hide()
+        label.hide()
 
         return label
 
 
 class Cursor(pg.GraphicsObject):
+    '''
+    Multi-plot cursor for use on a ``LinkedSplits`` chart (set).
 
+    '''
     def __init__(
 
         self,
@@ -291,7 +325,7 @@ class Cursor(pg.GraphicsObject):
 
         self.linked = linkedsplits
         self.graphics: dict[str, pg.GraphicsObject] = {}
-        self.plots: List['PlotChartWidget'] = []  # type: ignore # noqa
+        self.plots: list['PlotChartWidget'] = []  # type: ignore # noqa
         self.active_plot = None
         self.digits: int = digits
         self._datum_xy: tuple[int, float] = (0, 0)
@@ -353,7 +387,13 @@ class Cursor(pg.GraphicsObject):
         self,
         plot: 'ChartPlotWidget',  # noqa
         digits: int = 0,
+
     ) -> None:
+        '''
+        Add chart to tracked set such that a cross-hair and possibly
+        curve tracking cursor can be drawn on the plot.
+
+        '''
         # add ``pg.graphicsItems.InfiniteLine``s
         # vertical and horizonal lines and a y-axis label
 
@@ -366,7 +406,8 @@ class Cursor(pg.GraphicsObject):
 
         yl = YAxisLabel(
             chart=plot,
-            parent=plot.getAxis('right'),
+            # parent=plot.getAxis('right'),
+            parent=plot.pi_overlay.get_axis(plot.plotItem, 'right'),
             digits=digits or self.digits,
             opacity=_ch_label_opac,
             bg_color=self.label_color,
@@ -381,6 +422,7 @@ class Cursor(pg.GraphicsObject):
             slot=self.mouseMoved,
             delay=_debounce_delay,
         )
+
         px_enter = pg.SignalProxy(
             plot.sig_mouse_enter,
             rateLimit=_mouse_rate_limit,
@@ -406,24 +448,39 @@ class Cursor(pg.GraphicsObject):
         # keep x-axis right below main chart
         plot_index = -1 if _xaxis_at == 'bottom' else 0
 
-        self.xaxis_label = XAxisLabel(
-            parent=self.plots[plot_index].getAxis('bottom'),
-            opacity=_ch_label_opac,
-            bg_color=self.label_color,
-        )
-        # place label off-screen during startup
-        self.xaxis_label.setPos(self.plots[0].mapFromView(QPointF(0, 0)))
+        # ONLY create an x-axis label for the cursor
+        # if this plot owns the 'bottom' axis.
+        # if 'bottom' in plot.plotItem.axes:
+        if plot.linked.xaxis_chart is plot:
+            xlabel = self.xaxis_label = XAxisLabel(
+                parent=self.plots[plot_index].getAxis('bottom'),
+                # parent=self.plots[plot_index].pi_overlay.get_axis(
+                #     plot.plotItem, 'bottom'
+                # ),
+
+                opacity=_ch_label_opac,
+                bg_color=self.label_color,
+            )
+            # place label off-screen during startup
+            xlabel.setPos(
+                self.plots[0].mapFromView(QPointF(0, 0))
+            )
+            xlabel.show()
 
     def add_curve_cursor(
         self,
         plot: 'ChartPlotWidget',  # noqa
         curve: 'PlotCurveItem',  # noqa
+
     ) -> LineDot:
         # if this plot contains curves add line dot "cursors" to denote
         # the current sample under the mouse
+        main_flow = plot._flows[plot.name]
+        # read out last index
+        i = main_flow.shm.array[-1]['index']
         cursor = LineDot(
             curve,
-            index=plot._arrays['ohlc'][-1]['index'],
+            index=i,
             plot=plot
         )
         plot.addItem(cursor)
@@ -447,12 +504,15 @@ class Cursor(pg.GraphicsObject):
 
     def mouseMoved(
         self,
-        evt: 'tuple[QMouseEvent]',  # noqa
-    ) -> None:  # noqa
-        """Update horizonal and vertical lines when mouse moves inside
+        coords: tuple[QPointF],  # noqa
+
+    ) -> None:
+        '''
+        Update horizonal and vertical lines when mouse moves inside
         either the main chart or any indicator subplot.
-        """
-        pos = evt[0]
+
+        '''
+        pos = coords[0]
 
         # find position inside active plot
         try:
@@ -471,24 +531,27 @@ class Cursor(pg.GraphicsObject):
 
         ix = round(x)  # since bars are centered around index
 
+        # px perfect...
+        line_offset = self._lw / 2
+
         # round y value to nearest tick step
         m = self._y_incr_mult
         iy = round(y * m) / m
-
-        # px perfect...
-        line_offset = self._lw / 2
+        vl_y = iy - line_offset
 
         # update y-range items
         if iy != last_iy:
 
             if self._y_label_update:
                 self.graphics[self.active_plot]['yl'].update_label(
-                    abs_pos=plot.mapFromView(QPointF(ix, iy)),
+                    # abs_pos=plot.mapFromView(QPointF(ix, iy)),
+                    abs_pos=plot.mapFromView(QPointF(ix, vl_y)),
                     value=iy
                 )
 
                 # only update horizontal xhair line if label is enabled
-                self.graphics[plot]['hl'].setY(iy)
+                # self.graphics[plot]['hl'].setY(iy)
+                self.graphics[plot]['hl'].setY(vl_y)
 
             # update all trackers
             for item in self._trackers:
@@ -501,29 +564,39 @@ class Cursor(pg.GraphicsObject):
                 # with cursor movement
                 self.contents_labels.update_labels(ix)
 
+            vl_x = ix + line_offset
             for plot, opts in self.graphics.items():
 
-                # update the chart's "contents" label
-                # plot.update_contents_labels(ix)
-
                 # move the vertical line to the current "center of bar"
-                opts['vl'].setX(ix + line_offset)
+                opts['vl'].setX(vl_x)
 
                 # update all subscribed curve dots
                 for cursor in opts.get('cursors', ()):
                     cursor.setIndex(ix)
 
-            # update the label on the bottom of the crosshair
-            self.xaxis_label.update_label(
+                # Update the label on the bottom of the crosshair.
+                # TODO: make this an up-front calc that we update
+                # on axis-widget resize events instead of on every mouse
+                # update cylce.
 
-                # XXX: requires:
-                # https://github.com/pyqtgraph/pyqtgraph/pull/1418
-                # otherwise gobbles tons of CPU..
+                # left axis offset width for calcuating
+                # absolute x-axis label placement.
+                left_axis_width = 0
+                if len(plot.pi_overlay.overlays):
+                    # breakpoint()
+                    lefts = plot.pi_overlay.get_axes('left')
+                    if lefts:
+                        for left in lefts:
+                            left_axis_width += left.width()
 
                 # map back to abs (label-local) coordinates
-                abs_pos=plot.mapFromView(QPointF(ix + line_offset, iy)),
-                value=ix,
-            )
+                self.xaxis_label.update_label(
+                    abs_pos=(
+                        plot.mapFromView(QPointF(vl_x, iy)) -
+                        QPointF(left_axis_width, 0)
+                    ),
+                    value=ix,
+                )
 
         self._datum_xy = ix, iy
 

piker/ui/_curve.py

@@ -0,0 +1,484 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+Fast, smooth, sexy curves.
+
+"""
+from contextlib import contextmanager as cm
+from typing import Optional, Callable
+
+import numpy as np
+import pyqtgraph as pg
+from PyQt5 import QtWidgets
+from PyQt5.QtWidgets import QGraphicsItem
+from PyQt5.QtCore import (
+    Qt,
+    QLineF,
+    QSizeF,
+    QRectF,
+    # QRect,
+    QPointF,
+)
+from PyQt5.QtGui import (
+    QPainter,
+    QPainterPath,
+)
+from .._profile import pg_profile_enabled, ms_slower_then
+from ._style import hcolor
+# from ._compression import (
+#     # ohlc_to_m4_line,
+#     ds_m4,
+# )
+from ..log import get_logger
+
+
+log = get_logger(__name__)
+
+
+_line_styles: dict[str, int] = {
+    'solid': Qt.PenStyle.SolidLine,
+    'dash': Qt.PenStyle.DashLine,
+    'dot': Qt.PenStyle.DotLine,
+    'dashdot': Qt.PenStyle.DashDotLine,
+}
+
+
+class Curve(pg.GraphicsObject):
+    '''
+    A faster, simpler, append friendly version of
+    ``pyqtgraph.PlotCurveItem`` built for highly customizable real-time
+    updates.
+
+    This type is a much stripped down version of a ``pyqtgraph`` style
+    "graphics object" in the sense that the internal lower level
+    graphics which are drawn in the ``.paint()`` method are actually
+    rendered outside of this class entirely and instead are assigned as
+    state (instance vars) here and then drawn during a Qt graphics
+    cycle.
+
+    The main motivation for this more modular, composed design is that
+    lower level graphics data can be rendered in different threads and
+    then read and drawn in this main thread without having to worry
+    about dealing with Qt's concurrency primitives. See
+    ``piker.ui._flows.Renderer`` for details and logic related to lower
+    level path generation and incremental update. The main differences in
+    the path generation code include:
+
+    - avoiding regeneration of the entire historical path where possible
+      and instead only updating the "new" segment(s) via a ``numpy``
+      array diff calc.
+    - here, the "last" graphics datum-segment is drawn independently
+      such that near-term (high frequency) discrete-time-sampled style
+      updates don't trigger a full path redraw.
+
+    '''
+
+    # sub-type customization methods
+    sub_br: Optional[Callable] = None
+    sub_paint: Optional[Callable] = None
+    declare_paintables: Optional[Callable] = None
+
+    def __init__(
+        self,
+        *args,
+
+        step_mode: bool = False,
+        color: str = 'default_lightest',
+        fill_color: Optional[str] = None,
+        style: str = 'solid',
+        name: Optional[str] = None,
+        use_fpath: bool = True,
+
+        **kwargs
+
+    ) -> None:
+
+        self._name = name
+
+        # brutaaalll, see comments within..
+        self.yData = None
+        self.xData = None
+
+        # self._last_cap: int = 0
+        self.path: Optional[QPainterPath] = None
+
+        # additional path used for appends which tries to avoid
+        # triggering an update/redraw of the presumably larger
+        # historical ``.path`` above.
+        self.use_fpath = use_fpath
+        self.fast_path: Optional[QPainterPath] = None
+
+        # TODO: we can probably just dispense with the parent since
+        # we're basically only using the pen setting now...
+        super().__init__(*args, **kwargs)
+
+        # all history of curve is drawn in single px thickness
+        pen = pg.mkPen(hcolor(color))
+        pen.setStyle(_line_styles[style])
+
+        if 'dash' in style:
+            pen.setDashPattern([8, 3])
+
+        self._pen = pen
+
+        # last segment is drawn in 2px thickness for emphasis
+        # self.last_step_pen = pg.mkPen(hcolor(color), width=2)
+        self.last_step_pen = pg.mkPen(pen, width=2)
+
+        # self._last_line: Optional[QLineF] = None
+        self._last_line = QLineF()
+        self._last_w: float = 1
+
+        # flat-top style histogram-like discrete curve
+        # self._step_mode: bool = step_mode
+
+        # self._fill = True
+        self._brush = pg.functions.mkBrush(hcolor(fill_color or color))
+
+        # NOTE: this setting seems to mostly prevent redraws on mouse
+        # interaction which is a huge boon for avg interaction latency.
+
+        # TODO: one question still remaining is if this makes trasform
+        # interactions slower (such as zooming) and if so maybe if/when
+        # we implement a "history" mode for the view we disable this in
+        # that mode?
+        # don't enable caching by default for the case where the
+        # only thing drawn is the "last" line segment which can
+        # have a weird artifact where it won't be fully drawn to its
+        # endpoint (something we saw on trade rate curves)
+        self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
+
+        # XXX: see explanation for different caching modes:
+        # https://stackoverflow.com/a/39410081
+        # seems to only be useful if we don't re-generate the entire
+        # QPainterPath every time
+        # curve.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
+
+        # don't ever use this - it's a colossal nightmare of artefacts
+        # and is disastrous for performance.
+        # curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache)
+
+        # allow sub-type customization
+        declare = self.declare_paintables
+        if declare:
+            declare()
+
+    # TODO: probably stick this in a new parent
+    # type which will contain our own version of
+    # what ``PlotCurveItem`` had in terms of base
+    # functionality? A `FlowGraphic` maybe?
+    def x_uppx(self) -> int:
+
+        px_vecs = self.pixelVectors()[0]
+        if px_vecs:
+            xs_in_px = px_vecs.x()
+            return round(xs_in_px)
+        else:
+            return 0
+
+    def px_width(self) -> float:
+
+        vb = self.getViewBox()
+        if not vb:
+            return 0
+
+        vr = self.viewRect()
+        l, r = int(vr.left()), int(vr.right())
+
+        start, stop = self._xrange
+        lbar = max(l, start)
+        rbar = min(r, stop)
+
+        return vb.mapViewToDevice(
+            QLineF(lbar, 0, rbar, 0)
+        ).length()
+
+    # XXX: lol brutal, the internals of `CurvePoint` (inherited by
+    # our `LineDot`) required ``.getData()`` to work..
+    def getData(self):
+        return self.xData, self.yData
+
+    def clear(self):
+        '''
+        Clear internal graphics making object ready for full re-draw.
+
+        '''
+        # NOTE: original code from ``pg.PlotCurveItem``
+        self.xData = None
+        self.yData = None
+
+        # XXX: previously, if not trying to leverage `.reserve()` allocs
+        # then you might as well create a new one..
+        # self.path = None
+
+        # path reservation aware non-mem de-alloc cleaning
+        if self.path:
+            self.path.clear()
+
+            if self.fast_path:
+                # self.fast_path.clear()
+                self.fast_path = None
+
+    @cm
+    def reset_cache(self) -> None:
+        self.setCacheMode(QtWidgets.QGraphicsItem.NoCache)
+        yield
+        self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
+
+    def boundingRect(self):
+        '''
+        Compute and then cache our rect.
+        '''
+        if self.path is None:
+            return QPainterPath().boundingRect()
+        else:
+            # dynamically override this method after initial
+            # path is created to avoid requiring the above None check
+            self.boundingRect = self._path_br
+            return self._path_br()
+
+    def _path_br(self):
+        '''
+        Post init ``.boundingRect()```.
+
+        '''
+        # hb = self.path.boundingRect()
+        hb = self.path.controlPointRect()
+        hb_size = hb.size()
+
+        fp = self.fast_path
+        if fp:
+            fhb = fp.controlPointRect()
+            hb_size = fhb.size() + hb_size
+
+        # print(f'hb_size: {hb_size}')
+
+        # if self._last_step_rect:
+        #     hb_size += self._last_step_rect.size()
+
+        # if self._line:
+        #     br = self._last_step_rect.bottomRight()
+
+        # tl = QPointF(
+        #     # self._vr[0],
+        #     # hb.topLeft().y(),
+        #     # 0,
+        #     # hb_size.height() + 1
+        # )
+
+        #     br = self._last_step_rect.bottomRight()
+
+        w = hb_size.width()
+        h = hb_size.height()
+
+        sbr = self.sub_br
+        if sbr:
+            w, h = self.sub_br(w, h)
+        else:
+            # assume plain line graphic and use
+            # default unit step in each direction.
+
+            # only on a plane line do we include
+            # and extra index step's worth of width
+            # since in the step case the end of the curve
+            # actually terminates earlier so we don't need
+            # this for the last step.
+            w += self._last_w
+            # ll = self._last_line
+            h += 1  # ll.y2() - ll.y1()
+
+        # br = QPointF(
+        #     self._vr[-1],
+        #     # tl.x() + w,
+        #     tl.y() + h,
+        # )
+
+        br = QRectF(
+
+            # top left
+            # hb.topLeft()
+            # tl,
+            QPointF(hb.topLeft()),
+
+            # br,
+            # total size
+            # QSizeF(hb_size)
+            # hb_size,
+            QSizeF(w, h)
+        )
+        # print(f'bounding rect: {br}')
+        return br
+
+    def paint(
+        self,
+        p: QPainter,
+        opt: QtWidgets.QStyleOptionGraphicsItem,
+        w: QtWidgets.QWidget
+
+    ) -> None:
+
+        profiler = pg.debug.Profiler(
+            msg=f'Curve.paint(): `{self._name}`',
+            disabled=not pg_profile_enabled(),
+            ms_threshold=ms_slower_then,
+        )
+
+        sub_paint = self.sub_paint
+        if sub_paint:
+            sub_paint(p, profiler)
+
+        p.setPen(self.last_step_pen)
+        p.drawLine(self._last_line)
+        profiler('.drawLine()')
+        p.setPen(self._pen)
+
+        path = self.path
+        # cap = path.capacity()
+        # if cap != self._last_cap:
+        #     print(f'NEW CAPACITY: {self._last_cap} -> {cap}')
+        #     self._last_cap = cap
+
+        if path:
+            p.drawPath(path)
+            profiler(f'.drawPath(path): {path.capacity()}')
+
+        fp = self.fast_path
+        if fp:
+            p.drawPath(fp)
+            profiler('.drawPath(fast_path)')
+
+        # TODO: try out new work from `pyqtgraph` main which should
+        # repair horrid perf (pretty sure i did and it was still
+        # horrible?):
+        # https://github.com/pyqtgraph/pyqtgraph/pull/2032
+        # if self._fill:
+        #     brush = self.opts['brush']
+        #     p.fillPath(self.path, brush)
+
+    def draw_last_datum(
+        self,
+        path: QPainterPath,
+        src_data: np.ndarray,
+        render_data: np.ndarray,
+        reset: bool,
+        array_key: str,
+
+    ) -> None:
+        # default line draw last call
+        # with self.reset_cache():
+        x = render_data['index']
+        y = render_data[array_key]
+
+        # draw the "current" step graphic segment so it
+        # lines up with the "middle" of the current
+        # (OHLC) sample.
+        self._last_line = QLineF(
+            x[-2], y[-2],
+            x[-1], y[-1],
+        )
+
+        return x, y
+
+
+# TODO: this should probably be a "downsampled" curve type
+# that draws a bar-style (but for the px column) last graphics
+# element such that the current datum in view can be shown
+# (via it's max / min) even when highly zoomed out.
+class FlattenedOHLC(Curve):
+
+    def draw_last_datum(
+        self,
+        path: QPainterPath,
+        src_data: np.ndarray,
+        render_data: np.ndarray,
+        reset: bool,
+        array_key: str,
+
+    ) -> None:
+        lasts = src_data[-2:]
+        x = lasts['index']
+        y = lasts['close']
+
+        # draw the "current" step graphic segment so it
+        # lines up with the "middle" of the current
+        # (OHLC) sample.
+        self._last_line = QLineF(
+            x[-2], y[-2],
+            x[-1], y[-1]
+        )
+        return x, y
+
+
+class StepCurve(Curve):
+
+    def declare_paintables(
+        self,
+    ) -> None:
+        self._last_step_rect = QRectF()
+
+    def draw_last_datum(
+        self,
+        path: QPainterPath,
+        src_data: np.ndarray,
+        render_data: np.ndarray,
+        reset: bool,
+        array_key: str,
+
+        w: float = 0.5,
+
+    ) -> None:
+
+        # TODO: remove this and instead place all step curve
+        # updating into pre-path data render callbacks.
+        # full input data
+        x = src_data['index']
+        y = src_data[array_key]
+
+        x_last = x[-1]
+        y_last = y[-1]
+
+        # lol, commenting this makes step curves
+        # all "black" for me :eyeroll:..
+        self._last_line = QLineF(
+            x_last - w, 0,
+            x_last + w, 0,
+        )
+        self._last_step_rect = QRectF(
+            x_last - w, 0,
+            x_last + w, y_last,
+        )
+        return x, y
+
+    def sub_paint(
+        self,
+        p: QPainter,
+        profiler: pg.debug.Profiler,
+
+    ) -> None:
+        # p.drawLines(*tuple(filter(bool, self._last_step_lines)))
+        # p.drawRect(self._last_step_rect)
+        p.fillRect(self._last_step_rect, self._brush)
+        profiler('.fillRect()')
+
+    def sub_br(
+        self,
+        path_w: float,
+        path_h: float,
+
+    ) -> (float, float):
+        # passthrough
+        return path_w, path_h

piker/ui/_display.py

@@ -0,0 +1,863 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+real-time display tasks for charting graphics update.
+
+this module ties together quote and computational (fsp) streams with
+graphics update methods via our custom ``pyqtgraph`` charting api.
+
+'''
+from dataclasses import dataclass
+from functools import partial
+import time
+from typing import Optional, Any, Callable
+
+import numpy as np
+import tractor
+import trio
+import pendulum
+import pyqtgraph as pg
+
+# from .. import brokers
+from ..data.feed import open_feed
+from ._axes import YAxisLabel
+from ._chart import (
+    ChartPlotWidget,
+    LinkedSplits,
+    GodWidget,
+)
+from ._l1 import L1Labels
+from ._fsp import (
+    update_fsp_chart,
+    start_fsp_displays,
+    has_vlm,
+    open_vlm_displays,
+)
+from ..data._sharedmem import ShmArray
+from ..data._source import tf_in_1s
+from ._forms import (
+    FieldsForm,
+    mk_order_pane_layout,
+)
+from .order_mode import open_order_mode
+from .._profile import (
+    pg_profile_enabled,
+    ms_slower_then,
+)
+from ..log import get_logger
+
+log = get_logger(__name__)
+
+# TODO: load this from a config.toml!
+_quote_throttle_rate: int = 22  # Hz
+
+
+# a working tick-type-classes template
+_tick_groups = {
+    'clears': {'trade', 'utrade', 'last'},
+    'bids': {'bid', 'bsize'},
+    'asks': {'ask', 'asize'},
+}
+
+
+# TODO: delegate this to each `Flow.maxmin()` which includes
+# caching and further we should implement the following stream based
+# approach, likely with ``numba``:
+# https://arxiv.org/abs/cs/0610046
+# https://github.com/lemire/pythonmaxmin
+def chart_maxmin(
+    chart: ChartPlotWidget,
+    ohlcv_shm: ShmArray,
+    vlm_chart: Optional[ChartPlotWidget] = None,
+
+) -> tuple[
+
+    tuple[int, int, int, int],
+
+    float,
+    float,
+    float,
+]:
+    '''
+    Compute max and min datums "in view" for range limits.
+
+    '''
+    last_bars_range = chart.bars_range()
+    out = chart.maxmin()
+
+    if out is None:
+        return (last_bars_range, 0, 0, 0)
+
+    mn, mx = out
+
+    mx_vlm_in_view = 0
+    if vlm_chart:
+        out = vlm_chart.maxmin()
+        if out:
+            _, mx_vlm_in_view = out
+
+    return (
+        last_bars_range,
+        mx,
+        max(mn, 0),  # presuming price can't be negative?
+        mx_vlm_in_view,
+    )
+
+
+@dataclass
+class DisplayState:
+    '''
+    Chart-local real-time graphics state container.
+
+    '''
+    quotes: dict[str, Any]
+
+    maxmin: Callable
+    ohlcv: ShmArray
+
+    # high level chart handles
+    linked: LinkedSplits
+    chart: ChartPlotWidget
+    vlm_chart: ChartPlotWidget
+
+    # axis labels
+    l1: L1Labels
+    last_price_sticky: YAxisLabel
+    vlm_sticky: YAxisLabel
+
+    # misc state tracking
+    vars: dict[str, Any]
+
+    wap_in_history: bool = False
+
+
+async def graphics_update_loop(
+
+    linked: LinkedSplits,
+    stream: tractor.MsgStream,
+    ohlcv: np.ndarray,
+
+    wap_in_history: bool = False,
+    vlm_chart: Optional[ChartPlotWidget] = None,
+
+) -> None:
+    '''
+    The 'main' (price) chart real-time update loop.
+
+    Receive from the primary instrument quote stream and update the OHLC
+    chart.
+
+    '''
+    # TODO: bunch of stuff (some might be done already, can't member):
+    # - I'm starting to think all this logic should be
+    #   done in one place and "graphics update routines"
+    #   should not be doing any length checking and array diffing.
+    # - handle odd lot orders
+    # - update last open price correctly instead
+    #   of copying it from last bar's close
+    # - 1-5 sec bar lookback-autocorrection like tws does?
+    #   (would require a background history checker task)
+    display_rate = linked.godwidget.window.current_screen().refreshRate()
+
+    chart = linked.chart
+
+    # update last price sticky
+    last_price_sticky = chart._ysticks[chart.name]
+    last_price_sticky.update_from_data(
+        *ohlcv.array[-1][['index', 'close']]
+    )
+
+    if vlm_chart:
+        vlm_sticky = vlm_chart._ysticks['volume']
+
+    maxmin = partial(
+        chart_maxmin,
+        chart,
+        ohlcv,
+        vlm_chart,
+    )
+    last_bars_range: tuple[float, float]
+    (
+        last_bars_range,
+        last_mx,
+        last_mn,
+        last_mx_vlm,
+    ) = maxmin()
+
+    last, volume = ohlcv.array[-1][['close', 'volume']]
+
+    symbol = chart.linked.symbol
+
+    l1 = L1Labels(
+        chart,
+        # determine precision/decimal lengths
+        digits=symbol.tick_size_digits,
+        size_digits=symbol.lot_size_digits,
+    )
+    chart._l1_labels = l1
+
+    # TODO:
+    # - in theory we should be able to read buffer data faster
+    #   then msgs arrive.. needs some tinkering and testing
+
+    # - if trade volume jumps above / below prior L1 price
+    #   levels this might be dark volume we need to
+    #   present differently -> likely dark vlm
+
+    tick_size = chart.linked.symbol.tick_size
+    tick_margin = 3 * tick_size
+
+    chart.show()
+    # view = chart.view
+    last_quote = time.time()
+    i_last = ohlcv.index
+
+    # async def iter_drain_quotes():
+    #     # NOTE: all code below this loop is expected to be synchronous
+    #     # and thus draw instructions are not picked up jntil the next
+    #     # wait / iteration.
+    #     async for quotes in stream:
+    #         while True:
+    #             try:
+    #                 moar = stream.receive_nowait()
+    #             except trio.WouldBlock:
+    #                 yield quotes
+    #                 break
+    #             else:
+    #                 for sym, quote in moar.items():
+    #                     ticks_frame = quote.get('ticks')
+    #                     if ticks_frame:
+    #                         quotes[sym].setdefault(
+    #                             'ticks', []).extend(ticks_frame)
+    #                     print('pulled extra')
+
+    #                 yield quotes
+
+    # async for quotes in iter_drain_quotes():
+
+    ds = linked.display_state = DisplayState(**{
+        'quotes': {},
+        'linked': linked,
+        'maxmin': maxmin,
+        'ohlcv': ohlcv,
+        'chart': chart,
+        'last_price_sticky': last_price_sticky,
+        'vlm_chart': vlm_chart,
+        'vlm_sticky': vlm_sticky,
+        'l1': l1,
+
+        'vars': {
+            'tick_margin': tick_margin,
+            'i_last': i_last,
+            'i_last_append': i_last,
+            'last_mx_vlm': last_mx_vlm,
+            'last_mx': last_mx,
+            'last_mn': last_mn,
+        }
+    })
+
+    chart.default_view()
+
+    # main real-time quotes update loop
+    async for quotes in stream:
+
+        ds.quotes = quotes
+        quote_period = time.time() - last_quote
+        quote_rate = round(
+            1/quote_period, 1) if quote_period > 0 else float('inf')
+        if (
+            quote_period <= 1/_quote_throttle_rate
+
+            # in the absolute worst case we shouldn't see more then
+            # twice the expected throttle rate right!?
+            # and quote_rate >= _quote_throttle_rate * 2
+            and quote_rate >= display_rate
+        ):
+            log.warning(f'High quote rate {symbol.key}: {quote_rate}')
+
+        last_quote = time.time()
+
+        # chart isn't active/shown so skip render cycle and pause feed(s)
+        if chart.linked.isHidden():
+            chart.pause_all_feeds()
+            continue
+
+        ic = chart.view._ic
+        if ic:
+            chart.pause_all_feeds()
+            await ic.wait()
+            chart.resume_all_feeds()
+
+        # sync call to update all graphics/UX components.
+        graphics_update_cycle(ds)
+
+
+def graphics_update_cycle(
+    ds: DisplayState,
+    wap_in_history: bool = False,
+    trigger_all: bool = False,  # flag used by prepend history updates
+    prepend_update_index: Optional[int] = None,
+
+) -> None:
+    # TODO: eventually optimize this whole graphics stack with ``numba``
+    # hopefully XD
+
+    chart = ds.chart
+
+    profiler = pg.debug.Profiler(
+        msg=f'Graphics loop cycle for: `{chart.name}`',
+        delayed=True,
+        disabled=not pg_profile_enabled(),
+        # disabled=True,
+        ms_threshold=ms_slower_then,
+
+        # ms_threshold=1/12 * 1e3,
+    )
+
+    # unpack multi-referenced components
+    vlm_chart = ds.vlm_chart
+    l1 = ds.l1
+    ohlcv = ds.ohlcv
+    array = ohlcv.array
+    vars = ds.vars
+    tick_margin = vars['tick_margin']
+
+    update_uppx = 16
+
+    for sym, quote in ds.quotes.items():
+
+        # compute the first available graphic's x-units-per-pixel
+        uppx = vlm_chart.view.x_uppx()
+
+        # NOTE: vlm may be written by the ``brokerd`` backend
+        # event though a tick sample is not emitted.
+        # TODO: show dark trades differently
+        # https://github.com/pikers/piker/issues/116
+
+        # NOTE: this used to be implemented in a dedicated
+        # "increment task": ``check_for_new_bars()`` but it doesn't
+        # make sense to do a whole task switch when we can just do
+        # this simple index-diff and all the fsp sub-curve graphics
+        # are diffed on each draw cycle anyway; so updates to the
+        # "curve" length is already automatic.
+
+        # increment the view position by the sample offset.
+        i_step = ohlcv.index
+        i_diff = i_step - vars['i_last']
+        vars['i_last'] = i_step
+
+        append_diff = i_step - vars['i_last_append']
+
+        # update the "last datum" (aka extending the flow graphic with
+        # new data) only if the number of unit steps is >= the number of
+        # such unit steps per pixel (aka uppx). Iow, if the zoom level
+        # is such that a datum(s) update to graphics wouldn't span
+        # to a new pixel, we don't update yet.
+        do_append = (append_diff >= uppx)
+        if do_append:
+            vars['i_last_append'] = i_step
+
+        do_rt_update = uppx < update_uppx
+        # print(
+        #     f'append_diff:{append_diff}\n'
+        #     f'uppx:{uppx}\n'
+        #     f'do_append: {do_append}'
+        # )
+
+        # TODO: we should only run mxmn when we know
+        # an update is due via ``do_append`` above.
+        (
+            brange,
+            mx_in_view,
+            mn_in_view,
+            mx_vlm_in_view,
+        ) = ds.maxmin()
+        l, lbar, rbar, r = brange
+        mx = mx_in_view + tick_margin
+        mn = mn_in_view - tick_margin
+
+        profiler('`ds.maxmin()` call')
+
+        liv = r >= i_step  # the last datum is in view
+
+        if (
+            prepend_update_index is not None
+            and lbar > prepend_update_index
+        ):
+            # on a history update (usually from the FSP subsys)
+            # if the segment of history that is being prepended
+            # isn't in view there is no reason to do a graphics
+            # update.
+            log.debug('Skipping prepend graphics cycle: frame not in view')
+            return
+
+        # don't real-time "shift" the curve to the
+        # left unless we get one of the following:
+        if (
+            (
+                # i_diff > 0  # no new sample step
+                do_append
+                # and uppx < 4  # chart is zoomed out very far
+                and liv
+            )
+            or trigger_all
+        ):
+            # TODO: we should track and compute whether the last
+            # pixel in a curve should show new data based on uppx
+            # and then iff update curves and shift?
+            chart.increment_view(steps=i_diff)
+
+            if vlm_chart:
+                vlm_chart.increment_view(steps=i_diff)
+
+            profiler('view incremented')
+
+        ticks_frame = quote.get('ticks', ())
+
+        frames_by_type: dict[str, dict] = {}
+        lasts = {}
+
+        # build tick-type "frames" of tick sequences since
+        # likely the tick arrival rate is higher then our
+        # (throttled) quote stream rate.
+        for tick in ticks_frame:
+            price = tick.get('price')
+            ticktype = tick.get('type')
+
+            if ticktype == 'n/a' or price == -1:
+                # okkk..
+                continue
+
+            # keys are entered in olded-event-inserted-first order
+            # since we iterate ``ticks_frame`` in standard order
+            # above. in other words the order of the keys is the order
+            # of tick events by type from the provider feed.
+            frames_by_type.setdefault(ticktype, []).append(tick)
+
+            # overwrites so the last tick per type is the entry
+            lasts[ticktype] = tick
+
+        # from pprint import pformat
+        # frame_counts = {
+        #     typ: len(frame) for typ, frame in frames_by_type.items()
+        # }
+        # print(f'{pformat(frame_counts)}')
+        # print(f'framed: {pformat(frames_by_type)}')
+        # print(f'lasts: {pformat(lasts)}')
+
+        # TODO: eventually we want to separate out the utrade (aka
+        # dark vlm prices) here and show them as an additional
+        # graphic.
+        clear_types = _tick_groups['clears']
+
+        # XXX: if we wanted to iterate in "latest" (i.e. most
+        # current) tick first order as an optimization where we only
+        # update from the last tick from each type class.
+        # last_clear_updated: bool = False
+
+        # update ohlc sampled price bars
+        if (
+            do_rt_update
+            or do_append
+            or trigger_all
+        ):
+            chart.update_graphics_from_flow(
+                chart.name,
+                # do_append=uppx < update_uppx,
+                do_append=do_append,
+            )
+
+        # NOTE: we always update the "last" datum
+        # since the current range should at least be updated
+        # to it's max/min on the last pixel.
+
+        # iterate in FIFO order per tick-frame
+        for typ, tick in lasts.items():
+
+            price = tick.get('price')
+            size = tick.get('size')
+
+            # compute max and min prices (including bid/ask) from
+            # tick frames to determine the y-range for chart
+            # auto-scaling.
+            # TODO: we need a streaming minmax algo here, see def above.
+            if liv:
+                mx = max(price + tick_margin, mx)
+                mn = min(price - tick_margin, mn)
+
+            if typ in clear_types:
+
+                # XXX: if we only wanted to update graphics from the
+                # "current"/"latest received" clearing price tick
+                # once (see alt iteration order above).
+                # if last_clear_updated:
+                #     continue
+
+                # last_clear_updated = True
+                # we only want to update grahpics from the *last*
+                # tick event that falls under the "clearing price"
+                # set.
+
+                # update price sticky(s)
+                end = array[-1]
+                ds.last_price_sticky.update_from_data(
+                    *end[['index', 'close']]
+                )
+
+                if wap_in_history:
+                    # update vwap overlay line
+                    chart.update_graphics_from_flow(
+                        'bar_wap',
+                    )
+
+            # L1 book label-line updates
+            # XXX: is this correct for ib?
+            # if ticktype in ('trade', 'last'):
+            # if ticktype in ('last',):  # 'size'):
+            if typ in ('last',):  # 'size'):
+
+                label = {
+                    l1.ask_label.fields['level']: l1.ask_label,
+                    l1.bid_label.fields['level']: l1.bid_label,
+                }.get(price)
+
+                if (
+                    label is not None
+                    and liv
+                ):
+                    label.update_fields(
+                        {'level': price, 'size': size}
+                    )
+
+                    # TODO: on trades should we be knocking down
+                    # the relevant L1 queue?
+                    # label.size -= size
+
+            elif (
+                typ in _tick_groups['asks']
+                # TODO: instead we could check if the price is in the
+                # y-view-range?
+                and liv
+            ):
+                l1.ask_label.update_fields({'level': price, 'size': size})
+
+            elif (
+                typ in _tick_groups['bids']
+                # TODO: instead we could check if the price is in the
+                # y-view-range?
+                and liv
+            ):
+                l1.bid_label.update_fields({'level': price, 'size': size})
+
+        # check for y-range re-size
+        if (
+            (mx > vars['last_mx']) or (mn < vars['last_mn'])
+            and not chart._static_yrange == 'axis'
+            and liv
+        ):
+            main_vb = chart.view
+            if (
+                main_vb._ic is None
+                or not main_vb._ic.is_set()
+            ):
+                # print(f'updating range due to mxmn')
+                main_vb._set_yrange(
+                    # TODO: we should probably scale
+                    # the view margin based on the size
+                    # of the true range? This way you can
+                    # slap in orders outside the current
+                    # L1 (only) book range.
+                    # range_margin=0.1,
+                    yrange=(mn, mx),
+                )
+
+        # XXX: update this every draw cycle to make L1-always-in-view work.
+        vars['last_mx'], vars['last_mn'] = mx, mn
+
+        # run synchronous update on all linked flows
+        # TODO: should the "main" (aka source) flow be special?
+        for curve_name, flow in chart._flows.items():
+            # update any overlayed fsp flows
+            if curve_name != chart.data_key:
+                update_fsp_chart(
+                    chart,
+                    flow,
+                    curve_name,
+                    array_key=curve_name,
+                )
+
+            # even if we're downsampled bigly
+            # draw the last datum in the final
+            # px column to give the user the mx/mn
+            # range of that set.
+            if (
+                not do_append
+                # and not do_rt_update
+                and liv
+            ):
+                flow.draw_last(
+                    array_key=curve_name,
+                    only_last_uppx=True,
+                )
+
+        # volume chart logic..
+        # TODO: can we unify this with the above loop?
+        if vlm_chart:
+            # always update y-label
+            ds.vlm_sticky.update_from_data(
+                *array[-1][['index', 'volume']]
+            )
+
+            if (
+                (
+                    do_rt_update
+                    or do_append
+                    and liv
+                )
+                or trigger_all
+            ):
+                # TODO: make it so this doesn't have to be called
+                # once the $vlm is up?
+                vlm_chart.update_graphics_from_flow(
+                    'volume',
+                    # UGGGh, see ``maxmin()`` impl in `._fsp` for
+                    # the overlayed plotitems... we need a better
+                    # bay to invoke a maxmin per overlay..
+                    render=False,
+                    # XXX: ^^^^ THIS IS SUPER IMPORTANT! ^^^^
+                    # without this, since we disable the
+                    # 'volume' (units) chart after the $vlm starts
+                    # up we need to be sure to enable this
+                    # auto-ranging otherwise there will be no handler
+                    # connected to update accompanying overlay
+                    # graphics..
+                )
+                profiler('`vlm_chart.update_graphics_from_flow()`')
+
+                if (
+                    mx_vlm_in_view != vars['last_mx_vlm']
+                ):
+                    yrange = (0, mx_vlm_in_view * 1.375)
+                    vlm_chart.view._set_yrange(
+                        yrange=yrange,
+                    )
+                    profiler('`vlm_chart.view._set_yrange()`')
+                    # print(f'mx vlm: {last_mx_vlm} -> {mx_vlm_in_view}')
+                    vars['last_mx_vlm'] = mx_vlm_in_view
+
+            for curve_name, flow in vlm_chart._flows.items():
+
+                if (
+                    curve_name != 'volume' and
+                    flow.render and (
+                        liv and
+                        do_rt_update or do_append
+                    )
+                ):
+                    update_fsp_chart(
+                        vlm_chart,
+                        flow,
+                        curve_name,
+                        array_key=curve_name,
+                        # do_append=uppx < update_uppx,
+                        do_append=do_append,
+                    )
+                    # is this even doing anything?
+                    # (pretty sure it's the real-time
+                    # resizing from last quote?)
+                    fvb = flow.plot.vb
+                    fvb._set_yrange(
+                        name=curve_name,
+                    )
+
+                elif (
+                    curve_name != 'volume'
+                    and not do_append
+                    and liv
+                    and uppx >= 1
+                    # even if we're downsampled bigly
+                    # draw the last datum in the final
+                    # px column to give the user the mx/mn
+                    # range of that set.
+                ):
+                    # always update the last datum-element
+                    # graphic for all flows
+                    # print(f'drawing last {flow.name}')
+                    flow.draw_last(array_key=curve_name)
+
+
+async def display_symbol_data(
+    godwidget: GodWidget,
+    provider: str,
+    sym: str,
+    loglevel: str,
+    order_mode_started: trio.Event,
+
+) -> None:
+    '''
+    Spawn a real-time updated chart for ``symbol``.
+
+    Spawned ``LinkedSplits`` chart widgets can remain up but hidden so
+    that multiple symbols can be viewed and switched between extremely
+    fast from a cached watch-list.
+
+    '''
+    sbar = godwidget.window.status_bar
+    loading_sym_key = sbar.open_status(
+        f'loading {sym}.{provider} ->',
+        group_key=True
+    )
+
+    # historical data fetch
+    # brokermod = brokers.get_brokermod(provider)
+
+    # ohlc_status_done = sbar.open_status(
+    #     'retreiving OHLC history.. ',
+    #     clear_on_next=True,
+    #     group_key=loading_sym_key,
+    # )
+    fqsn = '.'.join((sym, provider))
+
+    async with open_feed(
+        [fqsn],
+        loglevel=loglevel,
+
+        # limit to at least display's FPS
+        # avoiding needless Qt-in-guest-mode context switches
+        tick_throttle=_quote_throttle_rate,
+
+    ) as feed:
+        ohlcv: ShmArray = feed.shm
+        bars = ohlcv.array
+        symbol = feed.symbols[sym]
+        fqsn = symbol.front_fqsn()
+
+        times = bars['time']
+        end = pendulum.from_timestamp(times[-1])
+        start = pendulum.from_timestamp(times[times != times[-1]][-1])
+        step_size_s = (end - start).seconds
+        tf_key = tf_in_1s[step_size_s]
+
+        # load in symbol's ohlc data
+        godwidget.window.setWindowTitle(
+            f'{fqsn} '
+            f'tick:{symbol.tick_size} '
+            f'step:{tf_key} '
+        )
+
+        linked = godwidget.linkedsplits
+        linked._symbol = symbol
+
+        # generate order mode side-pane UI
+        # A ``FieldsForm`` form to configure order entry
+        pp_pane: FieldsForm = mk_order_pane_layout(godwidget)
+
+        # add as next-to-y-axis singleton pane
+        godwidget.pp_pane = pp_pane
+
+        # create main OHLC chart
+        chart = linked.plot_ohlc_main(
+            symbol,
+            ohlcv,
+            sidepane=pp_pane,
+        )
+        chart.default_view()
+        chart._feeds[symbol.key] = feed
+        chart.setFocus()
+
+        # plot historical vwap if available
+        wap_in_history = False
+
+        # XXX: FOR SOME REASON THIS IS CAUSING HANGZ!?!
+        # if brokermod._show_wap_in_history:
+
+        #     if 'bar_wap' in bars.dtype.fields:
+        #         wap_in_history = True
+        #         chart.draw_curve(
+        #             name='bar_wap',
+        #             shm=ohlcv,
+        #             color='default_light',
+        #             add_label=False,
+        #         )
+
+        # size view to data once at outset
+        chart.cv._set_yrange()
+
+        # NOTE: we must immediately tell Qt to show the OHLC chart
+        # to avoid a race where the subplots get added/shown to
+        # the linked set *before* the main price chart!
+        linked.show()
+        linked.focus()
+        await trio.sleep(0)
+
+        vlm_chart: Optional[ChartPlotWidget] = None
+        async with trio.open_nursery() as ln:
+
+            # if available load volume related built-in display(s)
+            if has_vlm(ohlcv):
+                vlm_chart = await ln.start(
+                    open_vlm_displays,
+                    linked,
+                    ohlcv,
+                )
+
+            # load (user's) FSP set (otherwise known as "indicators")
+            # from an input config.
+            ln.start_soon(
+                start_fsp_displays,
+                linked,
+                ohlcv,
+                loading_sym_key,
+                loglevel,
+            )
+
+            # start graphics update loop after receiving first live quote
+            ln.start_soon(
+                graphics_update_loop,
+                linked,
+                feed.stream,
+                ohlcv,
+                wap_in_history,
+                vlm_chart,
+            )
+
+            async with (
+                open_order_mode(
+                    feed,
+                    chart,
+                    fqsn,
+                    order_mode_started
+                )
+            ):
+                # let Qt run to render all widgets and make sure the
+                # sidepanes line up vertically.
+                await trio.sleep(0)
+                linked.resize_sidepanes()
+
+                # NOTE: we pop the volume chart from the subplots set so
+                # that it isn't double rendered in the display loop
+                # above since we do a maxmin calc on the volume data to
+                # determine if auto-range adjustements should be made.
+                # linked.subplots.pop('volume', None)
+
+                # TODO: make this not so shit XD
+                # close group status
+                sbar._status_groups[loading_sym_key][1]()
+
+                # let the app run.. bby
+                # linked.graphics_cycle()
+                await trio.sleep_forever()

piker/ui/_editors.py

@@ -28,7 +28,7 @@ from PyQt5.QtCore import QPointF
 import numpy as np
 
 from ._style import hcolor, _font
-from ._graphics._lines import order_line, LevelLine
+from ._lines import LevelLine
 from ..log import get_logger
 
 
@@ -97,69 +97,21 @@ class LineEditor:
 
     def stage_line(
         self,
-        action: str,
+        line: LevelLine,
 
-        color: str = 'alert_yellow',
-        hl_on_hover: bool = False,
-        dotted: bool = False,
-
-        # fields settings
-        size: Optional[int] = None,
     ) -> LevelLine:
         """Stage a line at the current chart's cursor position
         and return it.
 
         """
-        # chart.setCursor(QtCore.Qt.PointingHandCursor)
-        cursor = self.chart.linked.cursor
-        if not cursor:
-            return None
-
-        chart = cursor.active_plot
-        y = cursor._datum_xy[1]
-
-        symbol = chart._lc.symbol
 
         # add a "staged" cursor-tracking line to view
         # and cash it in a a var
         if self._active_staged_line:
             self.unstage_line()
 
-        line = order_line(
-            chart,
-
-            level=y,
-            level_digits=symbol.digits(),
-            size=size,
-            size_digits=symbol.lot_digits(),
-
-            # just for the stage line to avoid
-            # flickering while moving the cursor
-            # around where it might trigger highlight
-            # then non-highlight depending on sensitivity
-            always_show_labels=True,
-
-            # kwargs
-            color=color,
-            # don't highlight the "staging" line
-            hl_on_hover=hl_on_hover,
-            dotted=dotted,
-            exec_type='dark' if dotted else 'live',
-            action=action,
-            show_markers=True,
-
-            # prevent flickering of marker while moving/tracking cursor
-            only_show_markers_on_hover=False,
-        )
-
         self._active_staged_line = line
 
-        # hide crosshair y-line and label
-        cursor.hide_xhair()
-
-        # add line to cursor trackers
-        cursor._trackers.add(line)
-
         return line
 
     def unstage_line(self) -> LevelLine:
@@ -181,41 +133,17 @@ class LineEditor:
         # show the crosshair y line and label
         cursor.show_xhair()
 
-    def create_order_line(
+    def submit_line(
         self,
+        line: LevelLine,
         uuid: str,
-        level: float,
-        chart: 'ChartPlotWidget',  # noqa
-        size: float,
-        action: str,
+
     ) -> LevelLine:
 
-        line = self._active_staged_line
-        if not line:
+        staged_line = self._active_staged_line
+        if not staged_line:
             raise RuntimeError("No line is currently staged!?")
 
-        sym = chart._lc.symbol
-
-        line = order_line(
-            chart,
-
-            # label fields default values
-            level=level,
-            level_digits=sym.digits(),
-
-            size=size,
-            size_digits=sym.lot_digits(),
-
-            # LevelLine kwargs
-            color=line.color,
-            dotted=line._dotted,
-
-            show_markers=True,
-            only_show_markers_on_hover=True,
-
-            action=action,
-        )
-
         # for now, until submission reponse arrives
         line.hide_labels()
 
@@ -237,7 +165,6 @@ class LineEditor:
             log.warning(f'No line for {uuid} could be found?')
             return
         else:
-            assert line.oid == uuid
             line.show_labels()
 
             # TODO: other flashy things to indicate the order is active
@@ -260,18 +187,16 @@ class LineEditor:
         self,
         line: LevelLine = None,
         uuid: str = None,
-    ) -> LevelLine:
-        """Remove a line by refernce or uuid.
+
+    ) -> Optional[LevelLine]:
+        '''Remove a line by refernce or uuid.
 
         If no lines or ids are provided remove all lines under the
         cursor position.
 
-        """
-        if line:
-            uuid = line.oid
-
+        '''
         # try to look up line from our registry
-        line = self._order_lines.pop(uuid, None)
+        line = self._order_lines.pop(uuid, line)
         if line:
 
             # if hovered remove from cursor set
@@ -284,8 +209,13 @@ class LineEditor:
                 # just because we never got a un-hover event
                 cursor.show_xhair()
 
+            log.debug(f'deleting {line} with oid: {uuid}')
             line.delete()
-            return line
+
+        else:
+            log.warning(f'Could not find line for {line}')
+
+        return line
 
 
 class SelectRect(QtGui.QGraphicsRectItem):
@@ -412,7 +342,8 @@ class SelectRect(QtGui.QGraphicsRectItem):
         ixmn, ixmx = round(xmn), round(xmx)
         nbars = ixmx - ixmn + 1
 
-        data = self._chart._arrays['ohlc'][ixmn:ixmx]
+        chart = self._chart
+        data = chart._flows[chart.name].shm.array[ixmn:ixmx]
 
         if len(data):
             std = data['close'].std()

piker/ui/_event.py

@@ -18,13 +18,66 @@
 Qt event proxying and processing using ``trio`` mem chans.
 
 """
-from contextlib import asynccontextmanager
+from contextlib import asynccontextmanager, AsyncExitStack
 from typing import Callable
 
-from PyQt5 import QtCore
-from PyQt5.QtCore import QEvent
-from PyQt5.QtWidgets import QWidget
+from pydantic import BaseModel
 import trio
+from PyQt5 import QtCore
+from PyQt5.QtCore import QEvent, pyqtBoundSignal
+from PyQt5.QtWidgets import QWidget
+from PyQt5.QtWidgets import (
+    QGraphicsSceneMouseEvent as gs_mouse,
+)
+
+
+MOUSE_EVENTS = {
+    gs_mouse.GraphicsSceneMousePress,
+    gs_mouse.GraphicsSceneMouseRelease,
+    QEvent.MouseButtonPress,
+    QEvent.MouseButtonRelease,
+    # QtGui.QMouseEvent,
+}
+
+
+# TODO: maybe consider some constrained ints down the road?
+# https://pydantic-docs.helpmanual.io/usage/types/#constrained-types
+
+class KeyboardMsg(BaseModel):
+    '''Unpacked Qt keyboard event data.
+
+    '''
+    class Config:
+        arbitrary_types_allowed = True
+
+    event: QEvent
+    etype: int
+    key: int
+    mods: int
+    txt: str
+
+    def to_tuple(self) -> tuple:
+        return tuple(self.dict().values())
+
+
+class MouseMsg(BaseModel):
+    '''Unpacked Qt keyboard event data.
+
+    '''
+    class Config:
+        arbitrary_types_allowed = True
+
+    event: QEvent
+    etype: int
+    button: int
+
+
+# TODO: maybe add some methods to detect key combos? Or is that gonna be
+# better with pattern matching?
+# # ctl + alt as combo
+# ctlalt = False
+# if (QtCore.Qt.AltModifier | QtCore.Qt.ControlModifier) == mods:
+#     ctlalt = True
 
 
 class EventRelay(QtCore.QObject):
@@ -38,8 +91,10 @@ class EventRelay(QtCore.QObject):
 
     def eventFilter(
         self,
+
         source: QWidget,
         ev: QEvent,
+
     ) -> None:
         '''
         Qt global event filter: return `False` to pass through and `True`
@@ -50,49 +105,57 @@ class EventRelay(QtCore.QObject):
 
         '''
         etype = ev.type()
-        # print(f'etype: {etype}')
+        # TODO: turn this on and see what we can filter by default (such
+        # as mouseWheelEvent).
+        # print(f'ev: {ev}')
 
-        if etype in self._event_types:
-            # ev.accept()
-
-            # TODO: what's the right way to allow this?
-            # if ev.isAutoRepeat():
-            #     ev.ignore()
-
-            # XXX: we unpack here because apparently doing it
-            # after pop from the mem chan isn't showing the same
-            # event object? no clue wtf is going on there, likely
-            # something to do with Qt internals and calling the
-            # parent handler?
-
-            if etype in {QEvent.KeyPress, QEvent.KeyRelease}:
-
-                # TODO: is there a global setting for this?
-                if ev.isAutoRepeat() and self._filter_auto_repeats:
-                    ev.ignore()
-                    return True
-
-                key = ev.key()
-                mods = ev.modifiers()
-                txt = ev.text()
-
-                # NOTE: the event object instance coming out
-                # the other side is mutated since Qt resumes event
-                # processing **before** running a ``trio`` guest mode
-                # tick, thus special handling or copying must be done.
-
-                # send elements to async handler
-                self._send_chan.send_nowait((ev, etype, key, mods, txt))
-
-            else:
-                # send event to async handler
-                self._send_chan.send_nowait(ev)
-
-            # **do not** filter out this event
-            # and instead forward to the source widget
+        if etype not in self._event_types:
             return False
 
-        # filter out this event
+        # XXX: we unpack here because apparently doing it
+        # after pop from the mem chan isn't showing the same
+        # event object? no clue wtf is going on there, likely
+        # something to do with Qt internals and calling the
+        # parent handler?
+
+        if etype in {QEvent.KeyPress, QEvent.KeyRelease}:
+
+            msg = KeyboardMsg(
+                event=ev,
+                etype=etype,
+                key=ev.key(),
+                mods=ev.modifiers(),
+                txt=ev.text(),
+            )
+
+            # TODO: is there a global setting for this?
+            if ev.isAutoRepeat() and self._filter_auto_repeats:
+                ev.ignore()
+                # filter out this event and stop it's processing
+                # https://doc.qt.io/qt-5/qobject.html#installEventFilter
+                return True
+
+            # NOTE: the event object instance coming out
+            # the other side is mutated since Qt resumes event
+            # processing **before** running a ``trio`` guest mode
+            # tick, thus special handling or copying must be done.
+
+        elif etype in MOUSE_EVENTS:
+            # print('f mouse event: {ev}')
+            msg = MouseMsg(
+                event=ev,
+                etype=etype,
+                button=ev.button(),
+            )
+
+        else:
+            msg = ev
+
+        # send event-msg to async handler
+        self._send_chan.send_nowait(msg)
+
+        # **do not** filter out this event
+        # and instead forward to the source widget
         # https://doc.qt.io/qt-5/qobject.html#installEventFilter
         return False
 
@@ -124,9 +187,34 @@ async def open_event_stream(
 
 
 @asynccontextmanager
-async def open_handler(
+async def open_signal_handler(
 
-    source_widget: QWidget,
+    signal: pyqtBoundSignal,
+    async_handler: Callable,
+
+) -> trio.abc.ReceiveChannel:
+
+    send, recv = trio.open_memory_channel(0)
+
+    def proxy_args_to_chan(*args):
+        send.send_nowait(args)
+
+    signal.connect(proxy_args_to_chan)
+
+    async def proxy_to_handler():
+        async for args in recv:
+            await async_handler(*args)
+
+    async with trio.open_nursery() as n:
+        n.start_soon(proxy_to_handler)
+        async with send:
+            yield
+
+
+@asynccontextmanager
+async def open_handlers(
+
+    source_widgets: list[QWidget],
     event_types: set[QEvent],
     async_handler: Callable[[QWidget, trio.abc.ReceiveChannel], None],
     **kwargs,
@@ -135,7 +223,13 @@ async def open_handler(
 
     async with (
         trio.open_nursery() as n,
-        open_event_stream(source_widget, event_types, **kwargs) as event_recv_stream,
+        AsyncExitStack() as stack,
     ):
-        n.start_soon(async_handler, source_widget, event_recv_stream)
+        for widget in source_widgets:
+
+            event_recv_stream = await stack.enter_async_context(
+                open_event_stream(widget, event_types, **kwargs)
+            )
+            n.start_soon(async_handler, widget, event_recv_stream)
+
         yield

piker/ui/_exec.py

@@ -49,10 +49,6 @@ from . import _style
 log = get_logger(__name__)
 
 # pyqtgraph global config
-# might as well enable this for now?
-pg.useOpenGL = True
-pg.enableExperimental = True
-
 # engage core tweaks that give us better response
 # latency then the average pg user
 _do_overrides()
@@ -61,7 +57,9 @@ _do_overrides()
 # XXX: pretty sure none of this shit works on linux as per:
 # https://bugreports.qt.io/browse/QTBUG-53022
 # it seems to work on windows.. no idea wtf is up.
+is_windows = False
 if platform.system() == "Windows":
+    is_windows = True
 
     # Proper high DPI scaling is available in Qt >= 5.6.0. This attibute
     # must be set before creating the application
@@ -99,6 +97,9 @@ def run_qtractor(
     # "This is substantially faster than using a signal... for some
     # reason Qt signal dispatch is really slow (and relies on events
     # underneath anyway, so this is strictly less work)."
+
+    # source gist and credit to njs:
+    # https://gist.github.com/njsmith/d996e80b700a339e0623f97f48bcf0cb
     REENTER_EVENT = QtCore.QEvent.Type(QtCore.QEvent.registerEventType())
 
     class ReenterEvent(QtCore.QEvent):
@@ -179,6 +180,8 @@ def run_qtractor(
 
     window.main_widget = main_widget
     window.setCentralWidget(instance)
+    if is_windows:
+        window.configure_to_desktop()
 
     # actually render to screen
     window.show()

piker/ui/_feedstatus.py

@@ -0,0 +1,83 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+Feed status and controls widget(s) for embedding in a UI-pane.
+
+"""
+
+from __future__ import annotations
+from textwrap import dedent
+from typing import TYPE_CHECKING
+
+# from PyQt5.QtCore import Qt
+
+from ._style import _font, _font_small
+# from ..calc import humanize
+from ._label import FormatLabel
+
+if TYPE_CHECKING:
+    from ._chart import ChartPlotWidget
+    from ..data.feed import Feed
+    from ._forms import FieldsForm
+
+
+def mk_feed_label(
+    form: FieldsForm,
+    feed: Feed,
+    chart: ChartPlotWidget,
+
+) -> FormatLabel:
+    '''
+    Generate a label from feed meta-data to be displayed
+    in a UI sidepane.
+
+    TODO: eventually buttons for changing settings over
+    a feed control protocol.
+
+    '''
+    status = feed.status
+    assert status
+
+    msg = dedent("""
+        actor: **{actor_name}**\n
+        |_ @**{host}:{port}**\n
+    """)
+
+    for key, val in status.items():
+        if key in ('host', 'port', 'actor_name'):
+            continue
+        msg += f'\n|_ {key}: **{{{key}}}**\n'
+
+    feed_label = FormatLabel(
+        fmt_str=msg,
+        # |_ streams: **{symbols}**\n
+        font=_font.font,
+        font_size=_font_small.px_size,
+        font_color='default_lightest',
+    )
+
+    # form.vbox.setAlignment(feed_label, Qt.AlignBottom)
+    # form.vbox.setAlignment(Qt.AlignBottom)
+    _ = chart.height() - (
+        form.height() +
+        form.fill_bar.height()
+        # feed_label.height()
+    )
+
+    feed_label.format(**feed.status)
+
+    return feed_label

piker/ui/_forms.py

@@ -0,0 +1,814 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Text entry "forms" widgets (mostly for configuration and UI user input).
+
+'''
+from __future__ import annotations
+from contextlib import asynccontextmanager
+from functools import partial
+from math import floor
+from typing import (
+    Optional, Any, Callable, Awaitable
+)
+
+import trio
+from PyQt5 import QtGui
+from PyQt5.QtCore import QSize, QModelIndex, Qt, QEvent
+from PyQt5.QtWidgets import (
+    QWidget,
+    QLabel,
+    QComboBox,
+    QLineEdit,
+    QHBoxLayout,
+    QVBoxLayout,
+    QFormLayout,
+    QProgressBar,
+    QSizePolicy,
+    QStyledItemDelegate,
+    QStyleOptionViewItem,
+)
+
+from ._event import open_handlers
+from ._icons import mk_icons
+from ._style import hcolor, _font, _font_small, DpiAwareFont
+from ._label import FormatLabel
+
+
+class Edit(QLineEdit):
+
+    def __init__(
+
+        self,
+        parent: QWidget,
+        # parent_chart: QWidget,  # noqa
+        font: DpiAwareFont = _font,
+        width_in_chars: int = None,
+
+    ) -> None:
+
+        # self.setContextMenuPolicy(Qt.CustomContextMenu)
+        # self.customContextMenuRequested.connect(self.show_menu)
+        # self.setStyleSheet(f"font: 18px")
+
+        self.dpi_font = font
+        # self.godwidget = parent_chart
+
+        if width_in_chars:
+            self._chars = int(width_in_chars)
+            x_size_policy = QSizePolicy.Fixed
+
+        else:
+            # chart count which will be used to calculate
+            # width of input field.
+            self._chars: int = 6
+            # fit to surroundingn frame width
+            x_size_policy = QSizePolicy.Expanding
+
+        super().__init__(parent)
+
+        # size it as we specify
+        # https://doc.qt.io/qt-5/qsizepolicy.html#Policy-enum
+        self.setSizePolicy(
+            x_size_policy,
+            QSizePolicy.Fixed,
+        )
+        self.setFont(font.font)
+
+        # witty bit of margin
+        self.setTextMargins(2, 2, 2, 2)
+
+    def sizeHint(self) -> QSize:
+        """
+        Scale edit box to size of dpi aware font.
+
+        """
+        psh = super().sizeHint()
+
+        dpi_font = self.dpi_font
+        psh.setHeight(dpi_font.px_size)
+
+        # make space for ``._chars: int`` for of characters in view
+        # TODO: somehow this math ain't right?
+        chars_w_pxs = dpi_font.boundingRect('0'*self._chars).width()
+        scale = round(dpi_font.scale())
+        psh.setWidth(int(chars_w_pxs * scale))
+        return psh
+
+    def set_width_in_chars(
+        self,
+        chars: int,
+
+    ) -> None:
+        self._chars = chars
+        self.sizeHint()
+        self.update()
+
+    def focus(self) -> None:
+        self.selectAll()
+        self.show()
+        self.setFocus()
+
+
+class FontScaledDelegate(QStyledItemDelegate):
+    '''
+    Super simple view delegate to render text in the same
+    font size as the search widget.
+
+    '''
+    def __init__(
+        self,
+
+        parent=None,
+        font: DpiAwareFont = _font,
+
+    ) -> None:
+
+        super().__init__(parent)
+        self.dpi_font = font
+
+    def sizeHint(
+        self,
+
+        option: QStyleOptionViewItem,
+        index: QModelIndex,
+
+    ) -> QSize:
+
+        # value = index.data()
+        # br = self.dpi_font.boundingRect(value)
+        # w, h = br.width(), br.height()
+        parent = self.parent()
+
+        if getattr(parent, '_max_item_size', None):
+            return QSize(*self.parent()._max_item_size)
+
+        else:
+            return super().sizeHint(option, index)
+
+    # NOTE: hack to display icons on RHS
+    # TODO: is there a way to set this stype option once?
+    # def paint(self, painter, option, index):
+    #     # display icons on RHS
+    #     # https://stackoverflow.com/a/39943629
+    #     option.decorationPosition = QtGui.QStyleOptionViewItem.Right
+    #     option.decorationAlignment = Qt.AlignRight | Qt.AlignVCenter
+    #     QStyledItemDelegate.paint(self, painter, option, index)
+
+
+class Selection(QComboBox):
+
+    def __init__(
+        self,
+        parent=None,
+    ) -> None:
+
+        self._items: dict[str, int] = {}
+        super().__init__(parent=parent)
+        self.setSizeAdjustPolicy(QComboBox.AdjustToContents)
+        # make line edit expand to surrounding frame
+        self.setSizePolicy(
+            QSizePolicy.Expanding,
+            QSizePolicy.Fixed,
+        )
+        view = self.view()
+        view.setUniformItemSizes(True)
+
+        # TODO: this doesn't seem to work for the currently selected item?
+        self.setItemDelegate(FontScaledDelegate(self))
+
+        self.resize()
+
+        self._icons = mk_icons(
+            self.style(),
+            self.iconSize()
+        )
+
+    def set_style(
+        self,
+        color: str,
+        font_size: int,
+
+    ) -> None:
+
+        self.setStyleSheet(
+            f"""QComboBox {{
+                color : {hcolor(color)};
+                font-size : {font_size}px;
+            }}
+            """
+        )
+
+    def resize(
+        self,
+        char: str = 'W',
+
+    ) -> None:
+        br = _font.boundingRect(str(char))
+        _, h = br.width(), int(br.height())
+
+        # TODO: something better then this monkey patch
+        view = self.view()
+
+        # XXX: see size policy settings of line edit
+        # view._max_item_size = w, h
+
+        self.setMinimumHeight(h)  # at least one entry in view
+        view.setMaximumHeight(6*h)  # limit to 6 items max in view
+
+        icon_size = round(h * 0.75)
+        self.setIconSize(QSize(icon_size, icon_size))
+
+    def set_items(
+        self,
+        keys: list[str],
+
+    ) -> None:
+        '''
+        Write keys to the selection verbatim.
+
+        All other items are cleared beforehand.
+
+        '''
+        self.clear()
+        self._items.clear()
+
+        for i, key in enumerate(keys):
+            strkey = str(key)
+            self.insertItem(i, strkey)
+
+            # store map of entry keys to row indexes
+            self._items[strkey] = i
+
+        # compute max item size so that the weird
+        # "style item delegate" thing can then specify
+        # that size on each item...
+        keys.sort()
+        self.resize(keys[-1])
+
+    def set_icon(
+        self,
+        key: str,
+        icon_name: Optional[str],
+
+    ) -> None:
+        self.setItemIcon(
+            self._items[key],
+            self._icons[icon_name],
+        )
+
+    def items(self) -> list[(str, int)]:
+        return list(self._items.items())
+
+    # NOTE: in theory we can put icons on the RHS side with this hackery:
+    # https://stackoverflow.com/a/64256969
+    # def showPopup(self):
+    #     print('show')
+    #     QComboBox.showPopup(self)
+
+    # def hidePopup(self):
+    #     # self.setItemDelegate(FontScaledDelegate(self.parent()))
+    #     print('hide')
+    #     QComboBox.hidePopup(self)
+
+
+# slew of resources which helped get this where it is:
+# https://stackoverflow.com/questions/20648210/qcombobox-adjusttocontents-changing-height
+# https://stackoverflow.com/questions/3151798/how-do-i-set-the-qcombobox-width-to-fit-the-largest-item
+# https://stackoverflow.com/questions/6337589/qlistwidget-adjust-size-to-content#6370892
+# https://stackoverflow.com/questions/25304267/qt-resize-of-qlistview
+# https://stackoverflow.com/questions/28227406/how-to-set-qlistview-rows-height-permanently
+class FieldsForm(QWidget):
+
+    vbox: QVBoxLayout
+    form: QFormLayout
+
+    def __init__(
+        self,
+        parent=None,
+
+    ) -> None:
+
+        super().__init__(parent)
+
+        # size it as we specify
+        self.setSizePolicy(
+            QSizePolicy.Expanding,
+            QSizePolicy.Expanding,
+        )
+
+        # XXX: not sure why we have to create this here exactly
+        # (instead of in the pane creation routine) but it's
+        # here and is managed by downstream layout routines.
+        # best guess is that you have to create layouts in order
+        # of hierarchy in order for things to display correctly?
+        # TODO: we may want to hand this *down* from some "pane manager"
+        # thing eventually?
+        self.vbox = QVBoxLayout(self)
+        # self.vbox.setAlignment(Qt.AlignVCenter)
+        self.vbox.setAlignment(Qt.AlignBottom)
+        self.vbox.setContentsMargins(3, 6, 3, 6)
+        self.vbox.setSpacing(0)
+
+        # split layout for the (<label>: |<widget>|) parameters entry
+        self.form = QFormLayout()
+        self.form.setAlignment(Qt.AlignTop | Qt.AlignLeft)
+        self.form.setContentsMargins(0, 0, 3, 0)
+        self.form.setSpacing(0)
+        self.form.setHorizontalSpacing(0)
+
+        self.vbox.addLayout(self.form, stretch=3)
+
+        self.labels: dict[str, QLabel] = {}
+        self.fields: dict[str, QWidget] = {}
+
+        self._font_size = _font_small.px_size - 2
+        self._max_item_width: (float, float) = 0, 0
+
+    def add_field_label(
+        self,
+
+        name: str,
+
+        font_size: Optional[int] = None,
+        font_color: str = 'default_lightest',
+
+    ) -> QtGui.QLabel:
+
+        # add label to left of search bar
+        # self.label = label = QtGui.QLabel()
+        font_size = font_size or self._font_size - 1
+
+        self.label = label = FormatLabel(
+            fmt_str=name,
+            font=_font.font,
+            font_size=font_size,
+            font_color=font_color,
+        )
+
+        # for later lookup
+        self.labels[name] = label
+
+        return label
+
+    def add_edit_field(
+        self,
+
+        key: str,
+        label_name: str,
+        value: str,
+        readonly: bool = False,
+
+    ) -> Edit:
+
+        # TODO: maybe a distint layout per "field" item?
+        label = self.add_field_label(label_name)
+
+        edit = Edit(
+            parent=self,
+            # width_in_chars=6,
+        )
+        edit.setStyleSheet(
+            f"""QLineEdit {{
+                color : {hcolor('gunmetal')};
+                font-size : {self._font_size}px;
+            }}
+            """
+        )
+        edit.setReadOnly(readonly)
+        edit.setText(str(value))
+        self.form.addRow(label, edit)
+
+        self.fields[key] = edit
+
+        return edit
+
+    def add_select_field(
+        self,
+
+        key: str,
+        label_name: str,
+        values: list[str],
+
+    ) -> Selection:
+
+        # TODO: maybe a distint layout per "field" item?
+        label = self.add_field_label(label_name)
+
+        select = Selection(self)
+        select.set_style(color='gunmetal', font_size=self._font_size)
+        select._key = key
+        select.set_items(values)
+
+        self.setSizePolicy(
+            QSizePolicy.Fixed,
+            QSizePolicy.Fixed,
+        )
+        select.show()
+        self.form.addRow(label, select)
+        self.fields[key] = select
+        return select
+
+
+async def handle_field_input(
+
+    widget: QWidget,
+    recv_chan: trio.abc.ReceiveChannel,
+    form: FieldsForm,
+    on_value_change: Callable[[str, Any], Awaitable[bool]],
+    focus_next: QWidget,
+
+) -> None:
+
+    async for kbmsg in recv_chan:
+
+        if kbmsg.etype in {QEvent.KeyPress, QEvent.KeyRelease}:
+            event, etype, key, mods, txt = kbmsg.to_tuple()
+            print(f'key: {kbmsg.key}, mods: {kbmsg.mods}, txt: {kbmsg.txt}')
+
+            # default controls set
+            ctl = False
+            if kbmsg.mods == Qt.ControlModifier:
+                ctl = True
+
+            if ctl and key in {  # cancel and refocus
+
+                Qt.Key_C,
+                Qt.Key_Space,  # i feel like this is the "native" one
+                Qt.Key_Alt,
+            }:
+
+                widget.clearFocus()
+                # normally the godwidget
+                focus_next.focus()
+                continue
+
+            # process field input
+            if key in (Qt.Key_Enter, Qt.Key_Return):
+
+                key = widget._key
+                value = widget.text()
+                on_value_change(key, value)
+
+
+def mk_form(
+
+    parent: QWidget,
+    fields_schema: dict,
+    font_size: Optional[int] = None,
+
+) -> FieldsForm:
+
+    # TODO: generate components from model
+    # instead of schema dict (aka use an ORM)
+    form = FieldsForm(parent=parent)
+    form._font_size = font_size or _font_small.px_size
+
+    # generate sub-components from schema dict
+    for key, conf in fields_schema.items():
+        wtype = conf['type']
+        label = str(conf.get('label', key))
+        kwargs = conf.get('kwargs', {})
+
+        # plain (line) edit field
+        if wtype == 'edit':
+            w = form.add_edit_field(
+                key,
+                label,
+                conf['default_value'],
+                **kwargs,
+            )
+
+        # drop-down selection
+        elif wtype == 'select':
+            values = list(conf['default_value'])
+            w = form.add_select_field(
+                key,
+                label,
+                values,
+                **kwargs,
+            )
+
+        w._key = key
+
+    return form
+
+
+@asynccontextmanager
+async def open_form_input_handling(
+
+    form: FieldsForm,
+    focus_next: QWidget,
+    on_value_change: Callable[[str, Any], Awaitable[bool]],
+
+) -> FieldsForm:
+
+    async with open_handlers(
+
+        list(form.fields.values()),
+        event_types={
+            QEvent.KeyPress,
+        },
+
+        async_handler=partial(
+            handle_field_input,
+            form=form,
+            focus_next=focus_next,
+            on_value_change=on_value_change,
+        ),
+
+        # block key repeats?
+        filter_auto_repeats=True,
+    ):
+        yield form
+
+
+class FillStatusBar(QProgressBar):
+    '''
+    A status bar for fills up to a position limit.
+
+    '''
+    border_px: int = 2
+    slot_margin_px: int = 1
+
+    def __init__(
+        self,
+        approx_height_px: float,
+        width_px: float,
+        font_size: int,
+        parent=None
+
+    ) -> None:
+        super().__init__(parent=parent)
+
+        self.approx_h = int(round(approx_height_px))
+        self.setMinimumHeight(self.approx_h)
+        self.setMaximumHeight(self.approx_h)
+
+        self.font_size = font_size
+
+        self.setFormat('')  # label format
+        self.setMinimumWidth(int(width_px))
+        self.setMaximumWidth(int(width_px))
+
+    def set_slots(
+        self,
+        slots: int,
+        value: float,
+
+    ) -> None:
+
+        self.setOrientation(Qt.Vertical)
+        h = self.height()
+
+        # TODO: compute "used height" thus far and mostly fill the rest
+        tot_slot_h, r = divmod(h, slots)
+
+        self.setStyleSheet(
+            f"""
+            QProgressBar {{
+
+                text-align: center;
+
+                font-size : {self.font_size - 2}px;
+
+                background-color: {hcolor('papas_special')};
+                color : {hcolor('papas_special')};
+
+                border: {self.border_px}px solid {hcolor('default_light')};
+                border-radius: 2px;
+            }}
+            QProgressBar::chunk {{
+
+                background-color: {hcolor('default_spotlight')};
+                color: {hcolor('bracket')};
+
+                border-radius: 2px;
+            }}
+            """
+        )
+
+        # to set a discrete "block" per slot...
+        # XXX: couldn't get the discrete math to work here such
+        # that it was always correctly showing a discretized value
+        # up to the limit; not sure if it's the ``.setRange()``
+        # / ``.setValue()`` api or not but i was able to get something
+        # close screwing with the divmod above above but after so large
+        # a value it would always be less chunks then the correct
+        # value..
+        # margin: {self.slot_margin_px}px;
+        # height: {slot_height_px}px;
+
+
+        # margin-bottom: {slot_margin_px*2}px;
+        # margin-top: {slot_margin_px*2}px;
+        # color: #19232D;
+        # width: 10px;
+
+        self.setRange(0, slots)
+        self.setValue(value)
+
+
+def mk_fill_status_bar(
+
+    parent_pane: QWidget,
+    form: FieldsForm,
+    pane_vbox: QVBoxLayout,
+    label_font_size: Optional[int] = None,
+
+) -> (
+    # TODO: turn this into a composite?
+    QHBoxLayout,
+    QProgressBar,
+    QLabel,
+    QLabel,
+    QLabel,
+):
+    # indent = 18
+    # bracket_val = 0.375 * 0.666 * w
+    # indent = bracket_val / (1 + 5/8)
+
+    # TODO: calc this height from the ``ChartnPane``
+    chart_h = round(parent_pane.height() * 5/8)
+    bar_h = chart_h * 0.375
+
+    # TODO: once things are sized to screen
+    bar_label_font_size = label_font_size or _font.px_size - 2
+
+    label_font = DpiAwareFont()
+    label_font._set_qfont_px_size(bar_label_font_size)
+    br = label_font.boundingRect(f'{3.32:.1f}% port')
+    _, h = br.width(), br.height()
+    # text_w = 8/3 * w
+
+    # PnL on lhs
+    bar_labels_lhs = QVBoxLayout()
+    left_label = form.add_field_label(
+        '{pnl:>+.2%} pnl',
+        font_size=bar_label_font_size,
+        font_color='gunmetal',
+    )
+    # size according to dpi scaled fonted contents to avoid
+    # resizes on magnitude changes (eg. 9 -> 10 %)
+    min_w = int(_font.boundingRect('1000.0M% pnl').width())
+    left_label.setMinimumWidth(min_w)
+    left_label.resize(
+        min_w,
+        left_label.size().height(),
+    )
+
+    bar_labels_lhs.addSpacing(int(5/8 * bar_h))
+
+    bar_labels_lhs.addWidget(
+        left_label,
+        # XXX: doesn't seem to actually push up against
+        # the status bar?
+        alignment=Qt.AlignRight | Qt.AlignTop,
+    )
+
+    # this hbox is added as a layout by the paner maker/caller
+    hbox = QHBoxLayout()
+
+    hbox.addLayout(bar_labels_lhs)
+    # hbox.addSpacing(indent)  # push to right a bit
+
+    # config
+    # hbox.setSpacing(indent * 0.375)
+    hbox.setSpacing(0)
+    hbox.setAlignment(Qt.AlignTop | Qt.AlignLeft)
+    hbox.setContentsMargins(0, 0, 0, 0)
+
+    # TODO: use percentage str formatter:
+    # https://docs.python.org/3/library/string.html#grammar-token-precision
+
+    top_label = form.add_field_label(
+        '{limit}',
+        font_size=bar_label_font_size,
+        font_color='gunmetal',
+    )
+
+    bottom_label = form.add_field_label(
+        'x: {step_size}',
+        font_size=bar_label_font_size,
+        font_color='gunmetal',
+    )
+
+    bar = FillStatusBar(
+        approx_height_px=bar_h,
+        width_px=h * (1 + 1/6),
+        font_size=form._font_size,
+        parent=form
+    )
+
+    hbox.addWidget(bar, alignment=Qt.AlignLeft | Qt.AlignTop)
+
+    bar_labels_rhs_vbox = QVBoxLayout()
+    bar_labels_rhs_vbox.addWidget(
+        top_label,
+        alignment=Qt.AlignLeft | Qt.AlignTop
+    )
+    bar_labels_rhs_vbox.addWidget(
+        bottom_label,
+        alignment=Qt.AlignLeft | Qt.AlignBottom
+    )
+
+    hbox.addLayout(bar_labels_rhs_vbox)
+
+    # compute "chunk" sizes for fill-status-bar based on some static height
+    slots = 4
+    bar.set_slots(slots, value=0)
+
+    return hbox, bar, left_label, top_label, bottom_label
+
+
+def mk_order_pane_layout(
+
+    parent: QWidget,
+    # accounts: dict[str, Optional[str]],
+
+) -> FieldsForm:
+
+    font_size: int = _font.px_size - 2
+
+    # TODO: maybe just allocate the whole fields form here
+    # and expect an async ctx entry?
+    form = mk_form(
+        parent=parent,
+        fields_schema={
+            'account': {
+                'label': '**accnt**:',
+                'type': 'select',
+                'default_value': ['paper'],
+            },
+            'size_unit': {
+                'label': '**alloc**:',
+                'type': 'select',
+                'default_value': [
+                    '$ size',
+                    '# units',
+                    # '% of port',
+                ],
+            },
+            # 'disti_weight': {
+            #     'label': '**weighting**:',
+            #     'type': 'select',
+            #     'default_value': ['uniform'],
+            # },
+            'limit': {
+                'label': '**limit**:',
+                'type': 'edit',
+                'default_value': 5000,
+            },
+            'slots': {
+                'label': '**slots**:',
+                'type': 'edit',
+                'default_value': 4,
+            },
+        },
+    )
+
+    # top level pane layout
+    # XXX: see ``FieldsForm.__init__()`` for why we can't do basic
+    # config of the vbox here
+    vbox = form.vbox
+
+    # _, h = form.width(), form.height()
+    # print(f'w, h: {w, h}')
+
+    hbox, fill_bar, left_label, top_label, bottom_label = mk_fill_status_bar(
+        parent,
+        form,
+        pane_vbox=vbox,
+        label_font_size=font_size,
+    )
+    # TODO: would be nice to have some better way of reffing these over
+    # monkey patching...
+    form.fill_bar = fill_bar
+    form.left_label = left_label
+    form.bottom_label = bottom_label
+    form.top_label = top_label
+
+    # add pp fill bar + spacing
+    vbox.addLayout(hbox, stretch=3)
+
+    # TODO: handle resize events and appropriately scale this
+    # to the sidepane height?
+    # https://doc.qt.io/qt-5/layout.html#adding-widgets-to-a-layout
+    # vbox.setSpacing(_font.px_size * 1.375)
+
+    form.show()
+    return form

piker/ui/_fsp.py

@@ -0,0 +1,970 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+FSP UI and graphics components.
+
+Financial signal processing cluster and real-time graphics management.
+
+'''
+from contextlib import asynccontextmanager as acm
+from functools import partial
+import inspect
+from itertools import cycle
+from typing import Optional, AsyncGenerator, Any
+
+import numpy as np
+from pydantic import create_model
+import tractor
+import pyqtgraph as pg
+import trio
+from trio_typing import TaskStatus
+
+from ._axes import PriceAxis
+from .._cacheables import maybe_open_context
+from ..calc import humanize
+from ..data._sharedmem import (
+    ShmArray,
+    _Token,
+    try_read,
+)
+from ._chart import (
+    ChartPlotWidget,
+    LinkedSplits,
+)
+from ._forms import (
+    FieldsForm,
+    mk_form,
+    open_form_input_handling,
+)
+from ..fsp._api import maybe_mk_fsp_shm, Fsp
+from ..fsp import cascade
+from ..fsp._volume import (
+    tina_vwap,
+    dolla_vlm,
+    flow_rates,
+)
+from ..log import get_logger
+
+log = get_logger(__name__)
+
+
+def has_vlm(ohlcv: ShmArray) -> bool:
+    # make sure that the instrument supports volume history
+    # (sometimes this is not the case for some commodities and
+    # derivatives)
+    vlm = ohlcv.array['volume']
+    return not bool(np.all(np.isin(vlm, -1)) or np.all(np.isnan(vlm)))
+
+
+def update_fsp_chart(
+    chart: ChartPlotWidget,
+    flow,
+    graphics_name: str,
+    array_key: Optional[str],
+    **kwargs,
+
+) -> None:
+
+    shm = flow.shm
+    if not shm:
+        return
+
+    array = shm.array
+    last_row = try_read(array)
+
+    # guard against unreadable case
+    if not last_row:
+        log.warning(f'Read-race on shm array: {graphics_name}@{shm.token}')
+        return
+
+    # update graphics
+    # NOTE: this does a length check internally which allows it
+    # staying above the last row check below..
+    chart.update_graphics_from_flow(
+        graphics_name,
+        array_key=array_key or graphics_name,
+        **kwargs,
+    )
+
+    # XXX: re: ``array_key``: fsp func names must be unique meaning we
+    # can't have duplicates of the underlying data even if multiple
+    # sub-charts reference it under different 'named charts'.
+
+    # read from last calculated value and update any label
+    last_val_sticky = chart._ysticks.get(graphics_name)
+    if last_val_sticky:
+        last = last_row[array_key]
+        last_val_sticky.update_from_data(-1, last)
+
+
+@acm
+async def open_fsp_sidepane(
+    linked: LinkedSplits,
+    conf: dict[str, dict[str, str]],
+
+) -> FieldsForm:
+
+    schema = {}
+
+    assert len(conf) == 1  # for now
+
+    # add (single) selection widget
+    for name, config in conf.items():
+        schema[name] = {
+                'label': '**fsp**:',
+                'type': 'select',
+                'default_value': [name],
+            }
+
+        # add parameters for selection "options"
+        params = config.get('params', {})
+        for name, config in params.items():
+
+            default = config['default_value']
+            kwargs = config.get('widget_kwargs', {})
+
+            # add to ORM schema
+            schema.update({
+                name: {
+                    'label': f'**{name}**:',
+                    'type': 'edit',
+                    'default_value': default,
+                    'kwargs': kwargs,
+                },
+            })
+
+    sidepane: FieldsForm = mk_form(
+        parent=linked.godwidget,
+        fields_schema=schema,
+    )
+
+    # https://pydantic-docs.helpmanual.io/usage/models/#dynamic-model-creation
+    FspConfig = create_model(
+        'FspConfig',
+        name=name,
+        **params,
+    )
+    sidepane.model = FspConfig()
+
+    # just a logger for now until we get fsp configs up and running.
+    async def settings_change(
+        key: str,
+        value: str
+
+    ) -> bool:
+        print(f'{key}: {value}')
+        return True
+
+    # TODO:
+    async with (
+        open_form_input_handling(
+            sidepane,
+            focus_next=linked.godwidget,
+            on_value_change=settings_change,
+        )
+    ):
+        yield sidepane
+
+
+@acm
+async def open_fsp_actor_cluster(
+    names: list[str] = ['fsp_0', 'fsp_1'],
+
+) -> AsyncGenerator[int, dict[str, tractor.Portal]]:
+
+    from tractor._clustering import open_actor_cluster
+
+    # profiler = pg.debug.Profiler(
+    #     delayed=False,
+    #     disabled=False
+    # )
+    async with open_actor_cluster(
+        count=2,
+        names=names,
+        modules=['piker.fsp._engine'],
+
+    ) as cluster_map:
+        # profiler('started fsp cluster')
+        yield cluster_map
+
+
+async def run_fsp_ui(
+
+    linkedsplits: LinkedSplits,
+    shm: ShmArray,
+    started: trio.Event,
+    target: Fsp,
+    conf: dict[str, dict],
+    loglevel: str,
+    # profiler: pg.debug.Profiler,
+    # _quote_throttle_rate: int = 58,
+
+) -> None:
+    '''
+    Taskf for UI spawning around a ``LinkedSplits`` chart for fsp
+    related graphics/UX management.
+
+    This is normally spawned/called once for each entry in the fsp
+    config.
+
+    '''
+    name = target.name
+    # profiler(f'started UI task for fsp: {name}')
+
+    async with (
+        # side UI for parameters/controls
+        open_fsp_sidepane(
+            linkedsplits,
+            {name: conf},
+        ) as sidepane,
+    ):
+        await started.wait()
+        # profiler(f'fsp:{name} attached to fsp ctx-stream')
+
+        overlay_with = conf.get('overlay', False)
+        if overlay_with:
+            if overlay_with == 'ohlc':
+                chart = linkedsplits.chart
+            else:
+                chart = linkedsplits.subplots[overlay_with]
+
+            chart.draw_curve(
+                name=name,
+                shm=shm,
+                overlay=True,
+                color='default_light',
+                array_key=name,
+                **conf.get('chart_kwargs', {})
+            )
+
+        else:
+            # create a new sub-chart widget for this fsp
+            chart = linkedsplits.add_plot(
+                name=name,
+                shm=shm,
+
+                array_key=name,
+                sidepane=sidepane,
+
+                # curve by default
+                ohlc=False,
+
+                # settings passed down to ``ChartPlotWidget``
+                **conf.get('chart_kwargs', {})
+            )
+
+            # should **not** be the same sub-chart widget
+            assert chart.name != linkedsplits.chart.name
+
+        array_key = name
+
+        # profiler(f'fsp:{name} chart created')
+
+        # first UI update, usually from shm pushed history
+        update_fsp_chart(
+            chart,
+            chart._flows[array_key],
+            name,
+            array_key=array_key,
+        )
+
+        chart.linked.focus()
+
+        # TODO: figure out if we can roll our own `FillToThreshold` to
+        # get brush filled polygons for OS/OB conditions.
+        # ``pg.FillBetweenItems`` seems to be one technique using
+        # generic fills between curve types while ``PlotCurveItem`` has
+        # logic inside ``.paint()`` for ``self.opts['fillLevel']`` which
+        # might be the best solution?
+
+        # graphics = chart.update_from_array(chart.name, array[name])
+        # graphics.curve.setBrush(50, 50, 200, 100)
+        # graphics.curve.setFillLevel(50)
+
+        # if func_name == 'rsi':
+        #     from ._lines import level_line
+        #     # add moveable over-[sold/bought] lines
+        #     # and labels only for the 70/30 lines
+        #     level_line(chart, 20)
+        #     level_line(chart, 30, orient_v='top')
+        #     level_line(chart, 70, orient_v='bottom')
+        #     level_line(chart, 80, orient_v='top')
+
+        chart.view._set_yrange()
+        # done()  # status updates
+
+        # profiler(f'fsp:{func_name} starting update loop')
+        # profiler.finish()
+
+        # update chart graphics
+        # last = time.time()
+
+        # XXX: this currently doesn't loop since
+        # the FSP engine does **not** push updates atm
+        # since we do graphics update in the main loop
+        # in ``._display.
+        # async for value in stream:
+        #     print(value)
+
+        #     # chart isn't actively shown so just skip render cycle
+        #     if chart.linked.isHidden():
+        #         continue
+
+        #     else:
+        #         now = time.time()
+        #         period = now - last
+
+        #         if period <= 1/_quote_throttle_rate:
+        #             # faster then display refresh rate
+        #             print(f'fsp too fast: {1/period}')
+        #             continue
+
+        #         # run synchronous update
+        #         update_fsp_chart(
+        #             chart,
+        #             shm,
+        #             display_name,
+        #             array_key=func_name,
+        #         )
+
+        #         # set time of last graphics update
+        #         last = time.time()
+
+
+class FspAdmin:
+    '''
+    Client API for orchestrating FSP actors and displaying
+    real-time graphics output.
+
+    '''
+    def __init__(
+        self,
+        tn: trio.Nursery,
+        cluster: dict[str, tractor.Portal],
+        linked: LinkedSplits,
+        src_shm: ShmArray,
+
+    ) -> None:
+        self.tn = tn
+        self.cluster = cluster
+        self.linked = linked
+        self._rr_next_actor = cycle(cluster.items())
+        self._registry: dict[
+            tuple,
+            tuple[tractor.MsgStream, ShmArray]
+        ] = {}
+        self._flow_registry: dict[_Token, str] = {}
+        self.src_shm = src_shm
+
+    def rr_next_portal(self) -> tractor.Portal:
+        name, portal = next(self._rr_next_actor)
+        return portal
+
+    async def open_chain(
+        self,
+
+        portal: tractor.Portal,
+        complete: trio.Event,
+        started: trio.Event,
+        fqsn: str,
+        dst_shm: ShmArray,
+        conf: dict,
+        target: Fsp,
+        loglevel: str,
+
+    ) -> None:
+        '''
+        Task which opens a remote FSP endpoint in the managed
+        cluster and sleeps until signalled to exit.
+
+        '''
+        ns_path = str(target.ns_path)
+        async with (
+            portal.open_context(
+
+                # chaining entrypoint
+                cascade,
+
+                # data feed key
+                fqsn=fqsn,
+
+                # mems
+                src_shm_token=self.src_shm.token,
+                dst_shm_token=dst_shm.token,
+
+                # target
+                ns_path=ns_path,
+
+                loglevel=loglevel,
+                zero_on_step=conf.get('zero_on_step', False),
+                shm_registry=[
+                    (token.as_msg(), fsp_name, dst_token.as_msg())
+                    for (token, fsp_name), dst_token
+                    in self._flow_registry.items()
+                ],
+
+            ) as (ctx, last_index),
+            ctx.open_stream() as stream,
+        ):
+
+            # register output data
+            self._registry[
+                (fqsn, ns_path)
+            ] = (
+                stream,
+                dst_shm,
+                complete
+            )
+
+            started.set()
+
+            # wait for graceful shutdown signal
+            async with stream.subscribe() as stream:
+                async for msg in stream:
+                    info = msg.get('fsp_update')
+                    if info:
+                        # if the chart isn't hidden try to update
+                        # the data on screen.
+                        if not self.linked.isHidden():
+                            log.debug(f'Re-syncing graphics for fsp: {ns_path}')
+                            self.linked.graphics_cycle(
+                                trigger_all=True,
+                                prepend_update_index=info['first'],
+                            )
+                    else:
+                        log.info(f'recved unexpected fsp engine msg: {msg}')
+
+            await complete.wait()
+
+    async def start_engine_task(
+        self,
+
+        target: Fsp,
+        conf: dict[str, dict[str, Any]],
+
+        worker_name: Optional[str] = None,
+        loglevel: str = 'info',
+
+    ) -> (ShmArray, trio.Event):
+
+        fqsn = self.linked.symbol.front_fqsn()
+
+        # allocate an output shm array
+        key, dst_shm, opened = maybe_mk_fsp_shm(
+            fqsn,
+            target=target,
+            readonly=True,
+        )
+        self._flow_registry[
+            (self.src_shm._token, target.name)
+        ] = dst_shm._token
+
+        # if not opened:
+        #     raise RuntimeError(
+        #         f'Already started FSP `{fqsn}:{func_name}`'
+        #     )
+
+        portal = self.cluster.get(worker_name) or self.rr_next_portal()
+        complete = trio.Event()
+        started = trio.Event()
+        self.tn.start_soon(
+            self.open_chain,
+            portal,
+            complete,
+            started,
+            fqsn,
+            dst_shm,
+            conf,
+            target,
+            loglevel,
+        )
+
+        return dst_shm, started
+
+    async def open_fsp_chart(
+        self,
+
+        target: Fsp,
+
+        conf: dict,  # yeah probably dumb..
+        loglevel: str = 'error',
+
+    ) -> (trio.Event, ChartPlotWidget):
+
+        shm, started = await self.start_engine_task(
+            target,
+            conf,
+            loglevel,
+        )
+
+        # init async
+        self.tn.start_soon(
+            partial(
+                run_fsp_ui,
+
+                self.linked,
+                shm,
+                started,
+                target,
+
+                conf=conf,
+                loglevel=loglevel,
+            )
+        )
+        return started
+
+
+@acm
+async def open_fsp_admin(
+    linked: LinkedSplits,
+    src_shm: ShmArray,
+    **kwargs,
+
+) -> AsyncGenerator[dict, dict[str, tractor.Portal]]:
+
+    async with (
+        maybe_open_context(
+            # for now make a cluster per client?
+            acm_func=open_fsp_actor_cluster,
+            kwargs=kwargs,
+        ) as (cache_hit, cluster_map),
+
+        trio.open_nursery() as tn,
+    ):
+        if cache_hit:
+            log.info('re-using existing fsp cluster')
+
+        admin = FspAdmin(
+            tn,
+            cluster_map,
+            linked,
+            src_shm,
+        )
+        try:
+            yield admin
+        finally:
+            # terminate all tasks via signals
+            for key, entry in admin._registry.items():
+                _, _, event = entry
+                event.set()
+
+
+async def open_vlm_displays(
+
+    linked: LinkedSplits,
+    ohlcv: ShmArray,
+    dvlm: bool = True,
+
+    task_status: TaskStatus[ChartPlotWidget] = trio.TASK_STATUS_IGNORED,
+
+) -> ChartPlotWidget:
+    '''
+    Volume subchart displays.
+
+    Since "volume" is often included directly alongside OHLCV price
+    data, we don't really need a separate FSP-actor + shm array for it
+    since it's likely already directly adjacent to OHLC samples from the
+    data provider.
+
+    Further only if volume data is detected (it sometimes isn't provided
+    eg. forex, certain commodities markets) will volume dependent FSPs
+    be spawned here.
+
+    '''
+    sig = inspect.signature(flow_rates.func)
+    params = sig.parameters
+
+    async with (
+        open_fsp_sidepane(
+            linked, {
+                'flows': {
+
+                    # TODO: add support for dynamically changing these
+                    'params': {
+                        u'\u03BC' + '_type': {
+                            'default_value': str(params['mean_type'].default),
+                        },
+                        'period': {
+                            'default_value': str(params['period'].default),
+                            # make widget un-editable for now.
+                            'widget_kwargs': {'readonly': True},
+                        },
+                    },
+                }
+            },
+        ) as sidepane,
+        open_fsp_admin(linked, ohlcv) as admin,
+    ):
+        # TODO: support updates
+        # period_field = sidepane.fields['period']
+        # period_field.setText(
+        #     str(period_param.default)
+        # )
+
+        # built-in vlm which we plot ASAP since it's
+        # usually data provided directly with OHLC history.
+        shm = ohlcv
+        chart = linked.add_plot(
+            name='volume',
+            shm=shm,
+
+            array_key='volume',
+            sidepane=sidepane,
+
+            # curve by default
+            ohlc=False,
+
+            # Draw vertical bars from zero.
+            # we do this internally ourselves since
+            # the curve item internals are pretty convoluted.
+            style='step',
+        )
+
+        # force 0 to always be in view
+        def multi_maxmin(
+            names: list[str],
+
+        ) -> tuple[float, float]:
+
+            mx = 0
+            for name in names:
+
+                mxmn = chart.maxmin(name=name)
+                if mxmn:
+                    ymax = mxmn[1]
+                    if ymax > mx:
+                        mx = ymax
+
+            return 0, mx
+
+        chart.view.maxmin = partial(multi_maxmin, names=['volume'])
+
+        # TODO: fix the x-axis label issue where if you put
+        # the axis on the left it's totally not lined up...
+        # show volume units value on LHS (for dinkus)
+        # chart.hideAxis('right')
+        # chart.showAxis('left')
+
+        # send back new chart to caller
+        task_status.started(chart)
+
+        # should **not** be the same sub-chart widget
+        assert chart.name != linked.chart.name
+
+        # sticky only on sub-charts atm
+        last_val_sticky = chart._ysticks[chart.name]
+
+        # read from last calculated value
+        value = shm.array['volume'][-1]
+
+        last_val_sticky.update_from_data(-1, value)
+
+        vlm_curve = chart.update_graphics_from_flow(
+            'volume',
+            # shm.array,
+        )
+
+        # size view to data once at outset
+        chart.view._set_yrange()
+
+        # add axis title
+        axis = chart.getAxis('right')
+        axis.set_title(' vlm')
+
+        if dvlm:
+
+            tasks_ready = []
+            # spawn and overlay $ vlm on the same subchart
+            dvlm_shm, started = await admin.start_engine_task(
+                dolla_vlm,
+
+                {  # fsp engine conf
+                    'func_name': 'dolla_vlm',
+                    'zero_on_step': True,
+                    'params': {
+                        'price_func': {
+                            'default_value': 'chl3',
+                        },
+                    },
+                },
+                # loglevel,
+            )
+            tasks_ready.append(started)
+
+            # FIXME: we should error on starting the same fsp right
+            # since it might collide with existing shm.. or wait we
+            # had this before??
+            # dolla_vlm,
+
+            tasks_ready.append(started)
+            # profiler(f'created shm for fsp actor: {display_name}')
+
+            # wait for all engine tasks to startup
+            async with trio.open_nursery() as n:
+                for event in tasks_ready:
+                    n.start_soon(event.wait)
+
+            # dolla vlm overlay
+            # XXX: the main chart already contains a vlm "units" axis
+            # so here we add an overlay wth a y-range in
+            # $ liquidity-value units (normally a fiat like USD).
+            dvlm_pi = chart.overlay_plotitem(
+                'dolla_vlm',
+                index=0,  # place axis on inside (nearest to chart)
+                axis_title=' $vlm',
+                axis_side='right',
+                axis_kwargs={
+                    'typical_max_str': ' 100.0 M ',
+                    'formatter': partial(
+                        humanize,
+                        digits=2,
+                    ),
+                },
+            )
+
+            # all to be overlayed curve names
+            fields = [
+               'dolla_vlm',
+               'dark_vlm',
+            ]
+            # dvlm_rate_fields = [
+            #     'dvlm_rate',
+            #     'dark_dvlm_rate',
+            # ]
+            trade_rate_fields = [
+                'trade_rate',
+                'dark_trade_rate',
+            ]
+
+            group_mxmn = partial(
+                multi_maxmin,
+                # keep both regular and dark vlm in view
+                names=fields,
+                # names=fields + dvlm_rate_fields,
+            )
+
+            # add custom auto range handler
+            dvlm_pi.vb._maxmin = group_mxmn
+
+            # use slightly less light (then bracket) gray
+            # for volume from "main exchange" and a more "bluey"
+            # gray for "dark" vlm.
+            vlm_color = 'i3'
+            dark_vlm_color = 'charcoal'
+
+            # add dvlm (step) curves to common view
+            def chart_curves(
+                names: list[str],
+                pi: pg.PlotItem,
+                shm: ShmArray,
+                step_mode: bool = False,
+                style: str = 'solid',
+
+            ) -> None:
+                for name in names:
+                    if 'dark' in name:
+                        color = dark_vlm_color
+                    elif 'rate' in name:
+                        color = vlm_color
+                    else:
+                        color = 'bracket'
+
+                    curve, _ = chart.draw_curve(
+                        name=name,
+                        shm=shm,
+                        array_key=name,
+                        overlay=pi,
+                        color=color,
+                        step_mode=step_mode,
+                        style=style,
+                        pi=pi,
+                    )
+
+                    # TODO: we need a better API to do this..
+                    # specially store ref to shm for lookup in display loop
+                    # since only a placeholder of `None` is entered in
+                    # ``.draw_curve()``.
+                    flow = chart._flows[name]
+                    assert flow.plot is pi
+
+            chart_curves(
+                fields,
+                dvlm_pi,
+                dvlm_shm,
+                step_mode=True,
+            )
+
+            # spawn flow rates fsp **ONLY AFTER** the 'dolla_vlm' fsp is
+            # up since this one depends on it.
+
+            fr_shm, started = await admin.start_engine_task(
+                flow_rates,
+                {  # fsp engine conf
+                    'func_name': 'flow_rates',
+                    'zero_on_step': False,
+                },
+                # loglevel,
+            )
+            await started.wait()
+
+            # chart_curves(
+            #     dvlm_rate_fields,
+            #     dvlm_pi,
+            #     fr_shm,
+            # )
+
+            # TODO: is there a way to "sync" the dual axes such that only
+            # one curve is needed?
+            # hide the original vlm curve since the $vlm one is now
+            # displayed and the curves are effectively the same minus
+            # liquidity events (well at least on low OHLC periods - 1s).
+            vlm_curve.hide()
+            chart.removeItem(vlm_curve)
+            vflow = chart._flows['volume']
+            vflow.render = False
+
+            # avoid range sorting on volume once disabled
+            chart.view.disable_auto_yrange()
+
+            # Trade rate overlay
+            # XXX: requires an additional overlay for
+            # a trades-per-period (time) y-range.
+            tr_pi = chart.overlay_plotitem(
+                'trade_rates',
+
+                # TODO: dynamically update period (and thus this axis?)
+                # title from user input.
+                axis_title='clears',
+
+                axis_side='left',
+                axis_kwargs={
+                    'typical_max_str': ' 10.0 M ',
+                    'formatter': partial(
+                        humanize,
+                        digits=2,
+                    ),
+                    'text_color': vlm_color,
+                },
+
+            )
+            # add custom auto range handler
+            tr_pi.vb.maxmin = partial(
+                multi_maxmin,
+                # keep both regular and dark vlm in view
+                names=trade_rate_fields,
+            )
+
+            chart_curves(
+                trade_rate_fields,
+                tr_pi,
+                fr_shm,
+                # step_mode=True,
+
+                # dashed line to represent "individual trades" being
+                # more "granular" B)
+                style='dash',
+            )
+
+            for pi in (
+                dvlm_pi,
+                tr_pi,
+            ):
+                for name, axis_info in pi.axes.items():
+                    # lol this sux XD
+                    axis = axis_info['item']
+                    if isinstance(axis, PriceAxis):
+                        axis.size_to_values()
+
+        # built-in vlm fsps
+        for target, conf in {
+            # tina_vwap: {
+            #     'overlay': 'ohlc',  # overlays with OHLCV (main) chart
+            #     'anchor': 'session',
+            # },
+        }.items():
+            started = await admin.open_fsp_chart(
+                target,
+                conf,
+            )
+
+
+async def start_fsp_displays(
+
+    linked: LinkedSplits,
+    ohlcv: ShmArray,
+    group_status_key: str,
+    loglevel: str,
+
+) -> None:
+    '''
+    Create fsp charts from a config input attached to a local actor
+    compute cluster.
+
+    Pass target entrypoint and historical data via ``ShmArray``.
+
+    '''
+    linked.focus()
+
+    # TODO: eventually we'll support some kind of n-compose syntax
+    fsp_conf = {
+        # 'rsi': {
+        #     'func_name': 'rsi',  # literal python func ref lookup name
+
+        #     # map of parameters to place on the fsp sidepane widget
+        #     # which should map to dynamic inputs available to the
+        #     # fsp function at runtime.
+        #     'params': {
+        #         'period': {
+        #             'default_value': 14,
+        #             'widget_kwargs': {'readonly': True},
+        #         },
+        #     },
+
+        #     # ``ChartPlotWidget`` options passthrough
+        #     'chart_kwargs': {
+        #         'static_yrange': (0, 100),
+        #     },
+        # },
+    }
+    profiler = pg.debug.Profiler(
+        delayed=False,
+        disabled=False
+    )
+
+    async with (
+
+        # NOTE: this admin internally opens an actor cluster
+        open_fsp_admin(linked, ohlcv) as admin,
+    ):
+        statuses = []
+        for target, conf in fsp_conf.items():
+            started = await admin.open_fsp_chart(
+                target,
+                conf,
+            )
+            done = linked.window().status_bar.open_status(
+                f'loading fsp, {target}..',
+                group_key=group_status_key,
+            )
+            statuses.append((started, done))
+
+        for fsp_loaded, status_cb in statuses:
+            await fsp_loaded.wait()
+            profiler(f'attached to fsp portal: {target}')
+            status_cb()
+
+    # blocks on nursery until all fsp actors complete

piker/ui/_graphics/__init__.py

@@ -1,20 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-"""
-Internal custom graphics mostly built for low latency and reuse.
-
-"""

piker/ui/_graphics/_curve.py

@@ -1,172 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for piker0)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-"""
-Fast, smooth, sexy curves.
-
-"""
-from typing import Tuple
-
-import pyqtgraph as pg
-from PyQt5 import QtCore, QtGui, QtWidgets
-
-from ..._profile import pg_profile_enabled
-
-
-# TODO: got a feeling that dropping this inheritance gets us even more speedups
-class FastAppendCurve(pg.PlotCurveItem):
-
-    def __init__(self, *args, **kwargs):
-
-        # TODO: we can probably just dispense with the parent since
-        # we're basically only using the pen setting now...
-        super().__init__(*args, **kwargs)
-
-        self._last_line: QtCore.QLineF = None
-        self._xrange: Tuple[int, int] = self.dataBounds(ax=0)
-
-        # TODO: one question still remaining is if this makes trasform
-        # interactions slower (such as zooming) and if so maybe if/when
-        # we implement a "history" mode for the view we disable this in
-        # that mode?
-        self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
-
-    def update_from_array(
-        self,
-        x,
-        y,
-    ) -> QtGui.QPainterPath:
-
-        profiler = pg.debug.Profiler(disabled=not pg_profile_enabled())
-        flip_cache = False
-
-        # print(f"xrange: {self._xrange}")
-        istart, istop = self._xrange
-
-        prepend_length = istart - x[0]
-        append_length = x[-1] - istop
-
-        if self.path is None or prepend_length:
-            self.path = pg.functions.arrayToQPath(
-                x[:-1],
-                y[:-1],
-                connect='all'
-            )
-            profiler('generate fresh path')
-
-        # TODO: get this working - right now it's giving heck on vwap...
-        # if prepend_length:
-        #     breakpoint()
-
-        #     prepend_path = pg.functions.arrayToQPath(
-        #         x[0:prepend_length],
-        #         y[0:prepend_length],
-        #         connect='all'
-        #     )
-
-        #     # swap prepend path in "front"
-        #     old_path = self.path
-        #     self.path = prepend_path
-        #     # self.path.moveTo(new_x[0], new_y[0])
-        #     self.path.connectPath(old_path)
-
-        if append_length:
-            # print(f"append_length: {append_length}")
-            new_x = x[-append_length - 2:-1]
-            new_y = y[-append_length - 2:-1]
-            # print((new_x, new_y))
-
-            append_path = pg.functions.arrayToQPath(
-                new_x,
-                new_y,
-                connect='all'
-            )
-            # print(f"append_path br: {append_path.boundingRect()}")
-            # self.path.moveTo(new_x[0], new_y[0])
-            # self.path.connectPath(append_path)
-            self.path.connectPath(append_path)
-
-            # XXX: pretty annoying but, without this there's little
-            # artefacts on the append updates to the curve...
-            self.setCacheMode(QtWidgets.QGraphicsItem.NoCache)
-            self.prepareGeometryChange()
-            flip_cache = True
-
-            # print(f"update br: {self.path.boundingRect()}")
-
-        # XXX: lol brutal, the internals of `CurvePoint` (inherited by
-        # our `LineDot`) required ``.getData()`` to work..
-        self.xData = x
-        self.yData = y
-
-        self._xrange = x[0], x[-1]
-        self._last_line = QtCore.QLineF(x[-2], y[-2], x[-1], y[-1])
-
-        # trigger redraw of path
-        # do update before reverting to cache mode
-        self.prepareGeometryChange()
-        self.update()
-
-        if flip_cache:
-            self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
-
-    def boundingRect(self):
-        if self.path is None:
-            return QtGui.QPainterPath().boundingRect()
-        else:
-            # dynamically override this method after initial
-            # path is created to avoid requiring the above None check
-            self.boundingRect = self._br
-            return self._br()
-
-    def _br(self):
-        """Post init ``.boundingRect()```.
-
-        """
-        hb = self.path.controlPointRect()
-        hb_size = hb.size()
-        # print(f'hb_size: {hb_size}')
-
-        w = hb_size.width() + 1
-        h = hb_size.height() + 1
-
-        br = QtCore.QRectF(
-
-            # top left
-            QtCore.QPointF(hb.topLeft()),
-
-            # total size
-            QtCore.QSizeF(w, h)
-        )
-        # print(f'bounding rect: {br}')
-        return br
-
-    def paint(
-        self,
-        p: QtGui.QPainter,
-        opt: QtWidgets.QStyleOptionGraphicsItem,
-        w: QtWidgets.QWidget
-    ) -> None:
-
-        profiler = pg.debug.Profiler(disabled=not pg_profile_enabled())
-        # p.setRenderHint(p.Antialiasing, True)
-
-        p.setPen(self.opts['pen'])
-        p.drawLine(self._last_line)
-        profiler('.drawLine()')
-
-        p.drawPath(self.path)
-        profiler('.drawPath()')

piker/ui/_graphics/_ohlc.py

@@ -1,423 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-"""
-Super fast OHLC sampling graphics types.
-
-"""
-from typing import List, Optional, Tuple
-
-import numpy as np
-import pyqtgraph as pg
-from numba import njit, float64, int64  # , optional
-from PyQt5 import QtCore, QtGui, QtWidgets
-from PyQt5.QtCore import QLineF, QPointF
-# from numba import types as ntypes
-# from ..data._source import numba_ohlc_dtype
-
-from ..._profile import pg_profile_enabled
-from .._style import hcolor
-
-
-def _mk_lines_array(
-    data: List,
-    size: int,
-    elements_step: int = 6,
-) -> np.ndarray:
-    """Create an ndarray to hold lines graphics info.
-
-    """
-    return np.zeros_like(
-        data,
-        shape=(int(size), elements_step),
-        dtype=object,
-    )
-
-
-def lines_from_ohlc(
-    row: np.ndarray,
-    w: float
-) -> Tuple[QLineF]:
-
-    open, high, low, close, index = row[
-        ['open', 'high', 'low', 'close', 'index']]
-
-    # TODO: maybe consider using `QGraphicsLineItem` ??
-    # gives us a ``.boundingRect()`` on the objects which may make
-    # computing the composite bounding rect of the last bars + the
-    # history path faster since it's done in C++:
-    # https://doc.qt.io/qt-5/qgraphicslineitem.html
-
-    # high -> low vertical (body) line
-    if low != high:
-        hl = QLineF(index, low, index, high)
-    else:
-        # XXX: if we don't do it renders a weird rectangle?
-        # see below for filtering this later...
-        hl = None
-
-    # NOTE: place the x-coord start as "middle" of the drawing range such
-    # that the open arm line-graphic is at the left-most-side of
-    # the index's range according to the view mapping coordinates.
-
-    # open line
-    o = QLineF(index - w, open, index, open)
-
-    # close line
-    c = QLineF(index, close, index + w, close)
-
-    return [hl, o, c]
-
-
-@njit(
-    # TODO: for now need to construct this manually for readonly arrays, see
-    # https://github.com/numba/numba/issues/4511
-    # ntypes.Tuple((float64[:], float64[:], float64[:]))(
-    #     numba_ohlc_dtype[::1],  # contiguous
-    #     int64,
-    #     optional(float64),
-    # ),
-    nogil=True
-)
-def path_arrays_from_ohlc(
-    data: np.ndarray,
-    start: int64,
-    bar_gap: float64 = 0.43,
-) -> np.ndarray:
-    """Generate an array of lines objects from input ohlc data.
-
-    """
-    size = int(data.shape[0] * 6)
-
-    x = np.zeros(
-        # data,
-        shape=size,
-        dtype=float64,
-    )
-    y, c = x.copy(), x.copy()
-
-    # TODO: report bug for assert @
-    # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
-    for i, q in enumerate(data[start:], start):
-
-        # TODO: ask numba why this doesn't work..
-        # open, high, low, close, index = q[
-        #     ['open', 'high', 'low', 'close', 'index']]
-
-        open = q['open']
-        high = q['high']
-        low = q['low']
-        close = q['close']
-        index = float64(q['index'])
-
-        istart = i * 6
-        istop = istart + 6
-
-        # x,y detail the 6 points which connect all vertexes of a ohlc bar
-        x[istart:istop] = (
-            index - bar_gap,
-            index,
-            index,
-            index,
-            index,
-            index + bar_gap,
-        )
-        y[istart:istop] = (
-            open,
-            open,
-            low,
-            high,
-            close,
-            close,
-        )
-
-        # specifies that the first edge is never connected to the
-        # prior bars last edge thus providing a small "gap"/"space"
-        # between bars determined by ``bar_gap``.
-        c[istart:istop] = (0, 1, 1, 1, 1, 1)
-
-    return x, y, c
-
-
-def gen_qpath(
-    data,
-    start,  # XXX: do we need this?
-    w,
-) -> QtGui.QPainterPath:
-
-    profiler = pg.debug.Profiler(disabled=not pg_profile_enabled())
-
-    x, y, c = path_arrays_from_ohlc(data, start, bar_gap=w)
-    profiler("generate stream with numba")
-
-    # TODO: numba the internals of this!
-    path = pg.functions.arrayToQPath(x, y, connect=c)
-    profiler("generate path with arrayToQPath")
-
-    return path
-
-
-class BarItems(pg.GraphicsObject):
-    """Price range bars graphics rendered from a OHLC sequence.
-    """
-    sigPlotChanged = QtCore.pyqtSignal(object)
-
-    # 0.5 is no overlap between arms, 1.0 is full overlap
-    w: float = 0.43
-
-    def __init__(
-        self,
-        # scene: 'QGraphicsScene',  # noqa
-        plotitem: 'pg.PlotItem',  # noqa
-        pen_color: str = 'bracket',
-    ) -> None:
-        super().__init__()
-
-        # XXX: for the mega-lulz increasing width here increases draw latency...
-        # so probably don't do it until we figure that out.
-        self.bars_pen = pg.mkPen(hcolor(pen_color), width=1)
-
-        # NOTE: this prevents redraws on mouse interaction which is
-        # a huge boon for avg interaction latency.
-
-        # TODO: one question still remaining is if this makes trasform
-        # interactions slower (such as zooming) and if so maybe if/when
-        # we implement a "history" mode for the view we disable this in
-        # that mode?
-        self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
-
-        # not sure if this is actually impoving anything but figured it
-        # was worth a shot:
-        # self.path.reserve(int(100e3 * 6))
-
-        self.path = QtGui.QPainterPath()
-
-        self._pi = plotitem
-
-        self._xrange: Tuple[int, int]
-        self._yrange: Tuple[float, float]
-
-        # TODO: don't render the full backing array each time
-        # self._path_data = None
-        self._last_bar_lines: Optional[Tuple[QLineF, ...]] = None
-
-        # track the current length of drawable lines within the larger array
-        self.start_index: int = 0
-        self.stop_index: int = 0
-
-    def draw_from_data(
-        self,
-        data: np.ndarray,
-        start: int = 0,
-    ) -> QtGui.QPainterPath:
-        """Draw OHLC datum graphics from a ``np.ndarray``.
-
-        This routine is usually only called to draw the initial history.
-        """
-        hist, last = data[:-1], data[-1]
-
-        self.path = gen_qpath(hist, start, self.w)
-
-        # save graphics for later reference and keep track
-        # of current internal "last index"
-        # self.start_index = len(data)
-        index = data['index']
-        self._xrange = (index[0], index[-1])
-        self._yrange = (
-            np.nanmax(data['high']),
-            np.nanmin(data['low']),
-        )
-
-        # up to last to avoid double draw of last bar
-        self._last_bar_lines = lines_from_ohlc(last, self.w)
-
-        # trigger render
-        # https://doc.qt.io/qt-5/qgraphicsitem.html#update
-        self.update()
-
-        return self.path
-
-    def update_from_array(
-        self,
-        array: np.ndarray,
-        just_history=False,
-    ) -> None:
-        """Update the last datum's bar graphic from input data array.
-
-        This routine should be interface compatible with
-        ``pg.PlotCurveItem.setData()``. Normally this method in
-        ``pyqtgraph`` seems to update all the data passed to the
-        graphics object, and then update/rerender, but here we're
-        assuming the prior graphics havent changed (OHLC history rarely
-        does) so this "should" be simpler and faster.
-
-        This routine should be made (transitively) as fast as possible.
-        """
-        # index = self.start_index
-        istart, istop = self._xrange
-
-        index = array['index']
-        first_index, last_index = index[0], index[-1]
-
-        # length = len(array)
-        prepend_length = istart - first_index
-        append_length = last_index - istop
-
-        flip_cache = False
-
-        # TODO: allow mapping only a range of lines thus
-        # only drawing as many bars as exactly specified.
-
-        if prepend_length:
-
-            # new history was added and we need to render a new path
-            new_bars = array[:prepend_length]
-            prepend_path = gen_qpath(new_bars, 0, self.w)
-
-            # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path
-            # y value not matching the first value from
-            # array[prepend_length + 1] ???
-
-            # update path
-            old_path = self.path
-            self.path = prepend_path
-            self.path.addPath(old_path)
-
-            # trigger redraw despite caching
-            self.prepareGeometryChange()
-
-        if append_length:
-            # generate new lines objects for updatable "current bar"
-            self._last_bar_lines = lines_from_ohlc(array[-1], self.w)
-
-            # generate new graphics to match provided array
-            # path appending logic:
-            # we need to get the previous "current bar(s)" for the time step
-            # and convert it to a sub-path to append to the historical set
-            # new_bars = array[istop - 1:istop + append_length - 1]
-            new_bars = array[-append_length - 1:-1]
-            append_path = gen_qpath(new_bars, 0, self.w)
-            self.path.moveTo(float(istop - self.w), float(new_bars[0]['open']))
-            self.path.addPath(append_path)
-
-            # trigger redraw despite caching
-            self.prepareGeometryChange()
-            self.setCacheMode(QtWidgets.QGraphicsItem.NoCache)
-            flip_cache = True
-
-        self._xrange = first_index, last_index
-
-        # last bar update
-        i, o, h, l, last, v = array[-1][
-            ['index', 'open', 'high', 'low', 'close', 'volume']
-        ]
-        # assert i == self.start_index - 1
-        # assert i == last_index
-        body, larm, rarm = self._last_bar_lines
-
-        # XXX: is there a faster way to modify this?
-        rarm.setLine(rarm.x1(), last, rarm.x2(), last)
-
-        # writer is responsible for changing open on "first" volume of bar
-        larm.setLine(larm.x1(), o, larm.x2(), o)
-
-        if l != h:  # noqa
-
-            if body is None:
-                body = self._last_bar_lines[0] = QLineF(i, l, i, h)
-            else:
-                # update body
-                body.setLine(i, l, i, h)
-
-            # XXX: pretty sure this is causing an issue where the bar has
-            # a large upward move right before the next sample and the body
-            # is getting set to None since the next bar is flat but the shm
-            # array index update wasn't read by the time this code runs. Iow
-            # we're doing this removal of the body for a bar index that is
-            # now out of date / from some previous sample. It's weird
-            # though because i've seen it do this to bars i - 3 back?
-
-        self.update()
-
-        if flip_cache:
-            self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
-
-    def paint(
-        self,
-        p: QtGui.QPainter,
-        opt: QtWidgets.QStyleOptionGraphicsItem,
-        w: QtWidgets.QWidget
-    ) -> None:
-
-        profiler = pg.debug.Profiler(disabled=not pg_profile_enabled())
-
-        # p.setCompositionMode(0)
-        p.setPen(self.bars_pen)
-
-        # TODO: one thing we could try here is pictures being drawn of
-        # a fixed count of bars such that based on the viewbox indices we
-        # only draw the "rounded up" number of "pictures worth" of bars
-        # as is necesarry for what's in "view". Not sure if this will
-        # lead to any perf gains other then when zoomed in to less bars
-        # in view.
-        p.drawLines(*tuple(filter(bool, self._last_bar_lines)))
-        profiler('draw last bar')
-
-        p.drawPath(self.path)
-        profiler('draw history path')
-
-    def boundingRect(self):
-        # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect
-
-        # TODO: Can we do rect caching to make this faster
-        # like `pg.PlotCurveItem` does? In theory it's just
-        # computing max/min stuff again like we do in the udpate loop
-        # anyway. Not really sure it's necessary since profiling already
-        # shows this method is faf.
-
-        # boundingRect _must_ indicate the entire area that will be
-        # drawn on or else we will get artifacts and possibly crashing.
-        # (in this case, QPicture does all the work of computing the
-        # bounding rect for us).
-
-        # apparently this a lot faster says the docs?
-        # https://doc.qt.io/qt-5/qpainterpath.html#controlPointRect
-        hb = self.path.controlPointRect()
-        hb_tl, hb_br = hb.topLeft(), hb.bottomRight()
-
-        # need to include last bar height or BR will be off
-        mx_y = hb_br.y()
-        mn_y = hb_tl.y()
-
-        body_line = self._last_bar_lines[0]
-        if body_line:
-            mx_y = max(mx_y, max(body_line.y1(), body_line.y2()))
-            mn_y = min(mn_y, min(body_line.y1(), body_line.y2()))
-
-        return QtCore.QRectF(
-
-            # top left
-            QPointF(
-                hb_tl.x(),
-                mn_y,
-            ),
-
-            # bottom right
-            QPointF(
-                hb_br.x() + 1,
-                mx_y,
-            )
-
-        )

piker/ui/_icons.py

@@ -0,0 +1,90 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+``QIcon`` hackery.
+
+'''
+from PyQt5.QtWidgets import QStyle
+from PyQt5.QtGui import (
+    QIcon, QPixmap, QColor
+)
+from PyQt5.QtCore import QSize
+
+from ._style import hcolor
+
+# https://www.pythonguis.com/faq/built-in-qicons-pyqt/
+# account status icons taken from built-in set
+_icon_names: dict[str, str] = {
+    # these two seem to work for our mask hack done below to
+    # change the coloring.
+    'long_pp': 'SP_TitleBarShadeButton',
+    'short_pp': 'SP_TitleBarUnshadeButton',
+    'ready': 'SP_MediaPlay',
+}
+_icons: dict[str, QIcon] = {}
+
+
+def mk_icons(
+
+    style: QStyle,
+    size: QSize,
+
+) -> dict[str, QIcon]:
+    '''This helper is indempotent.
+
+    '''
+    global _icons, _icon_names
+    if _icons:
+        return _icons
+
+    _icons[None] = QIcon()  # the "null" icon
+
+    # load account selection using current style
+    for name, icon_name in _icon_names.items():
+
+        stdpixmap = getattr(QStyle, icon_name)
+        stdicon = style.standardIcon(stdpixmap)
+        pixmap = stdicon.pixmap(size)
+
+        # fill hack from SO to change icon color:
+        # https://stackoverflow.com/a/38369468
+        out_pixmap = QPixmap(size)
+        out_pixmap.fill(QColor(hcolor('default_spotlight')))
+        out_pixmap.setMask(pixmap.createHeuristicMask())
+
+        # TODO: not idea why this doesn't work / looks like
+        # trash.  Sure would be nice to just generate our own
+        # pixmaps on the fly..
+        # p = QPainter(out_pixmap)
+        # p.setOpacity(1)
+        # p.setBrush(QColor(hcolor('papas_special')))
+        # p.setPen(QColor(hcolor('default_lightest')))
+        # path = mk_marker_path(style='|<')
+        # p.scale(6, 6)
+        # # p.translate(0, 0)
+        # p.drawPath(path)
+        # p.save()
+        # p.end()
+        # del p
+        # icon = QIcon(out_pixmap)
+
+        icon = QIcon()
+        icon.addPixmap(out_pixmap)
+
+        _icons[name] = icon
+
+    return _icons

piker/ui/_interaction.py

@@ -18,11 +18,14 @@
 Chart view box primitives
 
 """
+from __future__ import annotations
 from contextlib import asynccontextmanager
 import time
 from typing import Optional, Callable
 
 import pyqtgraph as pg
+# from pyqtgraph.GraphicsScene import mouseEvents
+from PyQt5.QtWidgets import QGraphicsSceneMouseEvent as gs_mouse
 from PyQt5.QtCore import Qt, QEvent
 from pyqtgraph import ViewBox, Point, QtCore
 from pyqtgraph import functions as fn
@@ -30,22 +33,42 @@ import numpy as np
 import trio
 
 from ..log import get_logger
-from ._style import _min_points_to_show
+from .._profile import pg_profile_enabled, ms_slower_then
+# from ._style import _min_points_to_show
 from ._editors import SelectRect
-from ._window import main_window
+from . import _event
 
 
 log = get_logger(__name__)
 
+NUMBER_LINE = {
+    Qt.Key_1,
+    Qt.Key_2,
+    Qt.Key_3,
+    Qt.Key_4,
+    Qt.Key_5,
+    Qt.Key_6,
+    Qt.Key_7,
+    Qt.Key_8,
+    Qt.Key_9,
+    Qt.Key_0,
+}
 
-async def handle_viewmode_inputs(
+ORDER_MODE = {
+    Qt.Key_A,
+    Qt.Key_F,
+    Qt.Key_D,
+}
+
+
+async def handle_viewmode_kb_inputs(
 
     view: 'ChartView',
     recv_chan: trio.abc.ReceiveChannel,
 
 ) -> None:
 
-    mode = view.mode
+    order_mode = view.order_mode
 
     # track edge triggered keys
     # (https://en.wikipedia.org/wiki/Interrupt#Triggering_methods)
@@ -55,6 +78,8 @@ async def handle_viewmode_inputs(
     trigger_mode: str
     action: str
 
+    on_next_release: Optional[Callable] = None
+
     # for quick key sequence-combo pattern matching
     # we have a min_tap period and these should not
     # ever be auto-repeats since we filter those at the
@@ -62,10 +87,11 @@ async def handle_viewmode_inputs(
     min_tap = 1/6
     fast_key_seq: list[str] = []
     fast_taps: dict[str, Callable] = {
-        'cc': mode.cancel_all_orders,
+        'cc': order_mode.cancel_all_orders,
     }
 
-    async for event, etype, key, mods, text in recv_chan:
+    async for kbmsg in recv_chan:
+        event, etype, key, mods, text = kbmsg.to_tuple()
         log.debug(f'key: {key}, mods: {mods}, text: {text}')
         now = time.time()
         period = now - last
@@ -115,7 +141,7 @@ async def handle_viewmode_inputs(
                     Qt.Key_Space,
                 }
             ):
-                view._chart._lc.godwidget.search.focus()
+                view._chart.linked.godwidget.search.focus()
 
             # esc and ctrl-c
             if key == Qt.Key_Escape or (ctrl and key == Qt.Key_C):
@@ -126,11 +152,12 @@ async def handle_viewmode_inputs(
             # cancel order or clear graphics
             if key == Qt.Key_C or key == Qt.Key_Delete:
 
-                mode.cancel_orders_under_cursor()
+                order_mode.cancel_orders_under_cursor()
 
             # View modes
             if key == Qt.Key_R:
 
+                # TODO: set this for all subplots
                 # edge triggered default view activation
                 view.chart.default_view()
 
@@ -144,10 +171,14 @@ async def handle_viewmode_inputs(
         # release branch
         elif etype in {QEvent.KeyRelease}:
 
+            if on_next_release:
+                on_next_release()
+                on_next_release = None
+
             if key in pressed:
                 pressed.remove(key)
 
-        # QUERY MODE #
+        # QUERY/QUOTE MODE #
         if {Qt.Key_Q}.intersection(pressed):
 
             view.linkedsplits.cursor.in_query_mode = True
@@ -155,7 +186,8 @@ async def handle_viewmode_inputs(
         else:
             view.linkedsplits.cursor.in_query_mode = False
 
-        # SELECTION MODE #
+        # SELECTION MODE
+        # --------------
 
         if shift:
             if view.state['mouseMode'] == ViewBox.PanMode:
@@ -163,25 +195,41 @@ async def handle_viewmode_inputs(
         else:
             view.setMouseMode(ViewBox.PanMode)
 
-        # ORDER MODE #
-        # live vs. dark trigger + an action {buy, sell, alert}
+        # Toggle position config pane
+        if (
+            ctrl and key in {
+                Qt.Key_P,
+            }
+        ):
+            pp_pane = order_mode.current_pp.pane
+            if pp_pane.isHidden():
+                pp_pane.show()
+            else:
+                pp_pane.hide()
 
-        order_keys_pressed = {
-            Qt.Key_A,
-            Qt.Key_F,
-            Qt.Key_D
-        }.intersection(pressed)
+        # ORDER MODE
+        # ----------
+
+        # live vs. dark trigger + an action {buy, sell, alert}
+        order_keys_pressed = ORDER_MODE.intersection(pressed)
 
         if order_keys_pressed:
+
+            # show the pp size label
+            order_mode.current_pp.show()
+
+            # TODO: show pp config mini-params in status bar widget
+            # mode.pp_config.show()
+
             if (
                 # 's' for "submit" to activate "live" order
                 Qt.Key_S in pressed or
                 ctrl
             ):
-                trigger_mode: str = 'live'
+                trigger_type: str = 'live'
 
             else:
-                trigger_mode: str = 'dark'
+                trigger_type: str = 'dark'
 
             # order mode trigger "actions"
             if Qt.Key_D in pressed:  # for "damp eet"
@@ -192,32 +240,89 @@ async def handle_viewmode_inputs(
 
             elif Qt.Key_A in pressed:
                 action = 'alert'
-                trigger_mode = 'live'
+                trigger_type = 'live'
 
-            view.order_mode = True
+            order_mode.active = True
 
             # XXX: order matters here for line style!
-            view.mode._exec_mode = trigger_mode
-            view.mode.set_exec(action)
+            order_mode._trigger_type = trigger_type
+            order_mode.stage_order(
+                action,
+                trigger_type=trigger_type,
+            )
 
-            prefix = trigger_mode + '-' if action != 'alert' else ''
-            view._chart.window().mode_label.setText(
-                f'mode: {prefix}{action}')
+            prefix = trigger_type + '-' if action != 'alert' else ''
+            view._chart.window().set_mode_name(f'{prefix}{action}')
+
+        elif (
+            (
+                Qt.Key_S in pressed or
+                order_keys_pressed or
+                Qt.Key_O in pressed
+            ) and
+            key in NUMBER_LINE
+        ):
+            # hot key to set order slots size.
+            # change edit field to current number line value,
+            # update the pp allocator bar, unhighlight the
+            # field when ctrl is released.
+            num = int(text)
+            pp_pane = order_mode.pane
+            pp_pane.on_ui_settings_change('slots', num)
+            edit = pp_pane.form.fields['slots']
+            edit.selectAll()
+            # un-highlight on ctrl release
+            on_next_release = edit.deselect
+            pp_pane.update_status_ui(pp_pane.order_mode.current_pp)
 
         else:  # none active
+
+            # hide pp label
+            order_mode.current_pp.hide_info()
+
             # if none are pressed, remove "staged" level
             # line under cursor position
-            view.mode.lines.unstage_line()
+            order_mode.lines.unstage_line()
 
             if view.hasFocus():
                 # update mode label
-                view._chart.window().mode_label.setText('mode: view')
+                view._chart.window().set_mode_name('view')
 
-            view.order_mode = False
+            order_mode.active = False
 
         last = time.time()
 
 
+async def handle_viewmode_mouse(
+
+    view: 'ChartView',
+    recv_chan: trio.abc.ReceiveChannel,
+
+) -> None:
+
+    async for msg in recv_chan:
+        button = msg.button
+
+        # XXX: ugggh ``pyqtgraph`` has its own mouse events..
+        # so we can't overried this easily.
+        # it's going to take probably some decent
+        # reworking of the mouseClickEvent() handler.
+
+        # if button == QtCore.Qt.RightButton and view.menuEnabled():
+        #     event = mouseEvents.MouseClickEvent(msg.event)
+        #     # event.accept()
+        #     view.raiseContextMenu(event)
+
+        if (
+            view.order_mode.active and
+            button == QtCore.Qt.LeftButton
+        ):
+            # when in order mode, submit execution
+            # msg.event.accept()
+            # breakpoint()
+            view.order_mode.submit_order()
+
+
 class ChartView(ViewBox):
     '''
     Price chart view box with interaction behaviors you'd expect from
@@ -229,20 +334,46 @@ class ChartView(ViewBox):
         - zoom on right-click-n-drag to cursor position
 
     '''
-    mode_name: str = 'mode: view'
+    mode_name: str = 'view'
+
+    # "relay events" for making overlaid views work.
+    # NOTE: these MUST be defined here (and can't be monkey patched
+    # on later) due to signal construction requiring refs to be
+    # in place during the run of meta-class machinery.
+    mouseDragEventRelay = QtCore.Signal(object, object, object)
+    wheelEventRelay = QtCore.Signal(object, object, object)
+
+    event_relay_source: 'Optional[ViewBox]' = None
+    relays: dict[str, QtCore.Signal] = {}
 
     def __init__(
-
         self,
+
         name: str,
+
         parent: pg.PlotItem = None,
+        static_yrange: Optional[tuple[float, float]] = None,
         **kwargs,
 
     ):
-        super().__init__(parent=parent, **kwargs)
+        super().__init__(
+            parent=parent,
+            name=name,
+            # TODO: look into the default view padding
+            # support that might replace somem of our
+            # ``ChartPlotWidget._set_yrange()`
+            # defaultPadding=0.,
+            **kwargs
+        )
+        # for "known y-range style"
+        self._static_yrange = static_yrange
+        self._maxmin = None
 
         # disable vertical scrolling
-        self.setMouseEnabled(x=True, y=False)
+        self.setMouseEnabled(
+            x=True,
+            y=True,
+        )
 
         self.linkedsplits = None
         self._chart: 'ChartPlotWidget' = None  # noqa
@@ -251,22 +382,61 @@ class ChartView(ViewBox):
         self.select_box = SelectRect(self)
         self.addItem(self.select_box, ignoreBounds=True)
 
-        self.name = name
         self.mode = None
         self.order_mode: bool = False
 
         self.setFocusPolicy(QtCore.Qt.StrongFocus)
+        self._ic = None
+
+    def start_ic(
+        self,
+    ) -> None:
+        '''
+        Signal the beginning of a click-drag interaction
+        to any interested task waiters.
+
+        '''
+        if self._ic is None:
+            self.chart.pause_all_feeds()
+            self._ic = trio.Event()
+
+    def signal_ic(
+        self,
+        *args,
+
+    ) -> None:
+        '''
+        Signal the end of a click-drag interaction
+        to any waiters.
+
+        '''
+        if self._ic:
+            self._ic.set()
+            self._ic = None
+            self.chart.resume_all_feeds()
 
     @asynccontextmanager
     async def open_async_input_handler(
         self,
-    ) -> 'ChartView':
-        from . import _event
 
-        async with _event.open_handler(
-            self,
-            event_types={QEvent.KeyPress, QEvent.KeyRelease},
-            async_handler=handle_viewmode_inputs,
+    ) -> 'ChartView':
+
+        async with (
+            _event.open_handlers(
+                [self],
+                event_types={
+                    QEvent.KeyPress,
+                    QEvent.KeyRelease,
+                },
+                async_handler=handle_viewmode_kb_inputs,
+            ),
+            _event.open_handlers(
+                [self],
+                event_types={
+                    gs_mouse.GraphicsSceneMousePress,
+                },
+                async_handler=handle_viewmode_mouse,
+            ),
         ):
             yield self
 
@@ -278,9 +448,25 @@ class ChartView(ViewBox):
     def chart(self, chart: 'ChartPlotWidget') -> None:  # type: ignore # noqa
         self._chart = chart
         self.select_box.chart = chart
+        if self._maxmin is None:
+            self._maxmin = chart.maxmin
 
-    def wheelEvent(self, ev, axis=None):
-        '''Override "center-point" location for scrolling.
+    @property
+    def maxmin(self) -> Callable:
+        return self._maxmin
+
+    @maxmin.setter
+    def maxmin(self, callback: Callable) -> None:
+        self._maxmin = callback
+
+    def wheelEvent(
+        self,
+        ev,
+        axis=None,
+        relayed_from: ChartView = None,
+    ):
+        '''
+        Override "center-point" location for scrolling.
 
         This is an override of the ``ViewBox`` method simply changing
         the center of the zoom to be the y-axis.
@@ -298,81 +484,127 @@ class ChartView(ViewBox):
 
         # don't zoom more then the min points setting
         l, lbar, rbar, r = chart.bars_range()
-        vl = r - l
+        # vl = r - l
 
-        if ev.delta() > 0 and vl <= _min_points_to_show:
-            log.debug("Max zoom bruh...")
-            return
+        # if ev.delta() > 0 and vl <= _min_points_to_show:
+        #     log.debug("Max zoom bruh...")
+        #     return
 
-        if ev.delta() < 0 and vl >= len(chart._arrays['ohlc']) + 666:
-            log.debug("Min zoom bruh...")
-            return
+        # if (
+        #     ev.delta() < 0
+        #     and vl >= len(chart._flows[chart.name].shm.array) + 666
+        # ):
+        #     log.debug("Min zoom bruh...")
+        #     return
 
         # actual scaling factor
         s = 1.015 ** (ev.delta() * -1 / 20)  # self.state['wheelScaleFactor'])
         s = [(None if m is False else s) for m in mask]
 
-        # center = pg.Point(
-        #     fn.invertQTransform(self.childGroup.transform()).map(ev.pos())
-        # )
+        if (
+            # zoom happened on axis
+            axis == 1
 
-        # XXX: scroll "around" the right most element in the view
-        # which stays "pinned" in place.
+            # if already in axis zoom mode then keep it
+            or self.chart._static_yrange == 'axis'
+        ):
+            self.chart._static_yrange = 'axis'
+            self.setLimits(yMin=None, yMax=None)
 
-        # furthest_right_coord = self.boundingRect().topRight()
-
-        # yaxis = pg.Point(
-        #     fn.invertQTransform(
-        #         self.childGroup.transform()
-        #     ).map(furthest_right_coord)
-        # )
-
-        # This seems like the most "intuitive option, a hybrid of
-        # tws and tv styles
-        last_bar = pg.Point(int(rbar)) + 1
-
-        ryaxis = chart.getAxis('right')
-        r_axis_x = ryaxis.pos().x()
-
-        end_of_l1 = pg.Point(
-            round(
-                chart._vb.mapToView(
-                    pg.Point(r_axis_x - chart._max_l1_line_len)
-                    # QPointF(chart._max_l1_line_len, 0)
-                ).x()
+            # print(scale_y)
+            # pos = ev.pos()
+            # lastPos = ev.lastPos()
+            # dif = pos - lastPos
+            # dif = dif * -1
+            center = Point(
+                fn.invertQTransform(
+                    self.childGroup.transform()
+                ).map(ev.pos())
             )
-        )  # .x()
+            # scale_y = 1.3 ** (center.y() * -1 / 20)
+            self.scaleBy(s, center)
 
-        # self.state['viewRange'][0][1] = end_of_l1
+        else:
 
-        # focal = pg.Point((last_bar.x() + end_of_l1)/2)
+            # center = pg.Point(
+            #     fn.invertQTransform(self.childGroup.transform()).map(ev.pos())
+            # )
 
-        focal = min(
-            last_bar,
-            end_of_l1,
-            key=lambda p: p.x()
-        )
-        # focal = pg.Point(last_bar.x() + end_of_l1)
+            # XXX: scroll "around" the right most element in the view
+            # which stays "pinned" in place.
 
-        self._resetTarget()
-        self.scaleBy(s, focal)
-        ev.accept()
-        self.sigRangeChangedManually.emit(mask)
+            # furthest_right_coord = self.boundingRect().topRight()
+
+            # yaxis = pg.Point(
+            #     fn.invertQTransform(
+            #         self.childGroup.transform()
+            #     ).map(furthest_right_coord)
+            # )
+
+            # This seems like the most "intuitive option, a hybrid of
+            # tws and tv styles
+            last_bar = pg.Point(int(rbar)) + 1
+
+            ryaxis = chart.getAxis('right')
+            r_axis_x = ryaxis.pos().x()
+
+            end_of_l1 = pg.Point(
+                round(
+                    chart.cv.mapToView(
+                        pg.Point(r_axis_x - chart._max_l1_line_len)
+                        # QPointF(chart._max_l1_line_len, 0)
+                    ).x()
+                )
+            )  # .x()
+
+            # self.state['viewRange'][0][1] = end_of_l1
+            # focal = pg.Point((last_bar.x() + end_of_l1)/2)
+
+            focal = min(
+                last_bar,
+                end_of_l1,
+                key=lambda p: p.x()
+            )
+            # focal = pg.Point(last_bar.x() + end_of_l1)
+
+            self._resetTarget()
+            self.scaleBy(s, focal)
+
+            # XXX: the order of the next 2 lines i'm pretty sure
+            # matters, we want the resize to trigger before the graphics
+            # update, but i gotta feelin that because this one is signal
+            # based (and thus not necessarily sync invoked right away)
+            # that calling the resize method manually might work better.
+            self.sigRangeChangedManually.emit(mask)
+
+            # XXX: without this is seems as though sometimes
+            # when zooming in from far out (and maybe vice versa?)
+            # the signal isn't being fired enough since if you pan
+            # just after you'll see further downsampling code run
+            # (pretty noticeable on the OHLC ds curve) but with this
+            # that never seems to happen? Only question is how much this
+            # "double work" is causing latency when these missing event
+            # fires don't happen?
+            self.maybe_downsample_graphics()
+
+            ev.accept()
 
     def mouseDragEvent(
         self,
         ev,
         axis: Optional[int] = None,
+        relayed_from: ChartView = None,
+
     ) -> None:
-        #  if axis is specified, event will only affect that axis.
-        ev.accept()  # we accept all buttons
-        button = ev.button()
 
         pos = ev.pos()
         lastPos = ev.lastPos()
         dif = pos - lastPos
         dif = dif * -1
 
+        # NOTE: if axis is specified, event will only affect that axis.
+        button = ev.button()
+
         # Ignore axes if mouse is disabled
         mouseEnabled = np.array(self.state['mouseEnabled'], dtype=np.float)
         mask = mouseEnabled.copy()
@@ -380,21 +612,28 @@ class ChartView(ViewBox):
             mask[1-axis] = 0.0
 
         # Scale or translate based on mouse button
-        if button & (QtCore.Qt.LeftButton | QtCore.Qt.MidButton):
-
+        if button & (
+            QtCore.Qt.LeftButton | QtCore.Qt.MidButton
+        ):
             # zoom y-axis ONLY when click-n-drag on it
-            if axis == 1:
-                # set a static y range special value on chart widget to
-                # prevent sizing to data in view.
-                self.chart._static_yrange = 'axis'
+            # if axis == 1:
+            #     # set a static y range special value on chart widget to
+            #     # prevent sizing to data in view.
+            #     self.chart._static_yrange = 'axis'
 
-                scale_y = 1.3 ** (dif.y() * -1 / 20)
-                self.setLimits(yMin=None, yMax=None)
+            #     scale_y = 1.3 ** (dif.y() * -1 / 20)
+            #     self.setLimits(yMin=None, yMax=None)
 
-                # print(scale_y)
-                self.scaleBy((0, scale_y))
+            #     # print(scale_y)
+            #     self.scaleBy((0, scale_y))
 
-            if self.state['mouseMode'] == ViewBox.RectMode:
+            # SELECTION MODE
+            if (
+                self.state['mouseMode'] == ViewBox.RectMode
+                and axis is None
+            ):
+                # XXX: WHY
+                ev.accept()
 
                 down_pos = ev.buttonDownPos()
 
@@ -403,23 +642,40 @@ class ChartView(ViewBox):
 
                     self.select_box.mouse_drag_released(down_pos, pos)
 
-                    # ax = QtCore.QRectF(down_pos, pos)
-                    # ax = self.childGroup.mapRectFromParent(ax)
-                    # print(ax)
+                    ax = QtCore.QRectF(down_pos, pos)
+                    ax = self.childGroup.mapRectFromParent(ax)
 
                     # this is the zoom transform cmd
-                    # self.showAxRect(ax)
+                    self.showAxRect(ax)
+
+                    # axis history tracking
+                    self.axHistoryPointer += 1
+                    self.axHistory = self.axHistory[
+                        :self.axHistoryPointer] + [ax]
 
-                    # self.axHistoryPointer += 1
-                    # self.axHistory = self.axHistory[
-                    #     :self.axHistoryPointer] + [ax]
                 else:
+                    print('drag finish?')
                     self.select_box.set_pos(down_pos, pos)
 
                     # update shape of scale box
                     # self.updateScaleBox(ev.buttonDownPos(), ev.pos())
+                    self.updateScaleBox(
+                        down_pos,
+                        ev.pos(),
+                    )
+
+            # PANNING MODE
             else:
-                # default bevavior: click to pan view
+                # XXX: WHY
+                ev.accept()
+
+                self.start_ic()
+                # if self._ic is None:
+                #     self.chart.pause_all_feeds()
+                #     self._ic = trio.Event()
+
+                if axis == 1:
+                    self.chart._static_yrange = 'axis'
 
                 tr = self.childGroup.transform()
                 tr = fn.invertQTransform(tr)
@@ -435,9 +691,14 @@ class ChartView(ViewBox):
 
                 self.sigRangeChangedManually.emit(self.state['mouseEnabled'])
 
-        elif button & QtCore.Qt.RightButton:
+                if ev.isFinish():
+                    self.signal_ic()
+                    # self._ic.set()
+                    # self._ic = None
+                    # self.chart.resume_all_feeds()
 
-            # right click zoom to center behaviour
+        # WEIRD "RIGHT-CLICK CENTER ZOOM" MODE
+        elif button & QtCore.Qt.RightButton:
 
             if self.state['aspectLocked'] is not False:
                 mask[0] = 0
@@ -458,22 +719,13 @@ class ChartView(ViewBox):
             self.scaleBy(x=x, y=y, center=center)
             self.sigRangeChangedManually.emit(self.state['mouseEnabled'])
 
-    def mouseClickEvent(self, ev):
-        """Full-click callback.
-
-        """
-        button = ev.button()
-        # pos = ev.pos()
-
-        if button == QtCore.Qt.RightButton and self.menuEnabled():
+            # XXX: WHY
             ev.accept()
-            self.raiseContextMenu(ev)
 
-        elif button == QtCore.Qt.LeftButton:
-            # when in order mode, submit execution
-            if self.order_mode:
-                ev.accept()
-                self.mode.submit_exec()
+    # def mouseClickEvent(self, event: QtCore.QEvent) -> None:
+    #      '''This routine is rerouted to an async handler.
+    #      '''
+    #     pass
 
     def keyReleaseEvent(self, event: QtCore.QEvent) -> None:
         '''This routine is rerouted to an async handler.
@@ -484,3 +736,212 @@ class ChartView(ViewBox):
         '''This routine is rerouted to an async handler.
         '''
         pass
+
+    def _set_yrange(
+        self,
+        *,
+
+        yrange: Optional[tuple[float, float]] = None,
+        range_margin: float = 0.06,
+        bars_range: Optional[tuple[int, int, int, int]] = None,
+
+        # flag to prevent triggering sibling charts from the same linked
+        # set from recursion errors.
+        autoscale_linked_plots: bool = False,
+        name: Optional[str] = None,
+
+    ) -> None:
+        '''
+        Set the viewable y-range based on embedded data.
+
+        This adds auto-scaling like zoom on the scroll wheel such
+        that data always fits nicely inside the current view of the
+        data set.
+
+        '''
+        name = self.name
+        # print(f'YRANGE ON {name}')
+        profiler = pg.debug.Profiler(
+            msg=f'`ChartView._set_yrange()`: `{name}`',
+            disabled=not pg_profile_enabled(),
+            ms_threshold=ms_slower_then,
+            delayed=True,
+        )
+        set_range = True
+        chart = self._chart
+
+        # view has been set in 'axis' mode
+        # meaning it can be panned and zoomed
+        # arbitrarily on the y-axis:
+        # - disable autoranging
+        # - remove any y range limits
+        if chart._static_yrange == 'axis':
+            set_range = False
+            self.setLimits(yMin=None, yMax=None)
+
+        # static y-range has been set likely by
+        # a specialized FSP configuration.
+        elif chart._static_yrange is not None:
+            ylow, yhigh = chart._static_yrange
+
+        # range passed in by caller, usually a
+        # maxmin detection algos inside the
+        # display loop for re-draw efficiency.
+        elif yrange is not None:
+            ylow, yhigh = yrange
+
+        if set_range:
+
+            # XXX: only compute the mxmn range
+            # if none is provided as input!
+            if not yrange:
+                # flow = chart._flows[name]
+                yrange = self._maxmin()
+
+                if yrange is None:
+                    log.warning(f'No yrange provided for {name}!?')
+                    print(f"WTF NO YRANGE {name}")
+                    return
+
+            ylow, yhigh = yrange
+
+            profiler(f'callback ._maxmin(): {yrange}')
+
+            # view margins: stay within a % of the "true range"
+            diff = yhigh - ylow
+            ylow = ylow - (diff * range_margin)
+            yhigh = yhigh + (diff * range_margin)
+
+            # XXX: this often needs to be unset
+            # to get different view modes to operate
+            # correctly!
+            self.setLimits(
+                yMin=ylow,
+                yMax=yhigh,
+            )
+            self.setYRange(ylow, yhigh)
+            profiler(f'set limits: {(ylow, yhigh)}')
+
+        profiler.finish()
+
+    def enable_auto_yrange(
+        self,
+        src_vb: Optional[ChartView] = None,
+
+    ) -> None:
+        '''
+        Assign callback for rescaling y-axis automatically
+        based on data contents and ``ViewBox`` state.
+
+        '''
+        if src_vb is None:
+            src_vb = self
+
+        # splitter(s) resizing
+        src_vb.sigResized.connect(self._set_yrange)
+
+        # TODO: a smarter way to avoid calling this needlessly?
+        # 2 things i can think of:
+        # - register downsample-able graphics specially and only
+        #   iterate those.
+        # - only register this when certain downsampleable graphics are
+        #   "added to scene".
+        src_vb.sigRangeChangedManually.connect(
+            self.maybe_downsample_graphics
+        )
+
+        # mouse wheel doesn't emit XRangeChanged
+        src_vb.sigRangeChangedManually.connect(self._set_yrange)
+
+        # src_vb.sigXRangeChanged.connect(self._set_yrange)
+        # src_vb.sigXRangeChanged.connect(
+        #     self.maybe_downsample_graphics
+        # )
+
+    def disable_auto_yrange(self) -> None:
+
+        self.sigResized.disconnect(
+            self._set_yrange,
+        )
+        self.sigRangeChangedManually.disconnect(
+            self.maybe_downsample_graphics
+        )
+        self.sigRangeChangedManually.disconnect(
+            self._set_yrange,
+        )
+
+        # self.sigXRangeChanged.disconnect(self._set_yrange)
+        # self.sigXRangeChanged.disconnect(
+        #     self.maybe_downsample_graphics
+        # )
+
+    def x_uppx(self) -> float:
+        '''
+        Return the "number of x units" within a single
+        pixel currently being displayed for relevant
+        graphics items which are our children.
+
+        '''
+        graphics = [f.graphics for f in self._chart._flows.values()]
+        if not graphics:
+            return 0
+
+        for graphic in graphics:
+            xvec = graphic.pixelVectors()[0]
+            if xvec:
+                return xvec.x()
+        else:
+            return 0
+
+    def maybe_downsample_graphics(
+        self,
+        autoscale_overlays: bool = True,
+    ):
+
+        profiler = pg.debug.Profiler(
+            msg=f'ChartView.maybe_downsample_graphics() for {self.name}',
+            disabled=not pg_profile_enabled(),
+
+            # XXX: important to avoid not seeing underlying
+            # ``.update_graphics_from_flow()`` nested profiling likely
+            # due to the way delaying works and garbage collection of
+            # the profiler in the delegated method calls.
+            ms_threshold=6,
+            # ms_threshold=ms_slower_then,
+        )
+
+        # TODO: a faster single-loop-iterator way of doing this XD
+        chart = self._chart
+        linked = self.linkedsplits
+        plots = linked.subplots | {chart.name: chart}
+        for chart_name, chart in plots.items():
+            for name, flow in chart._flows.items():
+
+                if (
+                    not flow.render
+
+                    # XXX: super important to be aware of this.
+                    # or not flow.graphics.isVisible()
+                ):
+                    continue
+
+                # pass in no array which will read and render from the last
+                # passed array (normally provided by the display loop.)
+                chart.update_graphics_from_flow(
+                    name,
+                    use_vr=True,
+                )
+
+                # for each overlay on this chart auto-scale the
+                # y-range to max-min values.
+                if autoscale_overlays:
+                    overlay = chart.pi_overlay
+                    if overlay:
+                        for pi in overlay.overlays:
+                            pi.vb._set_yrange(
+                                # TODO: get the range once up front...
+                                # bars_range=br,
+                            )
+                    profiler('autoscaled linked plots')
+
+                profiler(f'<{chart_name}>.update_graphics_from_flow({name})')

piker/ui/_l1.py

@@ -199,7 +199,14 @@ class LevelLabel(YAxisLabel):
         elif self._orient_v == 'top':
             lp, rp = rect.bottomLeft(), rect.bottomRight()
 
-        p.drawLine(lp.x(), lp.y(), rp.x(), rp.y())
+        p.drawLine(
+            *map(int, [
+                lp.x(),
+                lp.y(),
+                rp.x(),
+                rp.y(),
+            ])
+        )
 
     def highlight(self, pen) -> None:
         self._pen = pen

piker/ui/_label.py

@@ -19,76 +19,22 @@ Non-shitty labels that don't re-invent the wheel.
 
 """
 from inspect import isfunction
-from typing import Callable
+from typing import Callable, Optional, Any
 
 import pyqtgraph as pg
 from PyQt5 import QtGui, QtWidgets
-from PyQt5.QtCore import QPointF, QRectF
+from PyQt5.QtWidgets import QLabel, QSizePolicy
+from PyQt5.QtCore import QPointF, QRectF, Qt
 
 from ._style import (
     DpiAwareFont,
     hcolor,
+    _font,
 )
 
 
-def vbr_left(label) -> Callable[..., float]:
-    """Return a closure which gives the scene x-coordinate for the
-    leftmost point of the containing view box.
-
-    """
-    return label.vbr().left
-
-
-def right_axis(
-
-    chart: 'ChartPlotWidget',  # noqa
-    label: 'Label',  # noqa
-    side: str = 'left',
-    offset: float = 10,
-    avoid_book: bool = True,
-    width: float = None,
-
-) -> Callable[..., float]:
-    """Return a position closure which gives the scene x-coordinate for
-    the x point on the right y-axis minus the width of the label given
-    it's contents.
-
-    """
-    ryaxis = chart.getAxis('right')
-
-    if side == 'left':
-
-        if avoid_book:
-            def right_axis_offset_by_w() -> float:
-
-                # l1 spread graphics x-size
-                l1_len = chart._max_l1_line_len
-
-                # sum of all distances "from" the y-axis
-                right_offset = l1_len + label.w + offset
-
-                return ryaxis.pos().x() - right_offset
-
-        else:
-            def right_axis_offset_by_w() -> float:
-
-                return ryaxis.pos().x() - (label.w + offset)
-
-        return right_axis_offset_by_w
-
-    elif 'right':
-
-        # axis_offset = ryaxis.style['tickTextOffset'][0]
-
-        def on_axis() -> float:
-
-            return ryaxis.pos().x()  # + axis_offset - 2
-
-        return on_axis
-
-
 class Label:
-    """
+    '''
     A plain ol' "scene label" using an underlying ``QGraphicsTextItem``.
 
     After hacking for many days on multiple "label" systems inside
@@ -104,19 +50,19 @@ class Label:
     small, re-usable label components that can actually be used to build
     production grade UIs...
 
-    """
-
+    '''
     def __init__(
-
         self,
         view: pg.ViewBox,
-
         fmt_str: str,
-        color: str = 'bracket',
+
+        color: str = 'default_light',
         x_offset: float = 0,
         font_size: str = 'small',
-        opacity: float = 0.666,
-        fields: dict = {}
+        opacity: float = 1,
+        fields: dict = {},
+        parent: pg.GraphicsObject = None,
+        update_on_range_change: bool = True,
 
     ) -> None:
 
@@ -124,9 +70,15 @@ class Label:
         self._fmt_str = fmt_str
         self._view_xy = QPointF(0, 0)
 
+        self.scene_anchor: Optional[
+            Callable[..., QPointF]
+        ] = None
+
         self._x_offset = x_offset
 
-        txt = self.txt = QtWidgets.QGraphicsTextItem()
+        txt = self.txt = QtWidgets.QGraphicsTextItem(parent=parent)
+        txt.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
+
         vb.scene().addItem(txt)
 
         # configure font size based on DPI
@@ -135,11 +87,11 @@ class Label:
         )
         dpi_font.configure_to_dpi()
         txt.setFont(dpi_font.font)
-
         txt.setOpacity(opacity)
 
         # register viewbox callbacks
-        vb.sigRangeChanged.connect(self.on_sigrange_change)
+        if update_on_range_change:
+            vb.sigRangeChanged.connect(self.on_sigrange_change)
 
         self._hcolor: str = ''
         self.color = color
@@ -157,7 +109,7 @@ class Label:
         # self.setTextInteractionFlags(QtGui.Qt.TextEditorInteraction)
 
     @property
-    def color(self):
+    def color(self) -> str:
         return self._hcolor
 
     @color.setter
@@ -165,13 +117,35 @@ class Label:
         self.txt.setDefaultTextColor(pg.mkColor(hcolor(color)))
         self._hcolor = color
 
+    def update(self) -> None:
+        '''
+        Update this label either by invoking its user defined anchoring
+        function, or by positioning to the last recorded data view
+        coordinates.
+
+        '''
+        # move label in scene coords to desired position
+        anchor = self.scene_anchor
+        if anchor:
+            self.txt.setPos(anchor())
+
+        else:
+            # position based on last computed view coordinate
+            self.set_view_pos(self._view_xy.y())
+
     def on_sigrange_change(self, vr, r) -> None:
-        self.set_view_y(self._view_xy.y())
+        return self.update()
 
     @property
     def w(self) -> float:
         return self.txt.boundingRect().width()
 
+    def scene_br(self) -> QRectF:
+        txt = self.txt
+        return txt.mapToScene(
+            txt.boundingRect()
+        ).boundingRect()
+
     @property
     def h(self) -> float:
         return self.txt.boundingRect().height()
@@ -186,18 +160,20 @@ class Label:
         assert isinstance(func(), float)
         self._anchor_func = func
 
-    def set_view_y(
+    def set_view_pos(
         self,
+
         y: float,
+        x: Optional[float] = None,
+
     ) -> None:
 
-        scene_x = self._anchor_func() or self.txt.pos().x()
+        if x is None:
+            scene_x = self._anchor_func() or self.txt.pos().x()
+            x = self.vb.mapToView(QPointF(scene_x, scene_x)).x()
 
         # get new (inside the) view coordinates / position
-        self._view_xy = QPointF(
-            self.vb.mapToView(QPointF(scene_x, scene_x)).x(),
-            y,
-        )
+        self._view_xy = QPointF(x, y)
 
         # map back to the outer UI-land "scene" coordinates
         s_xy = self.vb.mapFromView(self._view_xy)
@@ -210,9 +186,6 @@ class Label:
 
         assert s_xy == self.txt.pos()
 
-    def orient_on(self, h: str, v: str) -> None:
-        pass
-
     @property
     def fmt_str(self) -> str:
         return self._fmt_str
@@ -221,7 +194,11 @@ class Label:
     def fmt_str(self, fmt_str: str) -> None:
         self._fmt_str = fmt_str
 
-    def format(self, **fields: dict) -> str:
+    def format(
+        self,
+        **fields: dict
+
+    ) -> str:
 
         out = {}
 
@@ -229,8 +206,10 @@ class Label:
         # calcs of field data from field data
         # ex. to calculate a $value = price * size
         for k, v in fields.items():
+
             if isfunction(v):
                 out[k] = v(fields)
+
             else:
                 out[k] = v
 
@@ -246,9 +225,66 @@ class Label:
 
     def show(self) -> None:
         self.txt.show()
+        self.txt.update()
 
     def hide(self) -> None:
         self.txt.hide()
 
     def delete(self) -> None:
         self.vb.scene().removeItem(self.txt)
+
+
+class FormatLabel(QLabel):
+    '''
+    Kinda similar to above but using the widget apis.
+
+    '''
+    def __init__(
+        self,
+
+        fmt_str: str,
+        font: QtGui.QFont,
+        font_size: int,
+        font_color: str,
+
+        parent=None,
+
+    ) -> None:
+
+        super().__init__(parent)
+
+        # by default set the format string verbatim and expect user to
+        # call ``.format()`` later (presumably they'll notice the
+        # unformatted content if ``fmt_str`` isn't meant to be
+        # unformatted).
+        self.fmt_str = fmt_str
+        self.setText(fmt_str)
+
+        self.setStyleSheet(
+            f"""QLabel {{
+                color : {hcolor(font_color)};
+                font-size : {font_size}px;
+            }}
+            """
+        )
+        self.setFont(_font.font)
+        self.setTextFormat(Qt.MarkdownText)  # markdown
+        self.setMargin(0)
+
+        self.setSizePolicy(
+            QSizePolicy.Expanding,
+            QSizePolicy.Expanding,
+        )
+        self.setAlignment(
+            Qt.AlignVCenter | Qt.AlignLeft
+        )
+        self.setText(self.fmt_str)
+
+    def format(
+        self,
+        **fields: dict[str, Any],
+
+    ) -> str:
+        out = self.fmt_str.format(**fields)
+        self.setText(out)
+        return out

piker/ui/_lines.py

@@ -18,17 +18,24 @@
 Lines for orders, alerts, L2.
 
 """
+from functools import partial
 from math import floor
-from typing import Tuple, Optional, List
+from typing import Optional, Callable
 
 import pyqtgraph as pg
 from pyqtgraph import Point, functions as fn
 from PyQt5 import QtCore, QtGui, QtWidgets
 from PyQt5.QtCore import QPointF
 
-from .._annotate import mk_marker, qgo_draw_markers
-from .._label import Label, vbr_left, right_axis
-from .._style import hcolor, _font
+from ._annotate import qgo_draw_markers, LevelMarker
+from ._anchors import (
+    vbr_left,
+    right_axis,
+    gpath_pin,
+)
+from ..calc import humanize
+from ._label import Label
+from ._style import hcolor, _font
 
 
 # TODO: probably worth investigating if we can
@@ -36,12 +43,6 @@ from .._style import hcolor, _font
 # https://stackoverflow.com/questions/26156486/determine-bounding-rect-of-line-in-qt
 class LevelLine(pg.InfiniteLine):
 
-    # TODO: fill in these slots for orders
-    # available parent signals
-    # sigDragged(self)
-    # sigPositionChangeFinished(self)
-    # sigPositionChanged(self)
-
     def __init__(
         self,
         chart: 'ChartPlotWidget',  # type: ignore # noqa
@@ -50,19 +51,20 @@ class LevelLine(pg.InfiniteLine):
         color: str = 'default',
         highlight_color: str = 'default_light',
         dotted: bool = False,
-        marker_size: int = 20,
 
         # UX look and feel opts
         always_show_labels: bool = False,
-        hl_on_hover: bool = True,
+        highlight_on_hover: bool = True,
         hide_xhair_on_hover: bool = True,
         only_show_markers_on_hover: bool = True,
         use_marker_margin: bool = False,
-
         movable: bool = True,
 
     ) -> None:
 
+        # TODO: at this point it's probably not worth the inheritance
+        # any more since  we've reimplemented ``.pain()`` among other
+        # things..
         super().__init__(
             movable=movable,
             angle=0,
@@ -72,13 +74,16 @@ class LevelLine(pg.InfiniteLine):
         )
 
         self._chart = chart
-        self._hoh = hl_on_hover
+        self.highlight_on_hover = highlight_on_hover
         self._dotted = dotted
         self._hide_xhair_on_hover = hide_xhair_on_hover
 
+        # callback that can be assigned by user code
+        # to get updates from each level change
+        self._on_level_change: Callable[[float], None] = lambda y: None
+
         self._marker = None
-        self._default_mkr_size = marker_size
-        self._moh = only_show_markers_on_hover
+        self.only_show_markers_on_hover = only_show_markers_on_hover
         self.show_markers: bool = True  # presuming the line is hovered at init
 
         # should line go all the way to far end or leave a "margin"
@@ -97,8 +102,8 @@ class LevelLine(pg.InfiniteLine):
 
         # list of labels anchored at one of the 2 line endpoints
         # inside the viewbox
-        self._labels: List[(int, Label)] = []
-        self._markers: List[(int, Label)] = []
+        self._labels: list[Label] = []
+        self._markers: list[(int, Label)] = []
 
         # whenever this line is moved trigger label updates
         self.sigPositionChanged.connect(self.on_pos_change)
@@ -109,17 +114,15 @@ class LevelLine(pg.InfiniteLine):
 
         # TODO: for when we want to move groups of lines?
         self._track_cursor: bool = False
-        self._always_show_labels = always_show_labels
+        self.always_show_labels = always_show_labels
 
         self._on_drag_start = lambda l: None
         self._on_drag_end = lambda l: None
 
-        self._y_incr_mult = 1 / chart._lc._symbol.tick_size
-        self._last_scene_y: float = 0
-
+        self._y_incr_mult = 1 / chart.linked.symbol.tick_size
         self._right_end_sc: float = 0
 
-    def txt_offsets(self) -> Tuple[int, int]:
+    def txt_offsets(self) -> tuple[int, int]:
         return 0, 0
 
     @property
@@ -143,52 +146,6 @@ class LevelLine(pg.InfiniteLine):
         hoverpen.setWidth(2)
         self.hoverPen = hoverpen
 
-    def add_label(
-        self,
-
-        # by default we only display the line's level value
-        # in the label
-        fmt_str: str = (
-            '{level:,.{level_digits}f}'
-        ),
-        side: str = 'right',
-        side_of_axis: str = 'left',
-        x_offset: float = 0,
-
-        color: str = None,
-        bg_color: str = None,
-        avoid_book: bool = True,
-
-        **label_kwargs,
-    ) -> Label:
-        """Add a ``LevelLabel`` anchored at one of the line endpoints in view.
-
-        """
-        label = Label(
-            view=self.getViewBox(),
-            fmt_str=fmt_str,
-            color=self.color,
-        )
-
-        # set anchor callback
-        if side == 'right':
-            label.set_x_anchor_func(
-                right_axis(
-                    self._chart,
-                    label,
-                    side=side_of_axis,
-                    offset=x_offset,
-                    avoid_book=avoid_book,
-                )
-            )
-
-        elif side == 'left':
-            label.set_x_anchor_func(vbr_left(label))
-
-        self._labels.append((side, label))
-
-        return label
-
     def on_pos_change(
         self,
         line: 'LevelLine',  # noqa
@@ -196,61 +153,75 @@ class LevelLine(pg.InfiniteLine):
         """Position changed handler.
 
         """
-        self.update_labels({'level': self.value()})
+        level = self.value()
+        self.update_labels({'level': level})
+        self.set_level(level, called_from_on_pos_change=True)
 
     def update_labels(
         self,
         fields_data: dict,
+
     ) -> None:
 
-        for at, label in self._labels:
-            label.color = self.color
-            # print(f'color is {self.color}')
+        for label in self._labels:
 
+            label.color = self.color
             label.fields.update(fields_data)
+            label.render()
 
             level = fields_data.get('level')
             if level:
-                label.set_view_y(level)
-
-            label.render()
+                label.set_view_pos(y=level)
 
         self.update()
 
     def hide_labels(self) -> None:
-        for at, label in self._labels:
+        for label in self._labels:
             label.hide()
 
     def show_labels(self) -> None:
-        for at, label in self._labels:
+        for label in self._labels:
             label.show()
 
     def set_level(
         self,
         level: float,
+        called_from_on_pos_change: bool = False,
+
     ) -> None:
-        last = self.value()
 
-        # if the position hasn't changed then ``.update_labels()``
-        # will not be called by a non-triggered `.on_pos_change()`,
-        # so we need to call it manually to avoid mismatching
-        # label-to-line color when the line is updated but not
-        # "moved".
-        if level == last:
-            self.update_labels({'level': level})
+        if not called_from_on_pos_change:
+            last = self.value()
+
+            # if the position hasn't changed then ``.update_labels()``
+            # will not be called by a non-triggered `.on_pos_change()`,
+            # so we need to call it manually to avoid mismatching
+            # label-to-line color when the line is updated but not
+            # from a "moved" event.
+            if level == last:
+                self.update_labels({'level': level})
+
+            self.setPos(level)
 
-        self.setPos(level)
         self.level = self.value()
         self.update()
 
+        # invoke any user code
+        self._on_level_change(level)
+
     def on_tracked_source(
         self,
+
         x: int,
         y: float
+
     ) -> None:
-        # XXX: this is called by our ``Cursor`` type once this
-        # line is set to track the cursor: for every movement
-        # this callback is invoked to reposition the line
+        '''Chart coordinates cursor tracking callback.
+
+        this is called by our ``Cursor`` type once this line is set to
+        track the cursor: for every movement this callback is invoked to
+        reposition the line with the current view coordinates.
+        '''
         self.movable = True
         self.set_level(y)  # implictly calls reposition handler
 
@@ -316,9 +287,10 @@ class LevelLine(pg.InfiniteLine):
         """
         scene = self.scene()
         if scene:
-            for at, label in self._labels:
+            for label in self._labels:
                 label.delete()
 
+            # gc managed labels?
             self._labels.clear()
 
             if self._marker:
@@ -341,51 +313,31 @@ class LevelLine(pg.InfiniteLine):
         # TODO: enter labels edit mode
         print(f'double click {ev}')
 
-    def right_point(
-        self,
-    ) -> float:
-
-        chart = self._chart
-        l1_len = chart._max_l1_line_len
-        ryaxis = chart.getAxis('right')
-        up_to_l1_sc = ryaxis.pos().x() - l1_len
-
-        return up_to_l1_sc
-
     def paint(
         self,
+
         p: QtGui.QPainter,
         opt: QtWidgets.QStyleOptionGraphicsItem,
         w: QtWidgets.QWidget
+
     ) -> None:
-        """Core paint which we override (yet again)
+        '''
+        Core paint which we override (yet again)
         from pg..
 
-        """
+        '''
         p.setRenderHint(p.Antialiasing)
 
         # these are in viewbox coords
         vb_left, vb_right = self._endPoints
-
-        chart = self._chart
-        l1_len = chart._max_l1_line_len
-        ryaxis = chart.getAxis('right')
-
-        r_axis_x = ryaxis.pos().x()
-        up_to_l1_sc = r_axis_x - l1_len
-
         vb = self.getViewBox()
 
-        size = self._default_mkr_size
-        marker_right = up_to_l1_sc - (1.375 * 2*size)
-        line_end = marker_right - (6/16 * size)
+        line_end, marker_right, r_axis_x = self._chart.marker_right_points()
 
         if self.show_markers and self.markers:
 
-            size = self.markers[0][2]
-
             p.setPen(self.pen)
-            size = qgo_draw_markers(
+            qgo_draw_markers(
                 self.markers,
                 self.pen.color(),
                 p,
@@ -400,9 +352,14 @@ class LevelLine(pg.InfiniteLine):
         # order lines.. not sure wtf is up with that.
         # for now we're just using it on the position line.
         elif self._marker:
+
+            # TODO: make this label update part of a scene-aware-marker
+            # composed annotation
             self._marker.setPos(
                 QPointF(marker_right, self.scene_y())
             )
+            if hasattr(self._marker, 'label'):
+                self._marker.label.update()
 
         elif not self.use_marker_margin:
             # basically means **don't** shorten the line with normally
@@ -424,39 +381,51 @@ class LevelLine(pg.InfiniteLine):
         super().hide()
         if self._marker:
             self._marker.hide()
+            # needed for ``order_line()`` lines currently
+            self._marker.label.hide()
 
-    def scene_right_xy(self) -> QPointF:
-        return self.getViewBox().mapFromView(
-            QPointF(0, self.value())
-        )
+    def show(self) -> None:
+        super().show()
+        if self._marker:
+            self._marker.show()
+            # self._marker.label.show()
 
     def scene_y(self) -> float:
-        return self.getViewBox().mapFromView(Point(0, self.value())).y()
+        return self.getViewBox().mapFromView(
+            Point(0, self.value())
+        ).y()
+
+    def scene_endpoint(self) -> QPointF:
+
+        if not self._right_end_sc:
+            line_end, _, _ = self._chart.marker_right_points()
+            self._right_end_sc = line_end - 10
+
+        return QPointF(self._right_end_sc, self.scene_y())
 
     def add_marker(
         self,
         path: QtWidgets.QGraphicsPathItem,
-    ) -> None:
 
-        # chart = self._chart
-        vb = self.getViewBox()
-        vb.scene().addItem(path)
+    ) -> QtWidgets.QGraphicsPathItem:
 
         self._marker = path
-
-        rsc = self.right_point()
-
         self._marker.setPen(self.currentPen)
         self._marker.setBrush(fn.mkBrush(self.currentPen.color()))
-        # y_in_sc = chart._vb.mapFromView(Point(0, self.value())).y()
+        # add path to scene
+        self.getViewBox().scene().addItem(path)
+
+        # place to just-left of L1 labels
+        rsc = self._chart.pre_l1_xs()[0]
         path.setPos(QPointF(rsc, self.scene_y()))
 
-        # self.update()
+        return path
 
     def hoverEvent(self, ev):
-        """Mouse hover callback.
+        '''
+        Mouse hover callback.
 
-        """
+        '''
         cur = self._chart.linked.cursor
 
         # hovered
@@ -466,11 +435,14 @@ class LevelLine(pg.InfiniteLine):
             if self.mouseHovering is True:
                 return
 
-            if self._moh:
+            if self.only_show_markers_on_hover:
                 self.show_markers = True
 
+                if self._marker:
+                    self._marker.show()
+
             # highlight if so configured
-            if self._hoh:
+            if self.highlight_on_hover:
 
                 self.currentPen = self.hoverPen
 
@@ -509,17 +481,18 @@ class LevelLine(pg.InfiniteLine):
 
             cur._hovered.remove(self)
 
-            if self._moh:
+            if self.only_show_markers_on_hover:
                 self.show_markers = False
 
+                if self._marker:
+                    self._marker.hide()
+                    self._marker.label.hide()
+
             if self not in cur._trackers:
                 cur.show_xhair(y_label_level=self.value())
 
-            if not self._always_show_labels:
-                for at, label in self._labels:
-                    label.hide()
-                    label.txt.update()
-                    # label.unhighlight()
+            if not self.always_show_labels:
+                self.hide_labels()
 
             self.mouseHovering = False
 
@@ -527,33 +500,28 @@ class LevelLine(pg.InfiniteLine):
 
 
 def level_line(
+
     chart: 'ChartPlotWidget',  # noqa
     level: float,
-    color: str = 'default',
-
-    # whether or not the line placed in view should highlight
-    # when moused over (aka "hovered")
-    hl_on_hover: bool = True,
 
     # line style
     dotted: bool = False,
+    color: str = 'default',
+
+    # ux
+    highlight_on_hover: bool = True,
 
     # label fields and options
-    digits: int = 1,
-
     always_show_labels: bool = False,
-
     add_label: bool = True,
-
     orient_v: str = 'bottom',
-
     **kwargs,
 
 ) -> LevelLine:
     """Convenience routine to add a styled horizontal line to a plot.
 
     """
-    hl_color = color + '_light' if hl_on_hover else color
+    hl_color = color + '_light' if highlight_on_hover else color
 
     line = LevelLine(
         chart,
@@ -565,7 +533,7 @@ def level_line(
         dotted=dotted,
 
         # UX related options
-        hl_on_hover=hl_on_hover,
+        highlight_on_hover=highlight_on_hover,
 
         # when set to True the label is always shown instead of just on
         # highlight (which is a privacy thing for orders)
@@ -578,17 +546,36 @@ def level_line(
 
     if add_label:
 
-        label = line.add_label(
-            side='right',
-            opacity=1,
-            x_offset=0,
-            avoid_book=False,
-        )
-        label.orient_v = orient_v
+        label = Label(
 
+            view=line.getViewBox(),
+
+            # by default we only display the line's level value
+            # in the label
+            fmt_str=('{level:,.{level_digits}f}'),
+            color=color,
+        )
+
+        # anchor to right side (of view ) label
+        label.set_x_anchor_func(
+            right_axis(
+                chart,
+                label,
+                side='left',  # side of axis
+                offset=0,
+                avoid_book=False,
+            )
+        )
+
+        # add to label set which will be updated on level changes
+        line._labels.append(label)
+
+        label.orient_v = orient_v
         line.update_labels({'level': level, 'level_digits': 2})
         label.render()
 
+        # keep pp label details private until
+        # the user edge triggers "order mode"
         line.hide_labels()
 
     # activate/draw label
@@ -598,128 +585,191 @@ def level_line(
 
 
 def order_line(
+
     chart,
     level: float,
-    level_digits: float,
-    action: str,  # buy or sell
+    action: Optional[str] = 'buy',  # buy or sell
 
+    marker_style: Optional[str] = None,
+    level_digits: Optional[float] = 3,
     size: Optional[int] = 1,
-    size_digits: int = 0,
+    size_digits: int = 1,
     show_markers: bool = False,
     submit_price: float = None,
-    exec_type: str = 'dark',
-    order_type: str = 'limit',
     orient_v: str = 'bottom',
 
     **line_kwargs,
 
 ) -> LevelLine:
-    """Convenience routine to add a line graphic representing an order
+    '''
+    Convenience routine to add a line graphic representing an order
     execution submitted to the EMS via the chart's "order mode".
 
-    """
+    '''
     line = level_line(
         chart,
         level,
         add_label=False,
         use_marker_margin=True,
-        # only_show_markers_on_hover=True,
         **line_kwargs
     )
 
-    if show_markers:
-        font_size = _font.font.pixelSize()
+    font_size = _font.font.pixelSize()
 
-        # scale marker size with dpi-aware font size
-        arrow_size = floor(1.375 * font_size)
-        alert_size = arrow_size * 0.666
-
-        # add arrow marker on end of line nearest y-axis
-        marker_style, marker_size = {
-            'buy': ('|<', arrow_size),
-            'sell': ('>|', arrow_size),
-            'alert': ('v', alert_size),
-        }[action]
-
-        # this fixes it the artifact issue! .. of course, bouding rect stuff
-        line._maxMarkerSize = marker_size
-
-        # use ``QPathGraphicsItem``s to draw markers in scene coords
-        # instead of the old way that was doing the same but by
-        # resetting the graphics item transform intermittently
-
-        # XXX: this is our new approach but seems slower?
-        # line.add_marker(mk_marker(marker_style, marker_size))
-
-        assert not line.markers
-
-        # the old way which is still somehow faster?
-        path = mk_marker(
-            marker_style,
-            # the "position" here is now ignored since we modified
-            # internals to pin markers to the right end of the line
-            marker_size,
-            use_qgpath=False,
-        )
-        # manually append for later ``InfiniteLine.paint()`` drawing
-        # XXX: this was manually tested as faster then using the
-        # QGraphicsItem around a painter path.. probably needs further
-        # testing to figure out why tf that's true.
-        line.markers.append((path, 0, marker_size))
+    # scale marker size with dpi-aware font size
+    marker_size = floor(1.375 * font_size)
 
     orient_v = 'top' if action == 'sell' else 'bottom'
 
     if action == 'alert':
-        # completely different labelling for alerts
-        fmt_str = 'alert => {level}'
+
+        label = Label(
+
+            view=line.getViewBox(),
+            color=line.color,
+
+            # completely different labelling for alerts
+            fmt_str='alert => {level}',
+        )
 
         # for now, we're just duplicating the label contents i guess..
-        llabel = line.add_label(
-            side='left',
-            fmt_str=fmt_str,
-        )
-        llabel.fields = {
+        line._labels.append(label)
+
+        # anchor to left side of view / line
+        label.set_x_anchor_func(vbr_left(label))
+
+        label.fields = {
             'level': level,
             'level_digits': level_digits,
         }
-        llabel.orient_v = orient_v
-        llabel.render()
-        llabel.show()
+
+        marker_size = marker_size * 0.666
 
     else:
-        # # left side label
-        # llabel = line.add_label(
-        #     side='left',
-        #     fmt_str=' {exec_type}-{order_type}:\n ${$value}',
-        # )
-        # llabel.fields = {
-        #     'order_type': order_type,
-        #     'level': level,
-        #     '$value': lambda f: f['level'] * f['size'],
-        #     'size': size,
-        #     'exec_type': exec_type,
-        # }
-        # llabel.orient_v = orient_v
-        # llabel.render()
-        # llabel.show()
+        view = line.getViewBox()
 
-        # right before L1 label
-        rlabel = line.add_label(
-            side='right',
-            side_of_axis='left',
-            x_offset=4*marker_size,
+        # far-side label
+        label = Label(
+            view=view,
+            # display the order pos size, which is some multiple
+            # of the user defined base unit size
             fmt_str=(
-                '{size:.{size_digits}f} '
+                '{account_text}{size:.{size_digits}f}u{fiat_text}'
             ),
+            color=line.color,
         )
-        rlabel.fields = {
+
+        label.set_x_anchor_func(vbr_left(label))
+
+        line._labels.append(label)
+
+        def maybe_show_fiat_text(fields: dict) -> str:
+            fiat_size = fields.get('fiat_size')
+            if not fiat_size:
+                return ''
+
+            return f' ~ ${humanize(fiat_size)}'
+
+        def maybe_show_account_name(fields: dict) -> str:
+            account = fields.get('account')
+            if not account:
+                return ''
+
+            return f'{account}: '
+
+        label.fields = {
             'size': size,
-            'size_digits': size_digits,
+            'size_digits': 0,
+            'fiat_size': None,
+            'fiat_text': maybe_show_fiat_text,
+            'account': None,
+            'account_text': maybe_show_account_name,
         }
 
-        rlabel.orient_v = orient_v
-        rlabel.render()
-        rlabel.show()
+    label.orient_v = orient_v
+    label.render()
+    label.show()
+
+    if show_markers:
+        # add arrow marker on end of line nearest y-axis
+        marker_style = marker_style or {
+            'buy': '|<',
+            'sell': '>|',
+            'alert': 'v',
+        }[action]
+
+        # the old way which is still somehow faster?
+        marker = LevelMarker(
+            chart=chart,
+            style=marker_style,
+            get_level=line.value,
+            size=marker_size,
+            keep_in_view=False,
+        )
+
+        # XXX: this is our new approach but seems slower?
+        marker = line.add_marker(marker)
+
+        # XXX: DON'T COMMENT THIS!
+        # this fixes it the artifact issue! .. of course, bounding rect stuff
+        line._maxMarkerSize = marker_size
+
+        assert line._marker is marker
+        assert not line.markers
+
+        # above we use ``QPathGraphicsItem``s directly to draw markers
+        # in scene coords instead of the way ``InfiniteLine`` does it
+        # internally: by resetting the graphics item transform
+        # intermittently inside ``.paint()`` which we've copied and
+        # seperated as ``qgo_draw_markers()`` if we ever want to go back
+        # to it; likely we can remove this.
+
+        # manually append for later ``InfiniteLine.paint()`` drawing
+        # XXX: this was manually tested as faster then using the
+        # QGraphicsItem around a painter path.. probably needs further
+        # testing to figure out why tf that's true.
+        # line.markers.append((marker, 0, marker_size))
+
+        if action != 'alert':
+
+            # add a partial position label if we also added a level marker
+            pp_size_label = Label(
+                view=view,
+                color=line.color,
+
+                # this is "static" label
+                # update_on_range_change=False,
+                fmt_str='\n'.join((
+                    '{slots_used:.1f}x',
+                )),
+
+                fields={
+                    'slots_used': 0,
+                },
+            )
+            pp_size_label.render()
+            pp_size_label.show()
+
+            line._labels.append(pp_size_label)
+
+            # TODO: pretty sure one of the reasons these "label
+            # updatess" are a bit "jittery" is because we aren't
+            # leveraging the "scene coordinates hierarchy" stuff:
+            # i.e. using some parent object as the coord "origin"
+            # which i presume would result in better pixel caching
+            # results? def something to dig into..
+            pp_size_label.scene_anchor = partial(
+                gpath_pin,
+                gpath=marker,
+                label=pp_size_label,
+            )
+            # XXX: without this the pp proportion label next the marker
+            # seems to lag?  this is the same issue we had with position
+            # lines which we handle with ``.update_graphcis()``.
+            # marker._on_paint=lambda marker: pp_size_label.update()
+            marker._on_paint = lambda marker: pp_size_label.update()
+
+        marker.label = label
 
     # sanity check
     line.update_labels({'level': level})
@@ -727,96 +777,27 @@ def order_line(
     return line
 
 
-def position_line(
-    chart,
-    size: float,
+# TODO: should probably consider making this a more general
+# purpose class method on the type?
+def copy_from_order_line(
 
-    level: float,
-
-    orient_v: str = 'bottom',
+    chart: 'ChartPlotWidget',  # noqa
+    line: LevelLine
 
 ) -> LevelLine:
-    """Convenience routine to add a line graphic representing an order
-    execution submitted to the EMS via the chart's "order mode".
 
-    """
-    line = level_line(
+    return order_line(
+
         chart,
-        level,
-        color='default_light',
-        add_label=False,
-        hl_on_hover=False,
-        movable=False,
-        always_show_labels=False,
-        hide_xhair_on_hover=False,
-        use_marker_margin=True,
+
+        # label fields default values
+        level=line.value(),
+        marker_style=line._marker.style,
+
+        # LevelLine kwargs
+        color=line.color,
+        dotted=line._dotted,
+
+        show_markers=line.show_markers,
+        only_show_markers_on_hover=line.only_show_markers_on_hover,
     )
-    #  hide position marker when out of view (for now)
-    vb = line.getViewBox()
-
-    def update_pp_nav(chartview):
-        vr = vb.state['viewRange']
-        ymn, ymx = vr[1]
-        level = line.value()
-
-        if gt := level > ymx or (lt := level < ymn):
-
-            if chartview.mode.name == 'order':
-
-                # provide "nav hub" like indicator for where
-                # the position is on the y-dimension
-                if gt:
-                    # pin to top of view since position is above current
-                    # y-range
-                    pass
-
-                elif lt:
-                    # pin to bottom of view since position is above
-                    # below y-range
-                    pass
-
-            else:
-                # order mode is not active
-                # so hide the pp market
-                line._marker.hide()
-
-        else:
-            # pp line is viewable so show marker
-            line._marker.show()
-
-    vb.sigYRangeChanged.connect(update_pp_nav)
-
-    rlabel = line.add_label(
-        side='right',
-        fmt_str='{direction}: {size} -> ${$:.2f}',
-    )
-    rlabel.fields = {
-        'direction': 'long' if size > 0 else 'short',
-        '$': size * level,
-        'size': size,
-    }
-    rlabel.orient_v = orient_v
-    rlabel.render()
-    rlabel.show()
-
-    # arrow marker
-    # scale marker size with dpi-aware font size
-    font_size = _font.font.pixelSize()
-
-    # scale marker size with dpi-aware font size
-    arrow_size = floor(1.375 * font_size)
-
-    if size > 0:
-        style = '|<'
-    elif size < 0:
-        style = '>|'
-
-    arrow_path = mk_marker(style, size=arrow_size)
-    # XXX: uses new marker drawing approach
-    line.add_marker(arrow_path)
-    line.set_level(level)
-
-    # sanity check
-    line.update_labels({'level': level})
-
-    return line

piker/ui/_ohlc.py

@@ -0,0 +1,250 @@
+# piker: trading gear for hackers
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Super fast OHLC sampling graphics types.
+
+"""
+from __future__ import annotations
+from typing import (
+    Optional,
+    TYPE_CHECKING,
+)
+
+import numpy as np
+import pyqtgraph as pg
+from PyQt5 import QtCore, QtGui, QtWidgets
+from PyQt5.QtCore import QLineF, QPointF
+from PyQt5.QtGui import QPainterPath
+
+from .._profile import pg_profile_enabled, ms_slower_then
+from ._style import hcolor
+from ..log import get_logger
+
+if TYPE_CHECKING:
+    from ._chart import LinkedSplits
+
+
+log = get_logger(__name__)
+
+
+def bar_from_ohlc_row(
+    row: np.ndarray,
+    # 0.5 is no overlap between arms, 1.0 is full overlap
+    w: float = 0.43
+
+) -> tuple[QLineF]:
+    '''
+    Generate the minimal ``QLineF`` lines to construct a single
+    OHLC "bar" for use in the "last datum" of a series.
+
+    '''
+    open, high, low, close, index = row[
+        ['open', 'high', 'low', 'close', 'index']]
+
+    # TODO: maybe consider using `QGraphicsLineItem` ??
+    # gives us a ``.boundingRect()`` on the objects which may make
+    # computing the composite bounding rect of the last bars + the
+    # history path faster since it's done in C++:
+    # https://doc.qt.io/qt-5/qgraphicslineitem.html
+
+    # high -> low vertical (body) line
+    if low != high:
+        hl = QLineF(index, low, index, high)
+    else:
+        # XXX: if we don't do it renders a weird rectangle?
+        # see below for filtering this later...
+        hl = None
+
+    # NOTE: place the x-coord start as "middle" of the drawing range such
+    # that the open arm line-graphic is at the left-most-side of
+    # the index's range according to the view mapping coordinates.
+
+    # open line
+    o = QLineF(index - w, open, index, open)
+
+    # close line
+    c = QLineF(index, close, index + w, close)
+
+    return [hl, o, c]
+
+
+class BarItems(pg.GraphicsObject):
+    '''
+    "Price range" bars graphics rendered from a OHLC sampled sequence.
+
+    '''
+    def __init__(
+        self,
+        linked: LinkedSplits,
+        plotitem: 'pg.PlotItem',  # noqa
+        pen_color: str = 'bracket',
+        last_bar_color: str = 'bracket',
+
+        name: Optional[str] = None,
+
+    ) -> None:
+        super().__init__()
+        self.linked = linked
+        # XXX: for the mega-lulz increasing width here increases draw
+        # latency...  so probably don't do it until we figure that out.
+        self._color = pen_color
+        self.bars_pen = pg.mkPen(hcolor(pen_color), width=1)
+        self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2)
+        self._name = name
+
+        self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
+        self.path = QPainterPath()
+        self._last_bar_lines: Optional[tuple[QLineF, ...]] = None
+
+    def x_uppx(self) -> int:
+        # we expect the downsample curve report this.
+        return 0
+
+    def boundingRect(self):
+        # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect
+
+        # TODO: Can we do rect caching to make this faster
+        # like `pg.PlotCurveItem` does? In theory it's just
+        # computing max/min stuff again like we do in the udpate loop
+        # anyway. Not really sure it's necessary since profiling already
+        # shows this method is faf.
+
+        # boundingRect _must_ indicate the entire area that will be
+        # drawn on or else we will get artifacts and possibly crashing.
+        # (in this case, QPicture does all the work of computing the
+        # bounding rect for us).
+
+        # apparently this a lot faster says the docs?
+        # https://doc.qt.io/qt-5/qpainterpath.html#controlPointRect
+        hb = self.path.controlPointRect()
+        hb_tl, hb_br = (
+            hb.topLeft(),
+            hb.bottomRight(),
+        )
+
+        # need to include last bar height or BR will be off
+        mx_y = hb_br.y()
+        mn_y = hb_tl.y()
+
+        last_lines = self._last_bar_lines
+        if last_lines:
+            body_line = self._last_bar_lines[0]
+            if body_line:
+                mx_y = max(mx_y, max(body_line.y1(), body_line.y2()))
+                mn_y = min(mn_y, min(body_line.y1(), body_line.y2()))
+
+        return QtCore.QRectF(
+
+            # top left
+            QPointF(
+                hb_tl.x(),
+                mn_y,
+            ),
+
+            # bottom right
+            QPointF(
+                hb_br.x() + 1,
+                mx_y,
+            )
+
+        )
+
+    def paint(
+        self,
+        p: QtGui.QPainter,
+        opt: QtWidgets.QStyleOptionGraphicsItem,
+        w: QtWidgets.QWidget
+
+    ) -> None:
+
+        profiler = pg.debug.Profiler(
+            disabled=not pg_profile_enabled(),
+            ms_threshold=ms_slower_then,
+        )
+
+        # p.setCompositionMode(0)
+
+        # TODO: one thing we could try here is pictures being drawn of
+        # a fixed count of bars such that based on the viewbox indices we
+        # only draw the "rounded up" number of "pictures worth" of bars
+        # as is necesarry for what's in "view". Not sure if this will
+        # lead to any perf gains other then when zoomed in to less bars
+        # in view.
+        p.setPen(self.last_bar_pen)
+        if self._last_bar_lines:
+            p.drawLines(*tuple(filter(bool, self._last_bar_lines)))
+            profiler('draw last bar')
+
+        p.setPen(self.bars_pen)
+        p.drawPath(self.path)
+        profiler(f'draw history path: {self.path.capacity()}')
+
+    def draw_last_datum(
+        self,
+        path: QPainterPath,
+        src_data: np.ndarray,
+        render_data: np.ndarray,
+        reset: bool,
+        array_key: str,
+
+        fields: list[str] = [
+            'index',
+            'open',
+            'high',
+            'low',
+            'close',
+        ],
+
+    ) -> None:
+
+        # relevant fields
+        ohlc = src_data[fields]
+        last_row = ohlc[-1:]
+
+        # individual values
+        last_row = i, o, h, l, last = ohlc[-1]
+
+        # generate new lines objects for updatable "current bar"
+        self._last_bar_lines = bar_from_ohlc_row(last_row)
+
+        # assert i == graphics.start_index - 1
+        # assert i == last_index
+        body, larm, rarm = self._last_bar_lines
+
+        # XXX: is there a faster way to modify this?
+        rarm.setLine(rarm.x1(), last, rarm.x2(), last)
+
+        # writer is responsible for changing open on "first" volume of bar
+        larm.setLine(larm.x1(), o, larm.x2(), o)
+
+        if l != h:  # noqa
+
+            if body is None:
+                body = self._last_bar_lines[0] = QLineF(i, l, i, h)
+            else:
+                # update body
+                body.setLine(i, l, i, h)
+
+            # XXX: pretty sure this is causing an issue where the
+            # bar has a large upward move right before the next
+            # sample and the body is getting set to None since the
+            # next bar is flat but the shm array index update wasn't
+            # read by the time this code runs. Iow we're doing this
+            # removal of the body for a bar index that is now out of
+            # date / from some previous sample. It's weird though
+            # because i've seen it do this to bars i - 3 back?
+
+        return ohlc['index'], ohlc['close']

piker/ui/_orm.py

@@ -0,0 +1,129 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+micro-ORM for coupling ``pydantic`` models with Qt input/output widgets.
+
+"""
+from __future__ import annotations
+from typing import (
+    Optional, Generic,
+    TypeVar, Callable,
+    Literal,
+)
+import enum
+import sys
+
+from pydantic import BaseModel, validator
+from pydantic.generics import GenericModel
+from PyQt5.QtWidgets import (
+    QWidget,
+    QComboBox,
+)
+
+from ._forms import (
+    # FontScaledDelegate,
+    Edit,
+)
+
+
+DataType = TypeVar('DataType')
+
+
+class Field(GenericModel, Generic[DataType]):
+    widget_factory: Optional[
+        Callable[
+            [QWidget, 'Field'],
+            QWidget
+        ]
+    ]
+    value: Optional[DataType] = None
+
+
+class Selection(Field[DataType], Generic[DataType]):
+    '''Model which maps to a finite set of drop down entries declared as
+    a ``dict[str, DataType]``.
+
+    '''
+    widget_factory = QComboBox
+    options: dict[str, DataType]
+    # value: DataType = None
+
+    @validator('value')  # , always=True)
+    def set_value_first(
+        cls,
+
+        v: DataType,
+        values: dict[str, DataType],
+
+    ) -> DataType:
+        '''If no initial value is set, use the first in
+        the ``options`` dict.
+
+        '''
+        # breakpoint()
+        options = values['options']
+        if v is None:
+            return next(options.values())
+        else:
+            assert v in options, f'{v} is not in {options}'
+            return v
+
+
+# class SizeUnit(Enum):
+
+#     currency = '$ size'
+#     percent_of_port = '% of port'
+#     shares = '# shares'
+
+
+# class Weighter(str, Enum):
+#     uniform = 'uniform'
+
+
+class Edit(Field[DataType], Generic[DataType]):
+    '''An edit field which takes a number.
+    '''
+    widget_factory = Edit
+
+
+class AllocatorPane(BaseModel):
+
+    account = Selection[str](
+        options=dict.fromkeys(
+            ['paper', 'ib.paper', 'ib.margin'],
+            'paper',
+        ),
+    )
+
+    allocate = Selection[str](
+        # options=list(Size),
+        options={
+            '$ size': 'currency',
+            '% of port': 'percent_of_port',
+            '# shares': 'shares',
+        },
+        # TODO: save/load from config and/or last session
+        # value='currency'
+    )
+    weight = Selection[str](
+        options={
+            'uniform': 'uniform',
+        },
+        # value='uniform',
+    )
+    size = Edit[float](value=1000)
+    slots = Edit[int](value=4)

piker/ui/_overlay.py

@@ -0,0 +1,648 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Charting overlay helpers.
+
+'''
+from typing import Callable, Optional
+
+from pyqtgraph.Qt.QtCore import (
+    # QObject,
+    # Signal,
+    Qt,
+    # QEvent,
+)
+
+from pyqtgraph.graphicsItems.AxisItem import AxisItem
+from pyqtgraph.graphicsItems.ViewBox import ViewBox
+from pyqtgraph.graphicsItems.GraphicsWidget import GraphicsWidget
+from pyqtgraph.graphicsItems.PlotItem.PlotItem import PlotItem
+from pyqtgraph.Qt.QtCore import QObject, Signal, QEvent
+from pyqtgraph.Qt.QtWidgets import QGraphicsGridLayout, QGraphicsLinearLayout
+
+from ._interaction import ChartView
+
+__all__ = ["PlotItemOverlay"]
+
+
+# Define the layout "position" indices as to be passed
+# to a ``QtWidgets.QGraphicsGridlayout.addItem()`` call:
+# https://doc.qt.io/qt-5/qgraphicsgridlayout.html#addItem
+# This was pulled from the internals of ``PlotItem.setAxisItem()``.
+_axes_layout_indices: dict[str] = {
+    # row incremented axes
+    'top': (1, 1),
+    'bottom': (3, 1),
+
+    # view is @ (2, 1)
+
+    # column incremented axes
+    'left': (2, 0),
+    'right': (2, 2),
+}
+# NOTE: To clarify this indexing, ``PlotItem.__init__()`` makes a grid
+# with dimensions 4x3 and puts the ``ViewBox`` at postiion (2, 1) (aka
+# row=2, col=1) in the grid layout since row (0, 1) is reserved for
+# a title label and row 1 is for any potential "top" axis. Column 1
+# is the "middle" (since 3 columns) and is where the plot/vb is placed.
+
+
+class ComposedGridLayout:
+    '''
+    List-like interface to managing a sequence of overlayed
+    ``PlotItem``s in the form:
+
+    |    |     |    |    | top0     |    |    |     |    |
+    |    |     |    |    | top1     |    |    |     |    |
+    |    |     |    |    | ...      |    |    |     |    |
+    |    |     |    |    | topN     |    |    |     |    |
+    | lN | ... | l1 | l0 | ViewBox  | r0 | r1 | ... | rN |
+    |    |     |    |    | bottom0  |    |    |     |    |
+    |    |     |    |    | bottom1  |    |    |     |    |
+    |    |     |    |    | ...      |    |    |     |    |
+    |    |     |    |    | bottomN  |    |    |     |    |
+
+    Where the index ``i`` in the sequence specifies the index
+    ``<axis_name>i`` in the layout.
+
+    The ``item: PlotItem`` passed to the constructor's grid layout is
+    used verbatim as the "main plot" who's view box is give precedence
+    for input handling. The main plot's axes are removed from it's
+    layout and placed in the surrounding exterior layouts to allow for
+    re-ordering if desired.
+
+    '''
+    def __init__(
+        self,
+        item: PlotItem,
+        grid: QGraphicsGridLayout,
+        reverse: bool = False,  # insert items to the "center"
+
+    ) -> None:
+        self.items: list[PlotItem] = []
+        # self.grid = grid
+        self.reverse = reverse
+
+        # TODO: use a ``bidict`` here?
+        self._pi2axes: dict[
+            int,
+            dict[str, AxisItem],
+        ] = {}
+
+        # TODO: better name?
+        # construct surrounding layouts for placing outer axes and
+        # their legends and title labels.
+        self.sides: dict[
+            str,
+            tuple[QGraphicsLinearLayout, list[AxisItem]]
+        ] = {}
+
+        for name, pos in _axes_layout_indices.items():
+            layout = QGraphicsLinearLayout()
+            self.sides[name] = (layout, [])
+
+            layout.setContentsMargins(0, 0, 0, 0)
+            layout.setSpacing(0)
+
+            if name in ('top', 'bottom'):
+                orient = Qt.Vertical
+            elif name in ('left', 'right'):
+                orient = Qt.Horizontal
+
+            layout.setOrientation(orient)
+
+        self.insert(0, item)
+
+        # insert surrounding linear layouts into the parent pi's layout
+        # such that additional axes can be appended arbitrarily without
+        # having to expand or resize the parent's grid layout.
+        for name, (linlayout, axes) in self.sides.items():
+
+            # TODO: do we need this?
+            # axis should have been removed during insert above
+            index = _axes_layout_indices[name]
+            axis = item.layout.itemAt(*index)
+            if axis and axis.isVisible():
+                assert linlayout.itemAt(0) is axis
+
+            # item.layout.removeItem(axis)
+            item.layout.addItem(linlayout, *index)
+            layout = item.layout.itemAt(*index)
+            assert layout is linlayout
+
+    def _register_item(
+        self,
+        index: int,
+        plotitem: PlotItem,
+
+    ) -> None:
+        for name, axis_info in plotitem.axes.items():
+            axis = axis_info['item']
+            # register this plot's (maybe re-placed) axes for lookup.
+            # print(f'inserting {name}:{axis} to index {index}')
+            self._pi2axes.setdefault(name, {})[index] = axis
+
+        # enter plot into list for index tracking
+        self.items.insert(index, plotitem)
+
+    def insert(
+        self,
+        index: int,
+        plotitem: PlotItem,
+
+    ) -> (int, int):
+        '''
+        Place item at index by inserting all axes into the grid
+        at list-order appropriate position.
+
+        '''
+        if index < 0:
+            raise ValueError('`insert()` only supports an index >= 0')
+
+        # add plot's axes in sequence to the embedded linear layouts
+        # for each "side" thus avoiding graphics collisions.
+        for name, axis_info in plotitem.axes.copy().items():
+            linlayout, axes = self.sides[name]
+            axis = axis_info['item']
+
+            if axis in axes:
+                # TODO: re-order using ``.pop()`` ?
+                ValueError(f'{axis} is already in {name} layout!?')
+
+            # linking sanity
+            axis_view = axis.linkedView()
+            assert axis_view is plotitem.vb
+
+            if (
+                not axis.isVisible()
+
+                # XXX: we never skip moving the axes for the *first*
+                # plotitem inserted (even if not shown) since we need to
+                # move all the hidden axes into linear sub-layouts for
+                # that "central" plot in the overlay. Also if we don't
+                # do it there's weird geomoetry calc offsets that make
+                # view coords slightly off somehow .. smh
+                and not len(self.items) == 0
+            ):
+                continue
+
+            # XXX: Remove old axis? No, turns out we don't need this?
+            # DON'T unlink it since we the original ``ViewBox``
+            # to still drive it B)
+            # popped = plotitem.removeAxis(name, unlink=False)
+            # assert axis is popped
+
+            # invert insert index for layouts which are
+            # not-left-to-right, top-to-bottom insert oriented
+            insert_index = index
+            if name in ('top', 'left'):
+                insert_index = min(len(axes) - index, 0)
+                assert insert_index >= 0
+
+            linlayout.insertItem(insert_index, axis)
+            axes.insert(index, axis)
+
+        self._register_item(index, plotitem)
+
+        return index
+
+    def append(
+        self,
+        item: PlotItem,
+
+    ) -> (int, int):
+        '''
+        Append item's axes at indexes which put its axes "outside"
+        previously overlayed entries.
+
+        '''
+        # for left and bottom axes we have to first remove
+        # items and re-insert to maintain a list-order.
+        return self.insert(len(self.items), item)
+
+    def get_axis(
+        self,
+        plot: PlotItem,
+        name: str,
+
+    ) -> Optional[AxisItem]:
+        '''
+        Retrieve the named axis for overlayed ``plot`` or ``None``
+        if axis for that name is not shown.
+
+        '''
+        index = self.items.index(plot)
+        named = self._pi2axes[name]
+        return named.get(index)
+
+    def pop(
+        self,
+        item: PlotItem,
+
+    ) -> PlotItem:
+        '''
+        Remove item and restack all axes in list-order.
+
+        '''
+        raise NotImplementedError
+
+
+# Unimplemented features TODO:
+# - 'A' (autobtn) should relay to all views
+# - context menu single handler + relay?
+# - layout unwind and re-pack for 'left' and 'top' axes
+# - add labels to layout if detected in source ``PlotItem``
+
+# UX nice-to-have TODO:
+# - optional "focussed" view box support for view boxes
+#   that have custom input handlers (eg. you might want to
+#   scale the view to some "focussed" data view and have overlayed
+#   viewboxes only respond to relayed events.)
+# - figure out how to deal with menu raise events for multi-viewboxes.
+#   (we might want to add a different menu which specs the name of the
+#   view box currently being handled?
+# - allow selection of a particular view box by interacting with its
+#   axis?
+
+
+# TODO: we might want to enabled some kind of manual flag to disable
+# this method wrapping during type creation? As example a user could
+# definitively decide **not** to enable broadcasting support by
+# setting something like ``ViewBox.disable_relays = True``?
+def mk_relay_method(
+
+    signame: str,
+    slot: Callable[
+        [ViewBox,
+         'QEvent',
+         Optional[AxisItem]],
+        None,
+    ],
+
+) -> Callable[
+    [
+        ViewBox,
+        # lol, there isn't really a generic type thanks
+        # to the rewrite of Qt's event system XD
+        'QEvent',
+
+        'Optional[AxisItem]',
+        'Optional[ViewBox]',  # the ``relayed_from`` arg we provide
+    ],
+    None,
+]:
+
+    def maybe_broadcast(
+        vb: 'ViewBox',
+        ev: 'QEvent',
+        axis: 'Optional[int]' = None,
+        relayed_from: 'ViewBox' = None,
+
+    ) -> None:
+        '''
+        (soon to be) Decorator which makes an event handler
+        "broadcastable" to overlayed ``GraphicsWidget``s.
+
+        Adds relay signals based on the decorated handler's name
+        and conducts a signal broadcast of the relay signal if there
+        are consumers registered.
+
+        '''
+        # When no relay source has been set just bypass all
+        # the broadcast machinery.
+        if vb.event_relay_source is None:
+            ev.accept()
+            return slot(
+                vb,
+                ev,
+                axis=axis,
+            )
+
+        if relayed_from:
+            assert axis is None
+
+            # this is a relayed event and should be ignored (so it does not
+            # halt/short circuit the graphicscene loop). Further the
+            # surrounding handler for this signal must be allowed to execute
+            # and get processed by **this consumer**.
+            # print(f'{vb.name} rx relayed from {relayed_from.name}')
+            ev.ignore()
+
+            return slot(
+                vb,
+                ev,
+                axis=axis,
+            )
+
+        if axis is not None:
+            # print(f'{vb.name} handling axis event:\n{str(ev)}')
+            ev.accept()
+            return slot(
+                vb,
+                ev,
+                axis=axis,
+            )
+
+        elif (
+            relayed_from is None
+            and vb.event_relay_source is vb  # we are the broadcaster
+            and axis is None
+        ):
+            # Broadcast case: this is a source event which will be
+            # relayed to attached consumers and accepted after all
+            # consumers complete their own handling followed by this
+            # routine's processing. Sequence is,
+            # - pre-relay to all consumers *first* - ``.emit()`` blocks
+            #   until all downstream relay handlers have run.
+            # - run the source handler for **this** event and accept
+            #   the event
+
+            # Access the "bound signal" that is created
+            # on the widget type as part of instantiation.
+            signal = getattr(vb, signame)
+            # print(f'{vb.name} emitting {signame}')
+
+            # TODO/NOTE: we could also just bypass a "relay" signal
+            # entirely and instead call the handlers manually in
+            # a loop? This probably is a lot simpler and also doesn't
+            # have any downside, and allows not touching target widget
+            # internals.
+            signal.emit(
+                ev,
+                axis,
+                # passing this demarks a broadcasted/relayed event
+                vb,
+            )
+            # accept event so no more relays are fired.
+            ev.accept()
+
+            # call underlying wrapped method with an extra
+            # ``relayed_from`` value to denote that this is a relayed
+            # event handling case.
+            return slot(
+                vb,
+                ev,
+                axis=axis,
+            )
+
+    return maybe_broadcast
+
+
+# XXX: :( can't define signals **after** class compile time
+# so this is not really useful.
+# def mk_relay_signal(
+#     func,
+#     name: str = None,
+
+# ) -> Signal:
+#     (
+#         args,
+#         varargs,
+#         varkw,
+#         defaults,
+#         kwonlyargs,
+#         kwonlydefaults,
+#         annotations
+#     ) = inspect.getfullargspec(func)
+
+#     # XXX: generate a relay signal with 1 extra
+#     # argument for a ``relayed_from`` kwarg. Since
+#     # ``'self'`` is already ignored by signals we just need
+#     # to count the arguments since we're adding only 1 (and
+#     # ``args`` will capture that).
+#     numargs = len(args + list(defaults))
+#     signal = Signal(*tuple(numargs * [object]))
+#     signame = name or func.__name__ + 'Relay'
+#     return signame, signal
+
+
+def enable_relays(
+    widget: GraphicsWidget,
+    handler_names: list[str],
+
+) -> list[Signal]:
+    '''
+    Method override helper which enables relay of a particular
+    ``Signal`` from some chosen broadcaster widget to a set of
+    consumer widgets which should operate their event handlers normally
+    but instead of signals "relayed" from the broadcaster.
+
+    Mostly useful for overlaying widgets that handle user input
+    that you want to overlay graphically. The target ``widget`` type must
+    define ``QtCore.Signal``s each with a `'Relay'` suffix for each
+    name provided in ``handler_names: list[str]``.
+
+    '''
+    signals = []
+    for name in handler_names:
+        handler = getattr(widget, name)
+        signame = name + 'Relay'
+        # ensure the target widget defines a relay signal
+        relay = getattr(widget, signame)
+        widget.relays[signame] = name
+        signals.append(relay)
+        method = mk_relay_method(signame, handler)
+        setattr(widget, name, method)
+
+    return signals
+
+
+enable_relays(
+    ChartView,
+    ['wheelEvent', 'mouseDragEvent']
+)
+
+
+class PlotItemOverlay:
+    '''
+    A composite for managing overlaid ``PlotItem`` instances such that
+    you can make multiple graphics appear on the same graph with
+    separate (non-colliding) axes apply ``ViewBox`` signal broadcasting
+    such that all overlaid items respond to input simultaneously.
+
+    '''
+    def __init__(
+        self,
+        root_plotitem: PlotItem
+
+    ) -> None:
+
+        self.root_plotitem: PlotItem = root_plotitem
+
+        vb = root_plotitem.vb
+        vb.event_relay_source = vb  # TODO: maybe change name?
+        vb.setZValue(1000)  # XXX: critical for scene layering/relaying
+
+        self.overlays: list[PlotItem] = []
+        self.layout = ComposedGridLayout(
+            root_plotitem,
+            root_plotitem.layout,
+        )
+        self._relays: dict[str, Signal] = {}
+
+    def add_plotitem(
+        self,
+        plotitem: PlotItem,
+        index: Optional[int] = None,
+
+        # TODO: we could also put the ``ViewBox.XAxis``
+        # style enum here?
+        # (0,),  # link x
+        # (1,),  # link y
+        # (0, 1),  # link both
+        link_axes: tuple[int] = (),
+
+    ) -> None:
+
+        index = index or len(self.overlays)
+        root = self.root_plotitem
+        # layout: QGraphicsGridLayout = root.layout
+        self.overlays.insert(index, plotitem)
+        vb: ViewBox = plotitem.vb
+
+        # mark this consumer overlay as ready to expect relayed events
+        # from the root plotitem.
+        vb.event_relay_source = root.vb
+
+        # TODO: some sane way to allow menu event broadcast XD
+        # vb.setMenuEnabled(False)
+
+        # TODO: inside the `maybe_broadcast()` (soon to be) decorator
+        # we need have checks that consumers have been attached to
+        # these relay signals.
+        if link_axes != (0, 1):
+
+            # wire up relay signals
+            for relay_signal_name, handler_name in vb.relays.items():
+                # print(handler_name)
+                # XXX: Signal class attrs are bound after instantiation
+                # of the defining type, so we need to access that bound
+                # version here.
+                signal = getattr(root.vb, relay_signal_name)
+                handler = getattr(vb, handler_name)
+                signal.connect(handler)
+
+        # link dim-axes to root if requested by user.
+        # TODO: solve more-then-wanted scaled panning on click drag
+        # which seems to be due to broadcast. So we probably need to
+        # disable broadcast when axes are linked in a particular
+        # dimension?
+        for dim in link_axes:
+            # link x and y axes to new view box such that the top level
+            # viewbox propagates to the root (and whatever other
+            # plotitem overlays that have been added).
+            vb.linkView(dim, root.vb)
+
+        # make overlaid viewbox impossible to focus since the top
+        # level should handle all input and relay to overlays.
+        # NOTE: this was solved with the `setZValue()` above!
+
+        # TODO: we will probably want to add a "focus" api such that
+        # a new "top level" ``PlotItem`` can be selected dynamically
+        # (and presumably the axes dynamically sorted to match).
+        vb.setFlag(
+            vb.GraphicsItemFlag.ItemIsFocusable,
+            False
+        )
+        vb.setFocusPolicy(Qt.NoFocus)
+
+        # append-compose into the layout all axes from this plot
+        self.layout.insert(index, plotitem)
+
+        plotitem.setGeometry(root.vb.sceneBoundingRect())
+
+        def size_to_viewbox(vb: 'ViewBox'):
+            plotitem.setGeometry(vb.sceneBoundingRect())
+
+        root.vb.sigResized.connect(size_to_viewbox)
+
+        # ensure the overlayed view is redrawn on each cycle
+        root.scene().sigPrepareForPaint.connect(vb.prepareForPaint)
+
+        # focus state sanity
+        vb.clearFocus()
+        assert not vb.focusWidget()
+        root.vb.setFocus()
+        assert root.vb.focusWidget()
+
+    # XXX: do we need this? Why would you build then destroy?
+    def remove_plotitem(self, plotItem: PlotItem) -> None:
+        '''
+        Remove this ``PlotItem`` from the overlayed set making not shown
+        and unable to accept input.
+
+        '''
+        ...
+
+    # TODO: i think this would be super hot B)
+    def focus_item(self, plotitem: PlotItem) -> PlotItem:
+        '''
+        Apply focus to a contained PlotItem thus making it the "top level"
+        item in the overlay able to accept peripheral's input from the user
+        and responsible for zoom and panning control via its ``ViewBox``.
+
+        '''
+        ...
+
+    def get_axis(
+        self,
+        plot: PlotItem,
+        name: str,
+
+    ) -> AxisItem:
+        '''
+        Retrieve the named axis for overlayed ``plot``.
+
+        '''
+        return self.layout.get_axis(plot, name)
+
+    def get_axes(
+        self,
+        name: str,
+
+    ) -> list[AxisItem]:
+        '''
+        Retrieve all axes for all plots with ``name: str``.
+
+        If a particular overlay doesn't have a displayed named axis
+        then it is not delivered in the returned ``list``.
+
+        '''
+        axes = []
+        for plot in self.overlays:
+            axis = self.layout.get_axis(plot, name)
+            if axis:
+                axes.append(axis)
+
+        return axes
+
+    # TODO: i guess we need this if you want to detach existing plots
+    # dynamically? XXX: untested as of now.
+    def _disconnect_all(
+        self,
+        plotitem: PlotItem,
+    ) -> list[Signal]:
+        '''
+        Disconnects all signals related to this widget for the given chart.
+
+        '''
+        disconnected = []
+        for pi, sig in self._relays.items():
+            QObject.disconnect(sig)
+            disconnected.append(sig)
+
+        return disconnected

piker/ui/_pathops.py

@@ -0,0 +1,236 @@
+# piker: trading gear for hackers
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Super fast ``QPainterPath`` generation related operator routines.
+
+"""
+from __future__ import annotations
+from typing import (
+    # Optional,
+    TYPE_CHECKING,
+)
+
+import numpy as np
+from numpy.lib import recfunctions as rfn
+from numba import njit, float64, int64  # , optional
+# import pyqtgraph as pg
+from PyQt5 import QtGui
+# from PyQt5.QtCore import QLineF, QPointF
+
+from ..data._sharedmem import (
+    ShmArray,
+)
+# from .._profile import pg_profile_enabled, ms_slower_then
+from ._compression import (
+    ds_m4,
+)
+
+if TYPE_CHECKING:
+    from ._flows import Renderer
+
+
+def xy_downsample(
+    x,
+    y,
+    uppx,
+
+    x_spacer: float = 0.5,
+
+) -> tuple[np.ndarray, np.ndarray]:
+
+    # downsample whenever more then 1 pixels per datum can be shown.
+    # always refresh data bounds until we get diffing
+    # working properly, see above..
+    bins, x, y = ds_m4(
+        x,
+        y,
+        uppx,
+    )
+
+    # flatten output to 1d arrays suitable for path-graphics generation.
+    x = np.broadcast_to(x[:, None], y.shape)
+    x = (x + np.array(
+        [-x_spacer, 0, 0, x_spacer]
+    )).flatten()
+    y = y.flatten()
+
+    return x, y
+
+
+@njit(
+    # TODO: for now need to construct this manually for readonly arrays, see
+    # https://github.com/numba/numba/issues/4511
+    # ntypes.tuple((float64[:], float64[:], float64[:]))(
+    #     numba_ohlc_dtype[::1],  # contiguous
+    #     int64,
+    #     optional(float64),
+    # ),
+    nogil=True
+)
+def path_arrays_from_ohlc(
+    data: np.ndarray,
+    start: int64,
+    bar_gap: float64 = 0.43,
+
+) -> np.ndarray:
+    '''
+    Generate an array of lines objects from input ohlc data.
+
+    '''
+    size = int(data.shape[0] * 6)
+
+    x = np.zeros(
+        # data,
+        shape=size,
+        dtype=float64,
+    )
+    y, c = x.copy(), x.copy()
+
+    # TODO: report bug for assert @
+    # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
+    for i, q in enumerate(data[start:], start):
+
+        # TODO: ask numba why this doesn't work..
+        # open, high, low, close, index = q[
+        #     ['open', 'high', 'low', 'close', 'index']]
+
+        open = q['open']
+        high = q['high']
+        low = q['low']
+        close = q['close']
+        index = float64(q['index'])
+
+        istart = i * 6
+        istop = istart + 6
+
+        # x,y detail the 6 points which connect all vertexes of a ohlc bar
+        x[istart:istop] = (
+            index - bar_gap,
+            index,
+            index,
+            index,
+            index,
+            index + bar_gap,
+        )
+        y[istart:istop] = (
+            open,
+            open,
+            low,
+            high,
+            close,
+            close,
+        )
+
+        # specifies that the first edge is never connected to the
+        # prior bars last edge thus providing a small "gap"/"space"
+        # between bars determined by ``bar_gap``.
+        c[istart:istop] = (1, 1, 1, 1, 1, 0)
+
+    return x, y, c
+
+
+def gen_ohlc_qpath(
+    r: Renderer,
+    data: np.ndarray,
+    array_key: str,  # we ignore this
+    vr: tuple[int, int],
+
+    start: int = 0,  # XXX: do we need this?
+    # 0.5 is no overlap between arms, 1.0 is full overlap
+    w: float = 0.43,
+
+) -> QtGui.QPainterPath:
+    '''
+    More or less direct proxy to ``path_arrays_from_ohlc()``
+    but with closed in kwargs for line spacing.
+
+    '''
+    x, y, c = path_arrays_from_ohlc(
+        data,
+        start,
+        bar_gap=w,
+    )
+    return x, y, c
+
+
+def ohlc_to_line(
+    ohlc_shm: ShmArray,
+    data_field: str,
+    fields: list[str] = ['open', 'high', 'low', 'close']
+
+) -> tuple[
+    np.ndarray,
+    np.ndarray,
+]:
+    '''
+    Convert an input struct-array holding OHLC samples into a pair of
+    flattened x, y arrays with the same size (datums wise) as the source
+    data.
+
+    '''
+    y_out = ohlc_shm.ustruct(fields)
+    first = ohlc_shm._first.value
+    last = ohlc_shm._last.value
+
+    # write pushed data to flattened copy
+    y_out[first:last] = rfn.structured_to_unstructured(
+        ohlc_shm.array[fields]
+    )
+
+    # generate an flat-interpolated x-domain
+    x_out = (
+        np.broadcast_to(
+            ohlc_shm._array['index'][:, None],
+            (
+                ohlc_shm._array.size,
+                # 4,  # only ohlc
+                y_out.shape[1],
+            ),
+        ) + np.array([-0.5, 0, 0, 0.5])
+    )
+    assert y_out.any()
+
+    return (
+        x_out,
+        y_out,
+    )
+
+
+def to_step_format(
+    shm: ShmArray,
+    data_field: str,
+    index_field: str = 'index',
+
+) -> tuple[int, np.ndarray, np.ndarray]:
+    '''
+    Convert an input 1d shm array to a "step array" format
+    for use by path graphics generation.
+
+    '''
+    i = shm._array['index'].copy()
+    out = shm._array[data_field].copy()
+
+    x_out = np.broadcast_to(
+        i[:, None],
+        (i.size, 2),
+    ) + np.array([-0.5, 0.5])
+
+    y_out = np.empty((len(out), 2), dtype=out.dtype)
+    y_out[:] = out[:, np.newaxis]
+
+    # start y at origin level
+    y_out[0, 0] = 0
+    return x_out, y_out

piker/ui/_position.py

@@ -0,0 +1,696 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+Position info and display
+
+"""
+from __future__ import annotations
+from dataclasses import dataclass
+from functools import partial
+from math import floor, copysign
+from typing import Optional
+
+
+# from PyQt5.QtWidgets import QStyle
+# from PyQt5.QtGui import (
+#     QIcon, QPixmap, QColor
+# )
+from pyqtgraph import functions as fn
+
+from ._annotate import LevelMarker
+from ._anchors import (
+    pp_tight_and_right,  # wanna keep it straight in the long run
+    gpath_pin,
+)
+from ..calc import humanize, pnl, puterize
+from ..clearing._allocate import Allocator, Position
+from ..data._normalize import iterticks
+from ..data.feed import Feed
+from ._label import Label
+from ._lines import LevelLine, order_line
+from ._style import _font
+from ._forms import FieldsForm, FillStatusBar, QLabel
+from ..log import get_logger
+
+log = get_logger(__name__)
+_pnl_tasks: dict[str, bool] = {}
+
+
+async def update_pnl_from_feed(
+
+    feed: Feed,
+    order_mode: OrderMode,  # noqa
+    tracker: PositionTracker,
+
+) -> None:
+    '''Real-time display the current pp's PnL in the appropriate label.
+
+    ``ValueError`` if this task is spawned where there is a net-zero pp.
+
+    '''
+    global _pnl_tasks
+
+    pp = order_mode.current_pp
+    live = pp.live_pp
+    key = live.symbol.key
+
+    log.info(f'Starting pnl display for {pp.alloc.account}')
+
+    if live.size < 0:
+        types = ('ask', 'last', 'last', 'dark_trade')
+
+    elif live.size > 0:
+        types = ('bid', 'last', 'last', 'dark_trade')
+
+    else:
+        log.info(f'No position (yet) for {tracker.alloc.account}@{key}')
+        return
+
+    # real-time update pnl on the status pane
+    try:
+        async with feed.stream.subscribe() as bstream:
+            # last_tick = time.time()
+            async for quotes in bstream:
+
+                # now = time.time()
+                # period = now - last_tick
+
+                for sym, quote in quotes.items():
+
+                    for tick in iterticks(quote, types):
+                        # print(f'{1/period} Hz')
+
+                        size = order_mode.current_pp.live_pp.size
+                        if size == 0:
+                            # terminate this update task since we're
+                            # no longer in a pp
+                            order_mode.pane.pnl_label.format(pnl=0)
+                            return
+
+                        else:
+                            # compute and display pnl status
+                            order_mode.pane.pnl_label.format(
+                                pnl=copysign(1, size) * pnl(
+                                    # live.avg_price,
+                                    order_mode.current_pp.live_pp.avg_price,
+                                    tick['price'],
+                                ),
+                            )
+
+                        # last_tick = time.time()
+    finally:
+        assert _pnl_tasks[key]
+        assert _pnl_tasks.pop(key)
+
+
+@dataclass
+class SettingsPane:
+    '''
+    Composite set of widgets plus an allocator model for configuring
+    order entry sizes and position limits per tradable instrument.
+
+    '''
+    # input fields
+    form: FieldsForm
+
+    # output fill status and labels
+    fill_bar: FillStatusBar
+
+    step_label: QLabel
+    pnl_label: QLabel
+    limit_label: QLabel
+
+    # encompasing high level namespace
+    order_mode: Optional['OrderMode'] = None  # typing: ignore # noqa
+
+    def set_accounts(
+        self,
+        names: list[str],
+        sizes: Optional[list[float]] = None,
+    ) -> None:
+
+        combo = self.form.fields['account']
+        return combo.set_items(names)
+
+    def on_selection_change(
+        self,
+        text: str,
+        key: str,
+
+    ) -> None:
+        '''
+        Called on any order pane drop down selection change.
+
+        '''
+        log.info(f'selection input {key}:{text}')
+        self.on_ui_settings_change(key, text)
+
+    def on_ui_settings_change(
+        self,
+
+        key: str,
+        value: str,
+
+    ) -> bool:
+        '''
+        Called on any order pane edit field value change.
+
+        '''
+        mode = self.order_mode
+
+        # an account switch request
+        if key == 'account':
+
+            # hide details on the old selection
+            old_tracker = mode.current_pp
+            old_tracker.hide_info()
+
+            # re-assign the order mode tracker
+            account_name = value
+            tracker = mode.trackers.get(account_name)
+
+            # if selection can't be found (likely never discovered with
+            # a ``brokerd`) then error and switch back to the last
+            # selection.
+            if tracker is None:
+                sym = old_tracker.chart.linked.symbol.key
+                log.error(
+                    f'Account `{account_name}` can not be set for {sym}'
+                )
+                self.form.fields['account'].setCurrentText(
+                    old_tracker.alloc.account)
+                return
+
+            self.order_mode.current_pp = tracker
+            assert tracker.alloc.account == account_name
+            self.form.fields['account'].setCurrentText(account_name)
+            tracker.show()
+            tracker.hide_info()
+
+            self.display_pnl(tracker)
+
+            # load the new account's allocator
+            alloc = tracker.alloc
+
+        else:
+            tracker = mode.current_pp
+            alloc = tracker.alloc
+
+        size_unit = alloc.size_unit
+
+        # WRITE any settings to current pp's allocator
+        try:
+            if key == 'size_unit':
+                # implicit re-write of value if input
+                # is the "text name" of the units.
+                # yah yah, i know this is badd..
+                alloc.size_unit = value
+            else:
+                value = puterize(value)
+                if key == 'limit':
+                    pp = mode.current_pp.live_pp
+
+                    if size_unit == 'currency':
+                        dsize = pp.dsize
+                        if dsize > value:
+                            log.error(
+                                f'limit must > then current pp: {dsize}'
+                            )
+                            raise ValueError
+
+                        alloc.currency_limit = value
+
+                    else:
+                        size = pp.size
+                        if size > value:
+                            log.error(
+                                f'limit must > then current pp: {size}'
+                            )
+                            raise ValueError
+
+                        alloc.units_limit = value
+
+                elif key == 'slots':
+                    if value <= 0:
+                        raise ValueError('slots must be > 0')
+                    alloc.slots = int(value)
+
+                else:
+                    log.error(f'Unknown setting {key}')
+                    raise ValueError
+
+            log.info(f'settings change: {key}: {value}')
+
+        except ValueError:
+            log.error(f'Invalid value for `{key}`: {value}')
+
+        # READ out settings and update the status UI / settings widgets
+        suffix = {'currency': ' $', 'units': ' u'}[size_unit]
+        limit = alloc.limit()
+
+        # TODO: a reverse look up from the position to the equivalent
+        # account(s), if none then look to user config for default?
+        self.update_status_ui(pp=tracker)
+
+        step_size, currency_per_slot = alloc.step_sizes()
+
+        if size_unit == 'currency':
+            step_size = currency_per_slot
+
+        self.step_label.format(
+            step_size=str(humanize(step_size)) + suffix
+        )
+        self.limit_label.format(
+            limit=str(humanize(limit)) + suffix
+        )
+
+        # update size unit in UI
+        self.form.fields['size_unit'].setCurrentText(
+            alloc._size_units[alloc.size_unit]
+        )
+        self.form.fields['slots'].setText(str(alloc.slots))
+        self.form.fields['limit'].setText(str(limit))
+
+        # update of level marker size label based on any new settings
+        tracker.update_from_pp()
+
+        # TODO: maybe return a diff of settings so if we can an error we
+        # can have general input handling code to report it through the
+        # UI in some way?
+        return True
+
+    def update_status_ui(
+        self,
+
+        pp: PositionTracker,
+
+    ) -> None:
+
+        alloc = pp.alloc
+        slots = alloc.slots
+        used = alloc.slots_used(pp.live_pp)
+
+        # calculate proportion of position size limit
+        # that exists and display in fill bar
+        # TODO: what should we do for fractional slot pps?
+        self.fill_bar.set_slots(
+            slots,
+
+            # TODO: how to show "partial" slots?
+            # min(round(prop * slots), slots)
+            min(used, slots)
+        )
+        self.update_account_icons({alloc.account: pp.live_pp})
+
+    def update_account_icons(
+        self,
+        pps: dict[str, Position],
+
+    ) -> None:
+
+        form = self.form
+        accounts = form.fields['account']
+
+        for account_name, pp in pps.items():
+            icon_name = None
+
+            if pp.size > 0:
+                icon_name = 'long_pp'
+            elif pp.size < 0:
+                icon_name = 'short_pp'
+
+            accounts.set_icon(account_name, icon_name)
+
+    def display_pnl(
+        self,
+        tracker: PositionTracker,
+
+    ) -> None:
+        '''Display the PnL for the current symbol and personal positioning (pp).
+
+        If a position is open start a background task which will
+        real-time update the pnl label in the settings pane.
+
+        '''
+        mode = self.order_mode
+        sym = mode.chart.linked.symbol
+        size = tracker.live_pp.size
+        feed = mode.quote_feed
+        pnl_value = 0
+
+        if size:
+            # last historical close price
+            last = feed.shm.array[-1][['close']][0]
+            pnl_value = copysign(1, size) * pnl(
+                tracker.live_pp.avg_price,
+                last,
+            )
+
+            # maybe start update task
+            global _pnl_tasks
+            if sym.key not in _pnl_tasks:
+                _pnl_tasks[sym.key] = True
+                self.order_mode.nursery.start_soon(
+                    update_pnl_from_feed,
+                    feed,
+                    mode,
+                    tracker,
+                )
+
+        # immediately display in status label
+        self.pnl_label.format(pnl=pnl_value)
+
+
+def position_line(
+
+    chart: 'ChartPlotWidget',  # noqa
+    size: float,
+    level: float,
+    color: str,
+
+    orient_v: str = 'bottom',
+    marker: Optional[LevelMarker] = None,
+
+) -> LevelLine:
+    '''
+    Convenience routine to create a line graphic representing a "pp"
+    aka the acro for a,
+    "{piker, private, personal, puny, <place your p-word here>} position".
+
+    If ``marker`` is provided it will be configured appropriately for
+    the "direction" of the position.
+
+    '''
+    line = order_line(
+        chart,
+        level,
+
+        # TODO: could we maybe add a ``action=None`` which
+        # would be a mechanism to check a marker was passed in?
+
+        color=color,
+        highlight_on_hover=False,
+        movable=False,
+        hide_xhair_on_hover=False,
+        only_show_markers_on_hover=False,
+        always_show_labels=False,
+
+        # explicitly disable ``order_line()`` factory's creation
+        # of a level marker since we do it in this tracer thing.
+        show_markers=False,
+    )
+
+    if marker:
+        # configure marker to position data
+
+        if size > 0:  # long
+            style = '|<'  # point "up to" the line
+        elif size < 0:  # short
+            style = '>|'  # point "down to" the line
+
+        marker.style = style
+
+        # set marker color to same as line
+        marker.setPen(line.currentPen)
+        marker.setBrush(fn.mkBrush(line.currentPen.color()))
+        marker.level = level
+        marker.update()
+        marker.show()
+
+        # show position marker on view "edge" when out of view
+        vb = line.getViewBox()
+        vb.sigRangeChanged.connect(marker.position_in_view)
+
+    line.set_level(level)
+
+    return line
+
+
+class PositionTracker:
+    '''
+    Track and display real-time positions for a single symbol
+    over multiple accounts on a single chart.
+
+    Graphically composed of a level line and marker as well as labels
+    for indcating current position information. Updates are made to the
+    corresponding "settings pane" for the chart's "order mode" UX.
+
+    '''
+    # inputs
+    chart: 'ChartPlotWidget'  # noqa
+
+    alloc: Allocator
+    startup_pp: Position
+    live_pp: Position
+
+    # allocated
+    pp_label: Label
+    size_label: Label
+    line: Optional[LevelLine] = None
+
+    _color: str = 'default_lightest'
+
+    def __init__(
+        self,
+        chart: 'ChartPlotWidget',  # noqa
+        alloc: Allocator,
+        startup_pp: Position,
+
+    ) -> None:
+
+        self.chart = chart
+
+        self.alloc = alloc
+        self.startup_pp = startup_pp
+        self.live_pp = startup_pp.copy()
+
+        view = chart.getViewBox()
+
+        # literally the 'pp' (pee pee) label that's always in view
+        self.pp_label = pp_label = Label(
+            view=view,
+            fmt_str='pp',
+            color=self._color,
+            update_on_range_change=False,
+        )
+
+        # create placeholder 'up' level arrow
+        self._level_marker = None
+        self._level_marker = self.level_marker(size=1)
+
+        pp_label.scene_anchor = partial(
+            gpath_pin,
+            gpath=self._level_marker,
+            label=pp_label,
+        )
+        pp_label.render()
+
+        self.size_label = size_label = Label(
+            view=view,
+            color=self._color,
+
+            # this is "static" label
+            # update_on_range_change=False,
+            fmt_str='\n'.join((
+                ':{slots_used:.1f}x',
+            )),
+
+            fields={
+                'slots_used': 0,
+            },
+        )
+        size_label.render()
+
+        size_label.scene_anchor = partial(
+            pp_tight_and_right,
+            label=self.pp_label,
+        )
+
+    @property
+    def pane(self) -> FieldsForm:
+        '''
+        Return handle to pp side pane form.
+
+        '''
+        return self.chart.linked.godwidget.pp_pane
+
+    def update_graphics(
+        self,
+        marker: LevelMarker
+
+    ) -> None:
+        '''
+        Update all labels.
+
+        Meant to be called from the maker ``.paint()``
+        for immediate, lag free label draws.
+
+        '''
+        self.pp_label.update()
+        self.size_label.update()
+
+    def update_from_pp(
+        self,
+        position: Optional[Position] = None,
+
+    ) -> None:
+        '''Update graphics and data from average price and size passed in our
+        EMS ``BrokerdPosition`` msg.
+
+        '''
+        # live pp updates
+        pp = position or self.live_pp
+
+        self.update_line(
+            pp.avg_price,
+            pp.size,
+            self.chart.linked.symbol.lot_size_digits,
+        )
+
+        # label updates
+        self.size_label.fields['slots_used'] = round(
+            self.alloc.slots_used(pp), ndigits=1)
+        self.size_label.render()
+
+        if pp.size == 0:
+            self.hide()
+
+        else:
+            self._level_marker.level = pp.avg_price
+
+            # these updates are critical to avoid lag on view/scene changes
+            self._level_marker.update()  # trigger paint
+            self.pp_label.update()
+            self.size_label.update()
+
+            self.show()
+
+            # don't show side and status widgets unless
+            # order mode is "engaged" (which done via input controls)
+            self.hide_info()
+
+    def level(self) -> float:
+        if self.line:
+            return self.line.value()
+        else:
+            return 0
+
+    def show(self) -> None:
+        if self.live_pp.size:
+            self.line.show()
+            self.line.show_labels()
+
+            self._level_marker.show()
+            self.pp_label.show()
+            self.size_label.show()
+
+    def hide(self) -> None:
+        self.pp_label.hide()
+        self._level_marker.hide()
+        self.size_label.hide()
+        if self.line:
+            self.line.hide()
+
+    def hide_info(self) -> None:
+        '''Hide details (right now just size label?) of position.
+
+        '''
+        self.size_label.hide()
+        if self.line:
+            self.line.hide_labels()
+
+    # TODO: move into annoate module
+    def level_marker(
+        self,
+        size: float,
+
+    ) -> LevelMarker:
+
+        if self._level_marker:
+            self._level_marker.delete()
+
+        # arrow marker
+        # scale marker size with dpi-aware font size
+        font_size = _font.font.pixelSize()
+
+        # scale marker size with dpi-aware font size
+        arrow_size = floor(1.375 * font_size)
+
+        if size > 0:
+            style = '|<'
+
+        elif size < 0:
+            style = '>|'
+
+        arrow = LevelMarker(
+            chart=self.chart,
+            style=style,
+            get_level=self.level,
+            size=arrow_size,
+            on_paint=self.update_graphics,
+        )
+
+        self.chart.getViewBox().scene().addItem(arrow)
+        arrow.show()
+
+        return arrow
+
+    def update_line(
+        self,
+        price: float,
+        size: float,
+        size_digits: int,
+
+    ) -> None:
+        '''Update personal position level line.
+
+        '''
+        # do line update
+        line = self.line
+
+        if size:
+            if line is None:
+
+                # create and show a pp line
+                line = self.line = position_line(
+                    chart=self.chart,
+                    level=price,
+                    size=size,
+                    color=self._color,
+                    marker=self._level_marker,
+                )
+
+            else:
+
+                line.set_level(price)
+                self._level_marker.level = price
+                self._level_marker.update()
+
+            # update LHS sizing label
+            line.update_labels({
+                'size': size,
+                'size_digits': size_digits,
+                'fiat_size': round(price * size, ndigits=2),
+
+                # TODO: per account lines on a single (or very related) symbol
+                'account': self.alloc.account,
+            })
+            line.show()
+
+        elif line:  # remove pp line from view if it exists on a net-zero pp
+            line.delete()
+            self.line = None

piker/ui/_search.py

@@ -35,9 +35,9 @@ from collections import defaultdict
 from contextlib import asynccontextmanager
 from functools import partial
 from typing import (
-    List, Optional, Callable,
-    Awaitable, Sequence, Dict,
-    Any, AsyncIterator, Tuple,
+    Optional, Callable,
+    Awaitable, Sequence,
+    Any, AsyncIterator
 )
 import time
 # from pprint import pformat
@@ -45,11 +45,10 @@ import time
 from fuzzywuzzy import process as fuzzy
 import trio
 from trio_typing import TaskStatus
-from PyQt5 import QtCore, QtGui
+from PyQt5 import QtCore
 from PyQt5 import QtWidgets
 from PyQt5.QtCore import (
     Qt,
-    # QSize,
     QModelIndex,
     QItemSelectionModel,
 )
@@ -63,40 +62,24 @@ from PyQt5.QtWidgets import (
     QTreeView,
     # QListWidgetItem,
     # QAbstractScrollArea,
-    QStyledItemDelegate,
+    # QStyledItemDelegate,
 )
 
 
 from ..log import get_logger
 from ._style import (
     _font,
-    DpiAwareFont,
-    # hcolor,
+    hcolor,
 )
+from ._forms import Edit, FontScaledDelegate
 
 
 log = get_logger(__name__)
 
 
-class SimpleDelegate(QStyledItemDelegate):
-    """
-    Super simple view delegate to render text in the same
-    font size as the search widget.
-
-    """
-
-    def __init__(
-        self,
-        parent=None,
-        font: DpiAwareFont = _font,
-    ) -> None:
-        super().__init__(parent)
-        self.dpi_font = font
-
-
 class CompleterView(QTreeView):
 
-    mode_name: str = 'mode: search-nav'
+    mode_name: str = 'search-nav'
 
     # XXX: relevant docs links:
     # - simple widget version of this:
@@ -121,7 +104,7 @@ class CompleterView(QTreeView):
     def __init__(
         self,
         parent=None,
-        labels: List[str] = [],
+        labels: list[str] = [],
     ) -> None:
 
         super().__init__(parent)
@@ -130,13 +113,11 @@ class CompleterView(QTreeView):
         self.labels = labels
 
         # a std "tabular" config
-        self.setItemDelegate(SimpleDelegate())
+        self.setItemDelegate(FontScaledDelegate(self))
         self.setModel(model)
         self.setAlternatingRowColors(True)
         # TODO: size this based on DPI font
-        self.setIndentation(20)
-
-        self.pressed.connect(self.on_pressed)
+        self.setIndentation(_font.px_size)
 
         # self.setUniformRowHeights(True)
         # self.setColumnWidth(0, 3)
@@ -144,6 +125,10 @@ class CompleterView(QTreeView):
         # self.setSizeAdjustPolicy(QAbstractScrollArea.AdjustIgnored)
 
         # ux settings
+        self.setSizePolicy(
+            QtWidgets.QSizePolicy.Expanding,
+            QtWidgets.QSizePolicy.Expanding,
+        )
         self.setItemsExpandable(True)
         self.setExpandsOnDoubleClick(False)
         self.setAnimated(False)
@@ -154,12 +139,12 @@ class CompleterView(QTreeView):
 
         self._font_size: int = 0  # pixels
 
-    def on_pressed(self, idx: QModelIndex) -> None:
+    async def on_pressed(self, idx: QModelIndex) -> None:
         '''Mouse pressed on view handler.
 
         '''
         search = self.parent()
-        search.chart_current_item(clear_to_cache=False)
+        await search.chart_current_item(clear_to_cache=False)
         search.focus()
 
     def set_font_size(self, size: int = 18):
@@ -171,23 +156,58 @@ class CompleterView(QTreeView):
 
         self.setStyleSheet(f"font: {size}px")
 
-    def resize(self):
+    # def resizeEvent(self, event: 'QEvent') -> None:
+    #     event.accept()
+    #     super().resizeEvent(event)
+
+    def on_resize(self) -> None:
+        '''
+        Resize relay event from god.
+
+        '''
+        self.resize_to_results()
+
+    def resize_to_results(self):
         model = self.model()
         cols = model.columnCount()
+        # rows = model.rowCount()
 
+        col_w_tot = 0
         for i in range(cols):
             self.resizeColumnToContents(i)
+            col_w_tot += self.columnWidth(i)
 
-        # inclusive of search bar and header "rows" in pixel terms
-        rows = 100
-        # max_rows = 8  # 6 + search and headers
-        row_px = self.rowHeight(self.currentIndex())
-        # print(f'font_h: {font_h}\n px_height: {px_height}')
+        win = self.window()
+        win_h = win.height()
+        edit_h = self.parent().bar.height()
+        sb_h = win.statusBar().height()
 
         # TODO: probably make this more general / less hacky
-        self.setMinimumSize(self.width(), rows * row_px)
-        self.setMaximumSize(self.width() + 10, rows * row_px)
-        self.setFixedWidth(333)
+        # we should figure out the exact number of rows to allow
+        # inclusive of search bar and header "rows", in pixel terms.
+        # Eventually when we have an "info" widget below the results we
+        # will want space for it and likely terminating the results-view
+        # space **exactly on a row** would be ideal.
+        # if row_px > 0:
+        #     rows = ceil(window_h / row_px) - 4
+        # else:
+        #     rows = 16
+        # self.setFixedHeight(rows * row_px)
+        # self.resize(self.width(), rows * row_px)
+
+        # NOTE: if the heigh set here is **too large** then the resize
+        # event will perpetually trigger as the window causes some kind
+        # of recompute of callbacks.. so we have to ensure it's limited.
+        h = win_h - (edit_h + 1.666*sb_h)
+        assert h > 0
+        self.setFixedHeight(round(h))
+
+        # size to width of longest result seen thus far
+        # TODO: should we always dynamically scale to longest result?
+        if self.width() < col_w_tot:
+            self.setFixedWidth(col_w_tot)
+
+        self.update()
 
     def is_selecting_d1(self) -> bool:
         cidx = self.selectionModel().currentIndex()
@@ -236,7 +256,8 @@ class CompleterView(QTreeView):
         idx: QModelIndex,
 
     ) -> QStandardItem:
-        '''Select and return the item at index ``idx``.
+        '''
+        Select and return the item at index ``idx``.
 
         '''
         sel = self.selectionModel()
@@ -251,7 +272,8 @@ class CompleterView(QTreeView):
         return model.itemFromIndex(idx)
 
     def select_first(self) -> QStandardItem:
-        '''Select the first depth >= 2 entry from the completer tree and
+        '''
+        Select the first depth >= 2 entry from the completer tree and
         return it's item.
 
         '''
@@ -314,7 +336,8 @@ class CompleterView(QTreeView):
         section: str,
 
     ) -> Optional[QModelIndex]:
-        '''Find the *first* depth = 1 section matching ``section`` in
+        '''
+        Find the *first* depth = 1 section matching ``section`` in
         the tree and return its index.
 
         '''
@@ -352,7 +375,7 @@ class CompleterView(QTreeView):
             else:
                 model.setItem(idx.row(), 1, QStandardItem())
 
-            self.resize()
+            self.resize_to_results()
 
             return idx
         else:
@@ -365,7 +388,8 @@ class CompleterView(QTreeView):
         clear_all: bool = False,
 
     ) -> None:
-        '''Set result-rows for depth = 1 tree section ``section``.
+        '''
+        Set result-rows for depth = 1 tree section ``section``.
 
         '''
         model = self.model()
@@ -422,62 +446,32 @@ class CompleterView(QTreeView):
 
     def show_matches(self) -> None:
         self.show()
-        self.resize()
+        self.resize_to_results()
 
 
-class SearchBar(QtWidgets.QLineEdit):
+class SearchBar(Edit):
 
-    mode_name: str = 'mode: search'
+    mode_name: str = 'search'
 
     def __init__(
 
         self,
         parent: QWidget,
-        parent_chart: QWidget,  # noqa
+        godwidget: QWidget,
         view: Optional[CompleterView] = None,
-        font: DpiAwareFont = _font,
+        **kwargs,
 
     ) -> None:
 
-        super().__init__(parent)
-
-        # self.setContextMenuPolicy(Qt.CustomContextMenu)
-        # self.customContextMenuRequested.connect(self.show_menu)
-        # self.setStyleSheet(f"font: 18px")
-
+        self.godwidget = godwidget
+        super().__init__(parent, **kwargs)
         self.view: CompleterView = view
-        self.dpi_font = font
-        self.godwidget = parent_chart
-
-        # size it as we specify
-        # https://doc.qt.io/qt-5/qsizepolicy.html#Policy-enum
-        self.setSizePolicy(
-            QtWidgets.QSizePolicy.Expanding,
-            QtWidgets.QSizePolicy.Fixed,
-        )
-        self.setFont(font.font)
-
-        # witty bit of margin
-        self.setTextMargins(2, 2, 2, 2)
-
-    def focus(self) -> None:
-        self.selectAll()
-        self.show()
-        self.setFocus()
+        godwidget._widgets[view.mode_name] = view
 
     def show(self) -> None:
         super().show()
         self.view.show_matches()
 
-    def sizeHint(self) -> QtCore.QSize:
-        """
-        Scale edit box to size of dpi aware font.
-
-        """
-        psh = super().sizeHint()
-        psh.setHeight(self.dpi_font.px_size + 2)
-        return psh
-
     def unfocus(self) -> None:
         self.parent().hide()
         self.clearFocus()
@@ -487,17 +481,18 @@ class SearchBar(QtWidgets.QLineEdit):
 
 
 class SearchWidget(QtWidgets.QWidget):
-    '''Composed widget of ``SearchBar`` + ``CompleterView``.
+    '''
+    Composed widget of ``SearchBar`` + ``CompleterView``.
 
     Includes helper methods for item management in the sub-widgets.
 
     '''
-    mode_name: str = 'mode: search'
+    mode_name: str = 'search'
 
     def __init__(
         self,
         godwidget: 'GodWidget',  # type: ignore # noqa
-        columns: List[str] = ['src', 'symbol'],
+        columns: list[str] = ['src', 'symbol'],
         parent=None,
 
     ) -> None:
@@ -506,13 +501,13 @@ class SearchWidget(QtWidgets.QWidget):
         # size it as we specify
         self.setSizePolicy(
             QtWidgets.QSizePolicy.Fixed,
-            QtWidgets.QSizePolicy.Fixed,
+            QtWidgets.QSizePolicy.Expanding,
         )
 
         self.godwidget = godwidget
 
         self.vbox = QtWidgets.QVBoxLayout(self)
-        self.vbox.setContentsMargins(0, 0, 0, 0)
+        self.vbox.setContentsMargins(0, 4, 4, 0)
         self.vbox.setSpacing(4)
 
         # split layout for the (label:| search bar entry)
@@ -522,10 +517,17 @@ class SearchWidget(QtWidgets.QWidget):
 
         # add label to left of search bar
         self.label = label = QtWidgets.QLabel(parent=self)
+        label.setStyleSheet(
+            f"""QLabel {{
+                color : {hcolor('default_lightest')};
+                font-size : {_font.px_size - 2}px;
+            }}
+            """
+        )
         label.setTextFormat(3)  # markdown
         label.setFont(_font.font)
         label.setMargin(4)
-        label.setText("`search`:")
+        label.setText("search:")
         label.show()
         label.setAlignment(
             QtCore.Qt.AlignVCenter
@@ -540,8 +542,8 @@ class SearchWidget(QtWidgets.QWidget):
         )
         self.bar = SearchBar(
             parent=self,
-            parent_chart=godwidget,
             view=self.view,
+            godwidget=godwidget,
         )
         self.bar_hbox.addWidget(self.bar)
 
@@ -564,7 +566,7 @@ class SearchWidget(QtWidgets.QWidget):
         self.bar.focus()
         self.show()
 
-    def get_current_item(self) -> Optional[Tuple[str, str]]:
+    def get_current_item(self) -> Optional[tuple[str, str]]:
         '''Return the current completer tree selection as
         a tuple ``(parent: str, child: str)`` if valid, else ``None``.
 
@@ -596,14 +598,15 @@ class SearchWidget(QtWidgets.QWidget):
         else:
             return None
 
-    def chart_current_item(
+    async def chart_current_item(
         self,
         clear_to_cache: bool = True,
+
     ) -> Optional[str]:
         '''Attempt to load and switch the current selected
         completion result to the affiliated chart app.
 
-        Return any loaded symbol
+        Return any loaded symbol.
 
         '''
         value = self.get_current_item()
@@ -615,7 +618,7 @@ class SearchWidget(QtWidgets.QWidget):
 
         log.info(f'Requesting symbol: {symbol}.{provider}')
 
-        chart.load_symbol(
+        await chart.load_symbol(
             provider,
             symbol,
             'info',
@@ -653,10 +656,11 @@ async def pack_matches(
 
     view: CompleterView,
     has_results: dict[str, set[str]],
-    matches: dict[(str, str), List[str]],
+    matches: dict[(str, str), list[str]],
     provider: str,
     pattern: str,
     search: Callable[..., Awaitable[dict]],
+
     task_status: TaskStatus[
         trio.CancelScope] = trio.TASK_STATUS_IGNORED,
 
@@ -834,7 +838,7 @@ async def handle_keyboard_input(
     # startup
     bar = searchbar
     search = searchbar.parent()
-    chart = search.godwidget
+    godwidget = search.godwidget
     view = bar.view
     view.set_font_size(bar.dpi_font.px_size)
 
@@ -853,7 +857,8 @@ async def handle_keyboard_input(
             )
         )
 
-        async for event, etype, key, mods, txt in recv_chan:
+        async for kbmsg in recv_chan:
+            event, etype, key, mods, txt = kbmsg.to_tuple()
 
             log.debug(f'key: {key}, mods: {mods}, txt: {txt}')
 
@@ -861,14 +866,9 @@ async def handle_keyboard_input(
             if mods == Qt.ControlModifier:
                 ctl = True
 
-            # # ctl + alt as combo
-            # ctlalt = False
-            # if (QtCore.Qt.AltModifier | QtCore.Qt.ControlModifier) == mods:
-            #     ctlalt = True
-
             if key in (Qt.Key_Enter, Qt.Key_Return):
 
-                search.chart_current_item(clear_to_cache=True)
+                await search.chart_current_item(clear_to_cache=True)
                 _search_enabled = False
                 continue
 
@@ -876,7 +876,7 @@ async def handle_keyboard_input(
                 # if nothing in search text show the cache
                 view.set_section_entries(
                     'cache',
-                    list(reversed(chart._chart_cache)),
+                    list(reversed(godwidget._chart_cache)),
                     clear_all=True,
                 )
                 continue
@@ -890,8 +890,8 @@ async def handle_keyboard_input(
                 search.bar.unfocus()
 
                 # kill the search and focus back on main chart
-                if chart:
-                    chart.linkedsplits.focus()
+                if godwidget:
+                    godwidget.focus()
 
                 continue
 
@@ -938,7 +938,7 @@ async def handle_keyboard_input(
                     if parent_item and parent_item.text() == 'cache':
 
                         # if it's a cache item, switch and show it immediately
-                        search.chart_current_item(clear_to_cache=False)
+                        await search.chart_current_item(clear_to_cache=False)
 
             elif not ctl:
                 # relay to completer task
@@ -950,7 +950,7 @@ async def handle_keyboard_input(
 async def search_simple_dict(
     text: str,
     source: dict,
-) -> Dict[str, Any]:
+) -> dict[str, Any]:
 
     # search routine can be specified as a function such
     # as in the case of the current app's local symbol cache
@@ -964,7 +964,7 @@ async def search_simple_dict(
 
 
 # cache of provider names to async search routines
-_searcher_cache: Dict[str, Callable[..., Awaitable]] = {}
+_searcher_cache: dict[str, Callable[..., Awaitable]] = {}
 
 
 @asynccontextmanager

piker/ui/_style.py

@@ -14,14 +14,16 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-"""
+'''
 Qt UI styling.
-"""
+
+'''
 from typing import Optional, Dict
 import math
 
 import pyqtgraph as pg
 from PyQt5 import QtCore, QtGui
+from PyQt5.QtCore import Qt, QCoreApplication
 from qdarkstyle import DarkPalette
 
 from ..log import get_logger
@@ -56,7 +58,6 @@ class DpiAwareFont:
         self._qfont = QtGui.QFont(name)
         self._font_size: str = font_size
         self._qfm = QtGui.QFontMetrics(self._qfont)
-        self._physical_dpi = None
         self._font_inches: float = None
         self._screen = None
 
@@ -82,6 +83,10 @@ class DpiAwareFont:
     def font(self):
         return self._qfont
 
+    def scale(self) -> float:
+        screen = self.screen
+        return screen.logicalDotsPerInch() / screen.physicalDotsPerInch()
+
     @property
     def px_size(self) -> int:
         return self._qfont.pixelSize()
@@ -99,9 +104,15 @@ class DpiAwareFont:
         # take the max since scaling can make things ugly in some cases
         pdpi = screen.physicalDotsPerInch()
         ldpi = screen.logicalDotsPerInch()
+
+        # XXX: this is needed on sway/wayland where you set
+        # ``QT_WAYLAND_FORCE_DPI=physical``
+        if ldpi == 0:
+            ldpi = pdpi
+
         mx_dpi = max(pdpi, ldpi)
         mn_dpi = min(pdpi, ldpi)
-        scale = round(ldpi/pdpi)
+        scale = round(ldpi/pdpi, ndigits=2)
 
         if mx_dpi <= 97:  # for low dpi use larger font sizes
             inches = _font_sizes['lo'][self._font_size]
@@ -111,18 +122,44 @@ class DpiAwareFont:
 
         dpi = mn_dpi
 
+        mult = 1.0
+
+        # No implicit DPI scaling was done by the DE so let's engage
+        # some hackery ad-hoc scaling shiat.
         # dpi is likely somewhat scaled down so use slightly larger font size
-        if scale > 1 and self._font_size:
-            # TODO: this denominator should probably be determined from
-            # relative aspect rations or something?
-            inches = inches * (1 / scale) * (1 + 6/16)
-            dpi = mx_dpi
+        if scale >= 1.1 and self._font_size:
 
+            # no idea why
+            if 1.2 <= scale:
+                mult = 1.0375
+
+            if scale >= 1.5:
+                mult = 1.375
+
+        # TODO: this multiplier should probably be determined from
+        # relative aspect ratios or something?
+        inches *= mult
+
+        # XXX: if additionally we detect a known DE scaling factor we
+        # also scale *up* our font size on top of the existing
+        # heuristical (aka no clue why it works) scaling from the block
+        # above XD
+        if (
+            hasattr(Qt, 'AA_EnableHighDpiScaling')
+            and QCoreApplication.testAttribute(Qt.AA_EnableHighDpiScaling)
+        ):
+            inches *= round(scale)
+
+        # TODO: we might want to fiddle with incrementing font size by
+        # +1 for the edge cases above. it seems doing it via scaling is
+        # always going to hit that error in range mapping from inches:
+        # float to px size: int.
         self._font_inches = inches
-
         font_size = math.floor(inches * dpi)
-        log.info(
-            f"\nscreen:{screen.name()} with pDPI: {pdpi}, lDPI: {ldpi}"
+
+        log.debug(
+            f"screen:{screen.name()}\n"
+            f"pDPI: {pdpi}, lDPI: {ldpi}, scale: {scale}\n"
             f"\nOur best guess font size is {font_size}\n"
         )
         # apply the size
@@ -166,8 +203,6 @@ _xaxis_at = 'bottom'
 # charting config
 CHART_MARGINS = (0, 0, 2, 2)
 _min_points_to_show = 6
-_bars_to_left_in_follow_mode = int(61*6)
-_bars_from_right_in_follow_mode = round(0.16 * _bars_to_left_in_follow_mode)
 _tina_mode = False
 
 
@@ -196,19 +231,26 @@ def hcolor(name: str) -> str:
         'svags': '#0a0e14',
 
         # fifty shades
+        'original': '#a9a9a9',
         'gray': '#808080',  # like the kick
         'grayer': '#4c4c4c',
         'grayest': '#3f3f3f',
-        'i3': '#494D4F',
-        'jet': '#343434',
         'cadet': '#91A3B0',
         'marengo': '#91A3B0',
-        'charcoal': '#36454F',
         'gunmetal': '#91A3B0',
         'battleship': '#848482',
-        'davies': '#555555',
+
+        # bluish
+        'charcoal': '#36454F',
+
+        # default bars
         'bracket': '#666666',  # like the logo
-        'original': '#a9a9a9',
+
+        # work well for filled polygons which want a 'bracket' feel
+        # going light to dark
+        'davies': '#555555',
+        'i3': '#494D4F',
+        'jet': '#343434',
 
         # from ``qdarkstyle`` palette
         'default_darkest': DarkPalette.COLOR_BACKGROUND_1,

piker/ui/_window.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for piker0)
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -25,7 +25,7 @@ from typing import Callable, Optional, Union
 import uuid
 
 from pyqtgraph import QtGui
-from PyQt5 import QtCore, QtWidgets
+from PyQt5 import QtCore
 from PyQt5.QtWidgets import QLabel, QStatusBar
 
 from ..log import get_logger
@@ -55,7 +55,8 @@ class MultiStatus:
         group_key: Optional[Union[bool, str]] = False,
 
     ) -> Union[Callable[..., None], str]:
-        '''Add a status to the status bar and return a close callback which
+        '''
+        Add a status to the status bar and return a close callback which
         when called will remove the status ``msg``.
 
         '''
@@ -124,8 +125,10 @@ class MultiStatus:
                     if not subs:
                         group_clear()
 
-            self._status_groups[group_key][0].add(msg)
-            ret = pop_from_group_and_maybe_clear_group
+            group = self._status_groups.get(group_key)
+            if group:
+                group[0].add(msg)
+                ret = pop_from_group_and_maybe_clear_group
 
         self.render()
 
@@ -135,7 +138,8 @@ class MultiStatus:
         return ret
 
     def render(self) -> None:
-        '''Display all open statuses to bar.
+        '''
+        Display all open statuses to bar.
 
         '''
         if self.statuses:
@@ -146,16 +150,21 @@ class MultiStatus:
 
 class MainWindow(QtGui.QMainWindow):
 
-    size = (800, 500)
+    # XXX: for tiling wms this should scale
+    # with the alloted window size.
+    # TODO: detect for tiling and if untrue set some size?
+    size = (300, 500)
+
     title = 'piker chart (ur symbol is loading bby)'
 
     def __init__(self, parent=None):
         super().__init__(parent)
-        self.setMinimumSize(*self.size)
+        # self.setMinimumSize(*self.size)
         self.setWindowTitle(self.title)
 
         self._status_bar: QStatusBar = None
         self._status_label: QLabel = None
+        self._size: Optional[tuple[int, int]] = None
 
     @property
     def mode_label(self) -> QtGui.QLabel:
@@ -165,7 +174,11 @@ class MainWindow(QtGui.QMainWindow):
 
             self._status_label = label = QtGui.QLabel()
             label.setStyleSheet(
-                f"QLabel {{ color : {hcolor('gunmetal')}; }}"
+                f"""QLabel {{
+                    color : {hcolor('gunmetal')};
+                }}
+                """
+                # font-size : {font_size}px;
             )
             label.setTextFormat(3)  # markdown
             label.setFont(_font_small.font)
@@ -181,11 +194,13 @@ class MainWindow(QtGui.QMainWindow):
 
     def closeEvent(
         self,
-        event: QtGui.QCloseEvent,
-    ) -> None:
-        """Cancel the root actor asap.
 
-        """
+        event: QtGui.QCloseEvent,
+
+    ) -> None:
+        '''Cancel the root actor asap.
+
+        '''
         # raising KBI seems to get intercepted by by Qt so just use the system.
         os.kill(os.getpid(), signal.SIGINT)
 
@@ -209,18 +224,28 @@ class MainWindow(QtGui.QMainWindow):
 
         return self._status_bar
 
-    def on_focus_change(
+    def set_mode_name(
         self,
-        old: QtGui.QWidget,
-        new: QtGui.QWidget,
+        name: str,
+
     ) -> None:
 
-        log.debug(f'widget focus changed from {old} -> {new}')
+        self.mode_label.setText(f'mode:{name}')
 
-        if new is not None:
+    def on_focus_change(
+        self,
+
+        last: QtGui.QWidget,
+        current: QtGui.QWidget,
+
+    ) -> None:
+
+        log.info(f'widget focus changed from {last} -> {current}')
+
+        if current is not None:
             # cursor left window?
-            name = getattr(new, 'mode_name', '')
-            self.mode_label.setText(name)
+            name = getattr(current, 'mode_name', '')
+            self.set_mode_name(name)
 
     def current_screen(self) -> QtGui.QScreen:
         """Get a frickin screen (if we can, gawd).
@@ -230,7 +255,7 @@ class MainWindow(QtGui.QMainWindow):
 
         for _ in range(3):
             screen = app.screenAt(self.pos())
-            print('trying to access QScreen...')
+            log.debug('trying to access QScreen...')
             if screen is None:
                 time.sleep(0.5)
                 continue
@@ -244,6 +269,29 @@ class MainWindow(QtGui.QMainWindow):
         assert screen, "Wow Qt is dumb as shit and has no screen..."
         return screen
 
+    def configure_to_desktop(
+        self,
+        size: Optional[tuple[int, int]] = None,
+
+    ) -> None:
+        '''
+        Explicitly size the window dimensions (for stacked window
+        managers).
+
+        For tina systems (like windoze) try to do a sane window size on
+        startup.
+
+        '''
+        # https://stackoverflow.com/a/18975846
+        if not size and not self._size:
+            app = QtGui.QApplication.instance()
+            geo = self.current_screen().geometry()
+            h, w = geo.height(), geo.width()
+            # use approx 1/3 of the area of the screen by default
+            self._size = round(w * .666), round(h * .666)
+
+        self.resize(*size or self._size)
+
 
 # singleton app per actor
 _qt_win: QtGui.QMainWindow = None

piker/ui/cli.py

@@ -122,7 +122,8 @@ def optschain(config, symbol, date, rate, test):
 @cli.command()
 @click.option(
     '--profile',
-    is_flag=True,
+    '-p',
+    default=None,
     help='Enable pyqtgraph profiling'
 )
 @click.option(
@@ -133,10 +134,17 @@ def optschain(config, symbol, date, rate, test):
 @click.argument('symbol', required=True)
 @click.pass_obj
 def chart(config, symbol, profile, pdb):
-    """Start a real-time chartng UI
-    """
-    from .. import _profile
-    from ._chart import _main
+    '''
+    Start a real-time chartng UI
+
+    '''
+    # eg. ``--profile 3`` reports profiling for anything slower then 3 ms.
+    if profile is not None:
+        from .. import _profile
+        _profile._pg_profile = True
+        _profile.ms_slower_then = float(profile)
+
+    from ._app import _main
 
     if '.' not in symbol:
         click.echo(click.style(
@@ -145,8 +153,6 @@ def chart(config, symbol, profile, pdb):
         ))
         return
 
-    # toggle to enable profiling
-    _profile._pg_profile = profile
 
     # global opts
     brokernames = config['brokers']

requirements.txt

@@ -1,3 +1,21 @@
-# no pypi package for tractor (yet)
-# we require the asyncio-via-guest-mode dev branch
--e git+git://github.com/goodboy/tractor.git@infect_asyncio#egg=tractor
+# we require a pinned dev branch to get some edge features that
+# are often untested in tractor's CI and/or being tested by us
+# first before committing as core features in tractor's base.
+-e git+https://github.com/goodboy/tractor.git@master#egg=tractor
+
+# `pyqtgraph` peeps keep breaking, fixing, improving so might as well
+# pin this to a dev branch that we have more control over especially
+# as more graphics stuff gets hashed out.
+-e git+https://github.com/pikers/pyqtgraph.git@piker_pin#egg=pyqtgraph
+
+
+# our async client for ``marketstore`` (the tsdb)
+-e git+https://github.com/pikers/anyio-marketstore.git@master#egg=anyio-marketstore
+
+
+# ``trimeter`` for asysnc history fetching
+-e git+https://github.com/python-trio/trimeter.git@master#egg=trimeter
+
+
+# ``asyncvnc`` for sending interactions to ib-gw inside docker
+-e git+https://github.com/pikers/asyncvnc.git@vid_passthrough#egg=asyncvnc

scripts/ib_data_reset.py

@@ -0,0 +1,84 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+IB api client data feed reset hack for i3.
+
+'''
+import subprocess
+
+import i3ipc
+
+i3 = i3ipc.Connection()
+t = i3.get_tree()
+
+orig_win_id = t.find_focused().window
+
+# for tws
+win_names: list[str] = [
+    'Interactive Brokers',  # tws running in i3
+    'IB Gateway',  # gw running in i3
+    # 'IB',  # gw running in i3 (newer version?)
+]
+
+for name in win_names:
+    results = t.find_titled(name)
+    print(f'results for {name}: {results}')
+    if results:
+        con = results[0]
+        print(f'Resetting data feed for {name}')
+        win_id = str(con.window)
+        w, h = con.rect.width, con.rect.height
+
+        # TODO: seems to be a few libs for python but not sure
+        # if they support all the sub commands we need, order of
+        # most recent commit history:
+        # https://github.com/rr-/pyxdotool
+        # https://github.com/ShaneHutter/pyxdotool
+        # https://github.com/cphyc/pyxdotool
+
+        # TODO: only run the reconnect (2nd) kc on a detected
+        # disconnect?
+        for key_combo, timeout in [
+            # only required if we need a connection reset.
+            # ('ctrl+alt+r', 12),
+            # data feed reset.
+            ('ctrl+alt+f', 6)
+        ]:
+            subprocess.call([
+                'xdotool',
+                'windowactivate', '--sync', win_id,
+
+                # move mouse to bottom left of window (where there should
+                # be nothing to click).
+                'mousemove_relative', '--sync', str(w-4), str(h-4),
+
+                # NOTE: we may need to stick a `--retry 3` in here..
+                'click', '--window', win_id,
+                '--repeat', '3', '1',
+
+                # hackzorzes
+                'key', key_combo,
+                ],
+                timeout=timeout,
+            )
+
+# re-activate and focus original window
+subprocess.call([
+    'xdotool',
+    'windowactivate', '--sync', str(orig_win_id),
+    'click', '--window', str(orig_win_id), '1',
+])

setup.py

@@ -51,42 +51,56 @@ setup(
         # async
         'trio',
         'trio-websocket',
-        # 'tractor',  # from github currently
+        'msgspec',  # performant IPC messaging
         'async_generator',
 
+        # from github currently (see requirements.txt)
+        # 'trimeter',  # not released yet..
+        # 'tractor',
+        # asyncvnc,
+
         # brokers
         'asks==2.4.8',
         'ib_insync',
 
         # numerics
-        'arrow',  # better datetimes
+        'pendulum', # easier datetimes
         'bidict',  # 2 way map
         'cython',
         'numpy',
         'numba',
-        'pandas',
-        'msgpack-numpy',
 
         # UI
         'PyQt5',
-        'pyqtgraph',
-        'qdarkstyle >= 3.0.2',
-        # fuzzy search
-        'fuzzywuzzy[speedup]',
+        # 'pyqtgraph',  from our fork see reqs.txt
+        'qdarkstyle >= 3.0.2',  # themeing
+        'fuzzywuzzy[speedup]',  # fuzzy search
 
         # tsdbs
-        'pymarketstore',
+        # anyio-marketstore  # from gh see reqs.txt
     ],
+    extras_require={
+        'tsdb': [
+            'docker',
+        ],
+
+    },
     tests_require=['pytest'],
-    python_requires=">=3.9",  # literally for ``datetime.datetime.fromisoformat``...
-    keywords=["async", "trading", "finance", "quant", "charting"],
+    python_requires=">=3.10",
+    keywords=[
+        "async",
+        "trading",
+        "finance",
+        "quant",
+        "charting",
+    ],
     classifiers=[
         'Development Status :: 3 - Alpha',
         'License :: OSI Approved :: ',
         'Operating System :: POSIX :: Linux',
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         'Intended Audience :: Financial and Insurance Industry',
         'Intended Audience :: Science/Research',
         'Intended Audience :: Developers',

tests/conftest.py

@@ -3,8 +3,8 @@ import os
 import pytest
 import tractor
 import trio
-from piker import log
-from piker.brokers import questrade, config
+from piker import log, config
+from piker.brokers import questrade
 
 
 def pytest_addoption(parser):